diff options
| author | Chris Evich <cevich@redhat.com> | 2020-06-25 17:38:33 -0400 |
|---|---|---|
| committer | Chris Evich <cevich@redhat.com> | 2020-10-02 11:53:04 -0400 |
| commit | 2c9084e2245834094e14e9105e64b6062d70a0f1 (patch) | |
| tree | e2e5c0be702cdae5919489fd9a1ed1b6f87d135a | |
| parent | b58980a43ccfcef5134274c3963bf68cc51a4983 (diff) | |
| download | podman-2c9084e2245834094e14e9105e64b6062d70a0f1.tar.gz podman-2c9084e2245834094e14e9105e64b6062d70a0f1.tar.bz2 podman-2c9084e2245834094e14e9105e64b6062d70a0f1.zip | |
Cirrus: Implement podman automation 2.0
Reimplement CI-automation to remove accumulated technical-debt and
optimize workflow. The task-dependency graph designed goal was to
shorten it's depth and increase width (i.e. more parallelism). A
reduction in redundant building (and 3rd party module download) was
also realized by caching `$GOPATH` and `$GOCACHE` early on. This
cache is then reused in favor of a fresh clone of the repository
(when possible).
Note: The system tests typically execute MUCH faster than the
integration tests. However, contrary to a fail-fast/fail-early
principal, they are executed last. This was implemented due to
debug-ability related concerns/preferences of the primary
(golang-centric) project developers.
Signed-off-by: Chris Evich <cevich@redhat.com>
26 files changed, 1179 insertions, 1787 deletions
diff --git a/.cirrus.yml b/.cirrus.yml index 6f8cd0a30..75f095fe0 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -1,13 +1,5 @@ --- -# Only github users with write-access can define or use encrypted variables -# Ref: https://cirrus-ci.org/guide/writing-tasks/#encrypted-variables -# more specifically: https://cirrus-ci.com/settings/repository/6707778565701632 -gcp_credentials: ENCRYPTED[a28959877b2c9c36f151781b0a05407218cda646c7d047fc556e42f55e097e897ab63ee78369dae141dcf0b46a9d0cdd] - -# Default timeout for each task -timeout_in: 120m - # Main collection of env. vars to set for all tasks and scripts. env: #### @@ -16,603 +8,643 @@ env: # Name of the ultimate destination branch for this CI run, PR or post-merge. DEST_BRANCH: "master" # Overrides default location (/tmp/cirrus) for repo clone - GOPATH: "/var/tmp/go" + GOPATH: &gopath "/var/tmp/go" GOBIN: "${GOPATH}/bin" - GOSRC: "/var/tmp/go/src/github.com/containers/podman" - CIRRUS_WORKING_DIR: "/var/tmp/go/src/github.com/containers/podman" + GOCACHE: "${GOPATH}/cache" + GOSRC: &gosrc "/var/tmp/go/src/github.com/containers/podman" + CIRRUS_WORKING_DIR: *gosrc # The default is 'sh' if unspecified CIRRUS_SHELL: "/bin/bash" # Save a little typing (path relative to $CIRRUS_WORKING_DIR) SCRIPT_BASE: "./contrib/cirrus" - # Command to prefix output lines with timing information - # (can't do inline awk script, Cirrus-CI or YAML mangles quoting) - TIMESTAMP: "awk --file ${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/timestamp.awk" - - # HTMLify ginkgo and bats logs - LOGFORMAT: "${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/logformatter" #### #### Cache-image names to test with (double-quotes around names are critical) - ### + #### FEDORA_NAME: "fedora-32" PRIOR_FEDORA_NAME: "fedora-31" UBUNTU_NAME: "ubuntu-20" PRIOR_UBUNTU_NAME: "ubuntu-19" - _BUILT_IMAGE_SUFFIX: "c4948709391728640" - FEDORA_CACHE_IMAGE_NAME: "fedora-${_BUILT_IMAGE_SUFFIX}" - PRIOR_FEDORA_CACHE_IMAGE_NAME: "prior-fedora-${_BUILT_IMAGE_SUFFIX}" - UBUNTU_CACHE_IMAGE_NAME: "ubuntu-${_BUILT_IMAGE_SUFFIX}" - PRIOR_UBUNTU_CACHE_IMAGE_NAME: "prior-ubuntu-${_BUILT_IMAGE_SUFFIX}" + # Google-cloud VM Images + IMAGE_SUFFIX: "c4948709391728640" + FEDORA_CACHE_IMAGE_NAME: "fedora-${IMAGE_SUFFIX}" + PRIOR_FEDORA_CACHE_IMAGE_NAME: "prior-fedora-${IMAGE_SUFFIX}" + UBUNTU_CACHE_IMAGE_NAME: "ubuntu-${IMAGE_SUFFIX}" + PRIOR_UBUNTU_CACHE_IMAGE_NAME: "prior-ubuntu-${IMAGE_SUFFIX}" - #### - #### Default to NOT operating in any special-case testing mode - #### - SPECIALMODE: "none" # don't do anything special - RCLI: 'false' # don't test remote client by default - ADD_SECOND_PARTITION: 'false' # will certainly fail inside containers - MOD_CONTAINERS_CONF: 'true' # Update containers.conf runtime if required by OS environment + # Container FQIN's + FEDORA_CONTAINER_FQIN: "quay.io/libpod/fedora_podman:${IMAGE_SUFFIX}" + PRIOR_FEDORA_CONTAINER_FQIN: "quay.io/libpod/prior-fedora_podman:${IMAGE_SUFFIX}" + UBUNTU_CONTAINER_FQIN: "quay.io/libpod/ubuntu_podman:${IMAGE_SUFFIX}" + PRIOR_UBUNTU_CONTAINER_FQIN: "quay.io/libpod/prior-ubuntu_podman:${IMAGE_SUFFIX}" #### - #### Credentials and other secret-sauces, decrypted at runtime when authorized. + #### Control variables that determine what to run and how to run it. + #### N/B: Required ALL of these are set for every single task. #### - # Name where this repositories VM images are stored - GCP_PROJECT_ID: libpod-218412 - # Service-account client_email - needed to build images - SERVICE_ACCOUNT: ENCRYPTED[702a8e07e27a6faf7988fcddcc068c2ef2bb182a5aa671f5ccb7fbbfb891c823aa4a7856fb17240766845dbd68bd3f90] - # Service account username part of client_email - for ssh'ing into VMs - GCE_SSH_USERNAME: 'cirrus-ci' - -# Default VM to use unless set or modified by task -gce_instance: - image_project: $GCP_PROJECT_ID - zone: "us-central1-a" # Required by Cirrus for the time being - cpu: 2 - memory: "4Gb" - disk: 200 - # A matrix could be used here, for now just one VM - image_name: "${FEDORA_CACHE_IMAGE_NAME}" - - -# Every *_task runs in parallel in separate VMsd. The name prefix only for reference -# in WebUI, and will be followed by matrix details. This task gates all others with -# quick format, lint, and unit tests on the standard platform. -gating_task: - - # Only run this on PRs, never during post-merge testing (for speed). - only_if: $CIRRUS_BRANCH != $DEST_BRANCH + TEST_FLAVOR: # int, sys, ext_svc, smoke, automation, etc. + TEST_ENVIRON: host # 'host' or 'container' + PODBIN_NAME: podman # 'podman' or 'remote' + PRIV_NAME: root # 'root' or 'rootless' + DISTRO_NV: $FEDORA_NAME # any {PRIOR_,}{FEDORA,UBUNTU}_NAME value + VM_IMAGE_NAME: # One of the "Google-cloud VM Images" (above) + CTR_FQIN: # One of the "Container FQIN's" (above) - env: - CIRRUS_WORKING_DIR: "/usr/src/libpod" - SRCPATH: "$CIRRUS_WORKING_DIR" - # Runs within Cirrus's "community cluster" - container: - # Note: Image has dual purpose, see contrib/gate/README.md - # The entrypoint.sh script ensures a prestine copy of $SRCPATH is - # available at $GOSRC before executing make instructions. - image: "quay.io/libpod/gate:master" - cpu: 8 - memory: 12 +# Default timeout for each task +timeout_in: 60m - timeout_in: 20m - # Custom cloning is required to satisfy lint/validation needs - clone_script: | +gcp_credentials: ENCRYPTED[a28959877b2c9c36f151781b0a05407218cda646c7d047fc556e42f55e097e897ab63ee78369dae141dcf0b46a9d0cdd] + + +# Attempt to prevent flakes by confirming all required external/3rd-party +# services are available and functional. +ext_svc_check_task: + alias: 'ext_svc_check' # int. ref. name - required for depends_on reference + name: "Ext. services" # Displayed Title - has no other significance + # Default/small container image to execute tasks with + container: &smallcontainer + image: ${CTR_FQIN} + # Resources are limited across ALL currently executing tasks + # ref: https://cirrus-ci.org/guide/linux/#linux-containers + cpu: 2 + memory: 2 + env: + TEST_FLAVOR: ext_svc + CTR_FQIN: ${FEDORA_CONTAINER_FQIN} + setup_script: &setup + - 'cd $GOSRC/$SCRIPT_BASE || exit 1' + - './setup_environment.sh' + main_script: &main + - 'cd $GOSRC/$SCRIPT_BASE || exit 1' + - './runner.sh' + + +# Execute some quick checks to confirm this YAML file and all +# automation-related shell scripts are sane. +automation_task: + alias: 'automation' + name: "Check Automation" + container: *smallcontainer + env: + TEST_FLAVOR: automation + CTR_FQIN: ${FEDORA_CONTAINER_FQIN} + TEST_ENVIRON: container + setup_script: *setup + main_script: *main + + +# This task use to be called 'gating', however that name is being +# used downstream for release testing. Renamed this to avoid confusion. +# All it does is run basic golang formatting and commit validation checks. +smoke_task: + alias: 'smoke' + name: "Smoke Test" + container: &bigcontainer + image: ${CTR_FQIN} + # Leave some resources for smallcontainer + cpu: 6 + memory: 22 + env: + TEST_FLAVOR: 'smoke' + CTR_FQIN: "${FEDORA_CONTAINER_FQIN}" + TEST_ENVIRON: container + # This clone script is also used to initially populate gopath_cache (below) + clone_script: &full_clone | + cd / + rm -rf $CIRRUS_WORKING_DIR + mkdir -p $CIRRUS_WORKING_DIR git clone --recursive --branch=$DEST_BRANCH https://x-access-token:${CIRRUS_REPO_CLONE_TOKEN}@github.com/${CIRRUS_REPO_FULL_NAME}.git $CIRRUS_WORKING_DIR + cd $CIRRUS_WORKING_DIR git remote update origin if [[ -n "$CIRRUS_PR" ]]; then # running for a PR git fetch origin pull/$CIRRUS_PR/head:pull/$CIRRUS_PR git checkout pull/$CIRRUS_PR + else + git reset --hard $CIRRUS_CHANGE_IN_REPO fi - git reset --hard $CIRRUS_CHANGE_IN_REPO - - # Don't bother going further if something is down - networking_script: '${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/networking.sh' - - gate_script: - # N/B: entrypoint.sh resets $GOSRC (same as make clean) - - '/usr/local/bin/entrypoint.sh install.tools |& ${TIMESTAMP}' - - '/usr/local/bin/entrypoint.sh validate |& ${TIMESTAMP}' - - # This task builds Podman with different buildtags to ensure the build does - # not break. It also verifies all sub-commands have man pages. - build_script: - - '/usr/local/bin/entrypoint.sh podman |& ${TIMESTAMP}' - - 'cd $GOSRC && ./hack/podman-commands.sh |& ${TIMESTAMP}' - # N/B: need 'clean' so some committed files are re-generated. - - '/usr/local/bin/entrypoint.sh clean podman-remote |& ${TIMESTAMP}' - - '/usr/local/bin/entrypoint.sh clean podman xref_helpmsgs_manpages BUILDTAGS="exclude_graphdriver_devicemapper selinux seccomp" |& ${TIMESTAMP}' - - '/usr/local/bin/entrypoint.sh clean BUILDTAGS="varlink" binaries |& ${TIMESTAMP}' - - '/usr/local/bin/entrypoint.sh local-cross |& ${TIMESTAMP}' - - # Verify some aspects of ci/related scripts - ci_script: - - '${GOSRC}/${SCRIPT_BASE}/lib.sh.t |& ${TIMESTAMP}' - - '${GOSRC}/${SCRIPT_BASE}/cirrus_yaml_test.py |& ${TIMESTAMP}' - - # Verify expected bash environment (-o pipefail) - pipefail_enabledscript: 'if /bin/false | /bin/true; then echo "pipefail fault" && exit 72; fi' - - -# Ensure these container images can build -container_image_build_task: - alias: 'container_image_build' - name: "build gate image $DEST_BRANCH branch" + cd $CIRRUS_WORKING_DIR + make install.tools + setup_script: *setup + main_script: *main + + +# N/B: This task is critical. It builds all binaries and release archives +# for the project, using all primary OS platforms and versions. Assuming +# the builds are successful, a cache is stored of the entire `$GOPATH` +# contents. For all subsequent tasks, the _BUILD_CACHE_HANDLE value +# is used as a key to reuse this cache, saving both time and money. +# The only exceptions are tasks which only run inside a container, they +# will not have access the cache and therefore must rely on cloning the +# repository. +build_task: + alias: 'build' + name: 'Build for $DISTRO_NV' + gce_instance: &standardvm + image_project: libpod-218412 + zone: "us-central1-a" + cpu: 2 + memory: "4Gb" + # Required to be 200gig, do not modify - has i/o performance impact + # according to gcloud CLI tool warning messages. + disk: 200 + image_name: "${VM_IMAGE_NAME}" # from stdenvars + matrix: &platform_axis + # Ref: https://cirrus-ci.org/guide/writing-tasks/#matrix-modification + - env: &stdenvars + DISTRO_NV: ${FEDORA_NAME} + # Not used here, is used in other tasks + VM_IMAGE_NAME: ${FEDORA_CACHE_IMAGE_NAME} + CTR_FQIN: ${FEDORA_CONTAINER_FQIN} + # ID for re-use of build output + _BUILD_CACHE_HANDLE: ${FEDORA_NAME}-build-${CIRRUS_BUILD_ID} + - env: + DISTRO_NV: ${PRIOR_FEDORA_NAME} + VM_IMAGE_NAME: ${PRIOR_FEDORA_CACHE_IMAGE_NAME} + CTR_FQIN: ${PRIOR_FEDORA_CONTAINER_FQIN} + _BUILD_CACHE_HANDLE: ${PRIOR_FEDORA_NAME}-build-${CIRRUS_BUILD_ID} + - env: + DISTRO_NV: ${UBUNTU_NAME} + VM_IMAGE_NAME: ${UBUNTU_CACHE_IMAGE_NAME} + CTR_FQIN: ${UBUNTU_CONTAINER_FQIN} + _BUILD_CACHE_HANDLE: ${UBUNTU_NAME}-build-${CIRRUS_BUILD_ID} + - env: + DISTRO_NV: ${PRIOR_UBUNTU_NAME} + VM_IMAGE_NAME: ${PRIOR_UBUNTU_CACHE_IMAGE_NAME} + CTR_FQIN: ${PRIOR_UBUNTU_CONTAINER_FQIN} + _BUILD_CACHE_HANDLE: ${PRIOR_UBUNTU_NAME}-build-${CIRRUS_BUILD_ID} + env: + TEST_FLAVOR: build + # Ref: https://cirrus-ci.org/guide/writing-tasks/#cache-instruction + gopath_cache: &gopath_cache + folder: *gopath # Required hard-coded path, no variables. + fingerprint_script: echo "$_BUILD_CACHE_HANDLE" + # Cheat: Clone here when cache is empty, guaranteeing consistency. + populate_script: *full_clone + # A normal clone would invalidate useful cache + clone_script: &noop mkdir -p $CIRRUS_WORKING_DIR + setup_script: *setup + main_script: *main + always: &artifacts + gosrc_artifacts: + path: ./* # Grab everything in top-level $GOSRC + type: application/octet-stream + binary_artifacts: + path: ./bin/* + type: application/octet-stream + # Required for `contrib/cirrus/logformatter` to work properly + html_artifacts: + path: ./*.html + type: text/html + +# Confirm the result of building on at least one platform appears sane. +# This confirms the binaries can be executed, checks --help vs docs, and +# other essential post-build validation checks. +validate_task: + name: "Validate $DISTRO_NV Build" + alias: validate depends_on: - - "gating" - - # Only run for PRs, quay.io will automatically build after branch-push - only_if: $CIRRUS_BRANCH != $DEST_BRANCH - - container: - dockerfile: contrib/gate/Dockerfile - - script: make install.remote - - -# This task checks to make sure that we can still build an rpm from the -# source code using contrib/rpm/podman.spec.in -rpmbuild_task: - - only_if: >- - $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*' && - $CIRRUS_BRANCH != $DEST_BRANCH - + - ext_svc_check + - automation + - smoke + - build + # golangci-lint is a very, very hungry beast. + gce_instance: &bigvm + <<: *standardvm + cpu: 8 + memory: "16Gb" + env: + <<: *stdenvars + TEST_FLAVOR: validate + gopath_cache: &ro_gopath_cache + <<: *gopath_cache + reupload_on_changes: false + clone_script: *noop + setup_script: *setup + main_script: *main + always: *artifacts + +# Exercise the "libpod" API with a small set of common +# operations to ensure they are functional. +bindings_task: + name: "Test Bindings" + alias: bindings depends_on: - - "gating" + - build + gce_instance: *standardvm env: - CIRRUS_WORKING_DIR: "/usr/src/libpod" - - container: - image: quay.io/libpod/rpmbuild:$DEST_BRANCH - cpu: 2 - memory: 4 - - rpmbuild_script: - - 'make -C ${CIRRUS_WORKING_DIR} -f ${CIRRUS_WORKING_DIR}/.copr/Makefile' - - 'rpmbuild --rebuild ${CIRRUS_WORKING_DIR}/podman-*.src.rpm' - -# This task runs `make vendor` followed by ./hack/tree_status.sh to check -# whether the git tree is clean. The reasoning for that is to make sure -# that the vendor.conf, the code and the vendored packages in ./vendor are -# in sync at all times. + <<: *stdenvars + TEST_FLAVOR: bindings + gopath_cache: *ro_gopath_cache + clone_script: *noop # Comes from cache + setup_script: *setup + main_script: *main + always: *artifacts + + +# Build the "libpod" API documentation `swagger.yaml` for eventual +# publishing along side the official podman documentation. +swagger_task: + name: "Test Swagger" + alias: swagger + depends_on: + - build + container: *smallcontainer + env: + <<: *stdenvars + TEST_FLAVOR: swagger + TEST_ENVIRON: container + CTR_FQIN: ${FEDORA_CONTAINER_FQIN} + clone_script: *full_clone # build-cache not available to container tasks + setup_script: *setup + main_script: *main + always: *artifacts + + +endpoint_task: + name: "Test Endpoint" + alias: endpoint + depends_on: + - build + container: *smallcontainer + env: + <<: *stdenvars + TEST_FLAVOR: endpoint + TEST_ENVIRON: container + CTR_FQIN: ${FEDORA_CONTAINER_FQIN} + clone_script: *full_clone # build-cache not available to container tasks + setup_script: *setup + main_script: *main + always: *artifacts + + +# Check that all included go modules from other sources match +# what is expected in `vendor/modules.txt` vs `go.mod`. vendor_task: - - only_if: $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*' - + name: "Test Vendoring" + alias: vendor depends_on: - - "gating" - + - build + container: *smallcontainer env: - CIRRUS_WORKING_DIR: "/var/tmp/go/src/github.com/containers/podman" - GOPATH: "/var/tmp/go" - GOSRC: "$CIRRUS_WORKING_DIR" - - # Runs within Cirrus's "community cluster" - container: - image: docker.io/library/golang:1.13 - cpu: 4 - memory: 12 - - timeout_in: 30m - - vendor_script: - - 'cd ${CIRRUS_WORKING_DIR} && make vendor' - - 'cd ${CIRRUS_WORKING_DIR} && ./hack/tree_status.sh' - - -# This task runs `make varlink_api_generate` followed by ./hack/tree_status.sh to check -# whether the git tree is clean. -varlink_api_task: - - only_if: $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*' - + <<: *stdenvars + TEST_FLAVOR: vendor + TEST_ENVIRON: container + CTR_FQIN: ${FEDORA_CONTAINER_FQIN} + clone_script: *full_clone # build-cache not available to container tasks + setup_script: *setup + main_script: *main + always: *artifacts + + +# There are several other important variations of podman which +# must always build successfully. Most of them are handled in +# this task, though a few need dedicated tasks which follow. +alt_build_task: + name: "$ALT_NAME" + alias: alt_build depends_on: - - "gating" - + - build env: - CIRRUS_WORKING_DIR: "/usr/src/libpod" - SRCPATH: "$CIRRUS_WORKING_DIR" - EPOCH_TEST_COMMIT: "${CIRRUS_BASE_SHA}" # repo clone missing this data - # Used by tree_status.sh - SUGGESTION: 'remove API.md, then "make varlink_api_generate" and commit changes.' - - # Runs within Cirrus's "community cluster" - container: - # Note: Image has dual purpose, see contrib/gate/README.md - image: "quay.io/libpod/gate:master" - cpu: 4 - memory: 12 - - timeout_in: 10m - - api_md_script: - - '/usr/local/bin/entrypoint.sh BUILDTAGS="varlink" varlink_api_generate |& ${TIMESTAMP}' - - 'cd ${GOSRC} && ./hack/tree_status.sh |& ${TIMESTAMP}' - - -build_each_commit_task: - + <<: *stdenvars + TEST_FLAVOR: "altbuild" + gce_instance: *standardvm + matrix: + - env: + ALT_NAME: 'Build Each Commit' + - env: + ALT_NAME: 'Windows Cross' + - env: + ALT_NAME: 'Build Without CGO' + - env: + ALT_NAME: 'Build varlink-API' + - env: + ALT_NAME: 'Test build RPM' + - env: + ALT_NAME: 'Build varlink-binaries' + setup_script: *setup + main_script: *main + always: *artifacts + +# Confirm building a statically-linked binary is successful +static_alt_build_task: + name: "Static Build" + alias: static_alt_build depends_on: - - "gating" - - "vendor" - - "varlink_api" - - only_if: >- - $CIRRUS_BRANCH != $DEST_BRANCH && - $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*' - - gce_instance: - cpu: 8 - memory: "8Gb" - + - build + # Community-maintained task, may fail on occasion. If so, uncomment + # the next line and file an issue with details about the failure. + # allow_failures: $CI == $CI + gce_instance: *bigvm env: - MOD_CONTAINERS_CONF: 'false' - - timeout_in: 30m - - setup_environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}' - build_each_commit_script: - # set -x by default, no need to spew contents of lib.sh - - 'source $SCRIPT_BASE/lib.sh &> /dev/null' - - 'git fetch --depth 50 origin $DEST_BRANCH |& ${TIMESTAMP}' - - 'make build-all-new-commits GIT_BASE_BRANCH=origin/$DEST_BRANCH |& ${TIMESTAMP}' - + <<: *stdenvars + TEST_FLAVOR: "altbuild" + # gce_instance variation prevents this being included in alt_build_task + ALT_NAME: 'Static build' + # Do not use 'latest', fixed-version tag for runtime stability. + CTR_FQIN: "docker.io/nixos/nix:2.3.6" + # This is critical, it helps to avoid a very lengthy process of + # statically building every dependency needed to build podman. + # Assuming the dependency and build description hasn't changed, + # this cache ensures only the static podman binary is built. + nix_cache: + folder: '/var/cache/nix' + fingerprint_script: cat nix/* + setup_script: *setup + main_script: *main + always: *artifacts -build_without_cgo_task: +# Confirm building the remote client, natively on a Mac OS-X VM. +osx_alt_build_task: + name: "OSX Cross" + alias: osx_alt_build depends_on: - - "gating" - - "vendor" - - "varlink_api" - - only_if: >- - $CIRRUS_BRANCH != $DEST_BRANCH && - $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*' - - gce_instance: - cpu: 8 - memory: "8Gb" - + - build env: - MOD_CONTAINERS_CONF: 'false' - - timeout_in: 30m - - setup_environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}' - build_without_cgo_script: - - 'source $SCRIPT_BASE/lib.sh' - - 'make build-no-cgo' - - -# Update metadata on VM images referenced by this repository state -meta_task: - - container: - image: "quay.io/libpod/imgts:${_BUILT_IMAGE_SUFFIX}" - cpu: 1 - memory: 1 - - env: &meta_env_vars - # Space-separated list of images used by this repository state - IMGNAMES: >- - ${FEDORA_CACHE_IMAGE_NAME} - ${PRIOR_FEDORA_CACHE_IMAGE_NAME} - ${UBUNTU_CACHE_IMAGE_NAME} - ${PRIOR_UBUNTU_CACHE_IMAGE_NAME} - BUILDID: "${CIRRUS_BUILD_ID}" - REPOREF: "${CIRRUS_CHANGE_IN_REPO}" - GCPJSON: ENCRYPTED[3a198350077849c8df14b723c0f4c9fece9ebe6408d35982e7adf2105a33f8e0e166ed3ed614875a0887e1af2b8775f4] - # needed for output-masking purposes - GCPNAME: ENCRYPTED[2f9738ef295a706f66a13891b40e8eaa92a89e0e87faf8bed66c41eca72bf76cfd190a6f2d0e8444c631fdf15ed32ef6] - GCPPROJECT: $GCP_PROJECT_ID - - timeout_in: 10m - - # Cirrus-CI ignores entrypoint defined in image - script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/update_meta.sh |& ${TIMESTAMP}' - + <<: *stdenvars + # OSX platform variation prevents this being included in alt_build_task + TEST_FLAVOR: "altbuild" + ALT_NAME: 'OSX Cross' + osx_instance: + image: 'catalina-base' + script: + - brew install go + - brew install go-md2man + - make podman-remote-darwin + - make install-podman-remote-darwin-docs + always: *artifacts -# This task does the unit and integration testing for every platform -testing_task: - alias: "testing" +# This task is a stub: In the future it will be used to verify +# podman is compatible with the docker python-module. +docker-py_test_task: + name: Docker-py Compat. + alias: docker-py_test depends_on: - - "gating" - - "vendor" - - "varlink_api" - - "build_each_commit" - - "build_without_cgo" - - "container_image_build" - - # Only test build cache-images, if that's what's requested - only_if: $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*' - - matrix: - - name: "test ${FEDORA_NAME}" - gce_instance: - image_name: "${FEDORA_CACHE_IMAGE_NAME}" - - name: "test ${PRIOR_FEDORA_NAME}" - gce_instance: - image_name: "${PRIOR_FEDORA_CACHE_IMAGE_NAME}" - - name: "test ${UBUNTU_NAME}" - gce_instance: - image_name: "${UBUNTU_CACHE_IMAGE_NAME}" - - name: "test ${PRIOR_UBUNTU_NAME}" - gce_instance: - image_name: "${PRIOR_UBUNTU_CACHE_IMAGE_NAME}" - + - build + container: *smallcontainer + env: + <<: *stdenvars + TEST_FLAVOR: docker-py + TEST_ENVIRON: container + gopath_cache: *ro_gopath_cache + clone_script: *full_clone # build-cache not available to container tasks + setup_script: *setup + main_script: *main + always: *artifacts + +# Does exactly what it says, execute the podman unit-tests on all primary +# platforms and release versions. +unit_test_task: + name: "Unit tests on $DISTRO_NV" + alias: unit_test + depends_on: + - validate + matrix: *platform_axis + gce_instance: *standardvm env: - ADD_SECOND_PARTITION: 'true' - matrix: - - RCLI: 'true' - - RCLI: 'false' - - networking_script: '${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/networking.sh' - setup_environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}' - unit_test_script: '$SCRIPT_BASE/unit_test.sh |& ${TIMESTAMP}' - integration_test_script: '$SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP} | ${LOGFORMAT} integration_test' - system_test_script: '$SCRIPT_BASE/system_test.sh |& ${TIMESTAMP} | ${LOGFORMAT} system_test' - apiv2_test_script: '$SCRIPT_BASE/apiv2_test.sh |& ${TIMESTAMP} | ${LOGFORMAT} apiv2_test' - - # When examining a particular run, provide convenient access to release files. - tar_artifacts: - path: "*.tar.gz" - type: "application/x-tar" - - always: &standardlogs + TEST_FLAVOR: unit + clone_script: *noop # Comes from cache + gopath_cache: *ro_gopath_cache + setup_script: *setup + main_script: *main + always: *artifacts + +# Execute the podman integration tests on all primary platforms and release +# versions, as root, without involving the podman-remote client. +local_integration_test_task: &local_integration_test_task + # Integration-test task name convention: + # <int.|sys.> <podman|remote> <Distro NV> <root|rootless> + name: &std_name_fmt "$TEST_FLAVOR $PODBIN_NAME $DISTRO_NV $PRIV_NAME $TEST_ENVIRON" + alias: local_integration_test + depends_on: + - unit_test + matrix: *platform_axis + gce_instance: *standardvm + timeout_in: 90m + env: + TEST_FLAVOR: int + clone_script: *noop # Comes from cache + gopath_cache: *ro_gopath_cache + setup_script: *setup + main_script: *main + always: &logs_artifacts + <<: *artifacts package_versions_script: '$SCRIPT_BASE/logcollector.sh packages' ginkgo_node_logs_script: '$SCRIPT_BASE/logcollector.sh ginkgo' df_script: '$SCRIPT_BASE/logcollector.sh df' audit_log_script: '$SCRIPT_BASE/logcollector.sh audit' journal_script: '$SCRIPT_BASE/logcollector.sh journal' - varlink_script: '$SCRIPT_BASE/logcollector.sh varlink' podman_system_info_script: '$SCRIPT_BASE/logcollector.sh podman' - html_artifacts: - path: "*.log.html" - type: "text/html" - -# This task executes tests under unique environments/conditions -special_testing_rootless_task: - - depends_on: - - "gating" - - "varlink_api" - - "vendor" - - "build_each_commit" - - "build_without_cgo" - - only_if: $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*' +# Nearly identical to `local_integration_test` except all operations +# are performed through the podman-remote client vs a podman "server" +# running on the same host. +remote_integration_test_task: + <<: *local_integration_test_task + alias: remote_integration_test env: - ADD_SECOND_PARTITION: 'true' - SPECIALMODE: 'rootless' # See docs - matrix: - - RCLI: 'true' - - RCLI: 'false' - - networking_script: '${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/networking.sh' - setup_environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}' - integration_test_script: '$SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP} | ${LOGFORMAT} integration_test' - system_test_script: '$SCRIPT_BASE/system_test.sh |& ${TIMESTAMP} | ${LOGFORMAT} system_test' - apiv2_test_script: '$SCRIPT_BASE/apiv2_test.sh |& ${TIMESTAMP} | ${LOGFORMAT} apiv2_test' - - always: - <<: *standardlogs + TEST_FLAVOR: int + PODBIN_NAME: remote -special_testing_in_podman_task: - - alias: "special_testing_in_podman" +# Run the complete set of integration tests from inside a container. +# This verifies all/most operations function with "podman-in-podman". +container_integration_test_task: + name: *std_name_fmt + alias: container_integration_test depends_on: - - "gating" - - "varlink_api" - - "vendor" - - "build_each_commit" - - "build_without_cgo" - - only_if: $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*' - - matrix: - - name: "in-podman ${PRIOR_FEDORA_NAME}" - gce_instance: - image_name: "${PRIOR_FEDORA_CACHE_IMAGE_NAME}" - - name: "in-podman ${FEDORA_NAME}" - gce_instance: - image_name: "${FEDORA_CACHE_IMAGE_NAME}" - + - unit_test + matrix: &fedora_vm_axis + - env: + DISTRO_NV: ${FEDORA_NAME} + _BUILD_CACHE_HANDLE: ${FEDORA_NAME}-build-${CIRRUS_BUILD_ID} + VM_IMAGE_NAME: ${FEDORA_CACHE_IMAGE_NAME} + CTR_FQIN: ${FEDORA_CONTAINER_FQIN} + - env: + DISTRO_NV: ${PRIOR_FEDORA_NAME} + _BUILD_CACHE_HANDLE: ${PRIOR_FEDORA_NAME}-build-${CIRRUS_BUILD_ID} + VM_IMAGE_NAME: ${PRIOR_FEDORA_CACHE_IMAGE_NAME} + CTR_FQIN: ${PRIOR_FEDORA_CONTAINER_FQIN} + gce_instance: *standardvm + timeout_in: 90m env: - ADD_SECOND_PARTITION: 'true' - MOD_CONTAINERS_CONF: 'false' # Use existing/native setup - SPECIALMODE: 'in_podman' # See docs - - networking_script: '${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/networking.sh' - setup_environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}' - integration_test_script: '$SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP} | ${LOGFORMAT} integration_test' - - always: - <<: *standardlogs - - -special_testing_cross_task: - - alias: "special_testing_cross" + TEST_FLAVOR: int + TEST_ENVIRON: container + clone_script: *noop # Comes from cache + gopath_cache: *ro_gopath_cache + setup_script: *setup + main_script: *main + always: *logs_artifacts + +# Execute most integration tests as a regular (non-root) user. +rootless_integration_test_task: + name: *std_name_fmt + alias: rootless_integration_test depends_on: - - "gating" - - "varlink_api" - - "vendor" - - only_if: $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*' - - matrix: - - name: 'cross-platform: windows' - env: - CROSS_PLATFORM: 'windows' - - name: 'cross-platform: darwin' - env: - CROSS_PLATFORM: 'darwin' - - timeout_in: 20m - - networking_script: '${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/networking.sh' - setup_environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}' - build_release_script: '$SCRIPT_BASE/build_release.sh |& ${TIMESTAMP}' - - # When examining a particular run, provide convenient access to release files. - zip_artifacts: - path: "*.zip" - type: "application/zip" - - msi_artifacts: - path: "*.msi" - type: "application/octet-stream" - - -special_testing_bindings_task: - + - unit_test + matrix: *fedora_vm_axis + gce_instance: *standardvm + timeout_in: 90m + env: + TEST_FLAVOR: int + PRIV_NAME: rootless + clone_script: *noop # Comes from cache + gopath_cache: *ro_gopath_cache + setup_script: *setup + main_script: *main + always: *logs_artifacts + + +# Always run subsequent to integration tests. While parallelism is lost +# with runtime, debugging system-test failures can be more challenging +# for some golang developers. Otherwise the following tasks run across +# the same matrix as the integration-tests (above). +local_system_test_task: &local_system_test_task + name: *std_name_fmt + alias: local_system_test depends_on: - - "gating" - - "varlink_api" - - "vendor" - - only_if: $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*' - + - local_integration_test + matrix: *platform_axis + gce_instance: *standardvm env: - SPECIALMODE: 'bindings' # See docs - ADD_SECOND_PARTITION: 'true' # More root fs space is required - - timeout_in: 40m - - setup_environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}' - integration_test_script: '$SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP} | ${LOGFORMAT} integration_test' - - always: - <<: *standardlogs + TEST_FLAVOR: sys + clone_script: *noop # Comes from cache + gopath_cache: *ro_gopath_cache + setup_script: *setup + main_script: *main + always: *logs_artifacts -special_testing_endpoint_task: - +remote_system_test_task: + <<: *local_system_test_task + alias: remote_system_test depends_on: - - "gating" - - "varlink_api" - - "vendor" - - only_if: $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*' - + - remote_integration_test env: - SPECIALMODE: 'endpoint' # See docs - - timeout_in: 20m - - setup_environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}' - integration_test_script: '$SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP} | ${LOGFORMAT} integration_test' - always: - <<: *standardlogs - + TEST_FLAVOR: sys + PODBIN_NAME: remote -docs_task: +rootless_system_test_task: + name: *std_name_fmt + alias: rootless_system_test depends_on: - - "gating" + - rootless_integration_test + matrix: *fedora_vm_axis + gce_instance: *standardvm env: - RELEASE_GCPJSON: ENCRYPTED[927dc01e755eaddb4242b0845cf86c9098d1e3dffac38c70aefb1487fd8b4fe6dd6ae627b3bffafaba70e2c63172664e] - RELEASE_GCPNAME: ENCRYPTED[c145e9c16b6fb88d476944a454bf4c1ccc84bb4ecaca73bdd28bdacef0dfa7959ebc8171a27b2e4064d66093b2cdba49] - RELEASE_GCPROJECT: $GCP_PROJECT_ID - - script: - - "$SCRIPT_BASE/build_swagger.sh |& ${TIMESTAMP}" - - "$SCRIPT_BASE/upload_release_archive.sh |& ${TIMESTAMP}" + TEST_FLAVOR: sys + PRIV_NAME: rootless + PODBIN_NAME: remote + clone_script: *noop # Comes from cache + gopath_cache: *ro_gopath_cache + setup_script: *setup + main_script: *main + always: *logs_artifacts + +# This task is critical. It updates the "last-used by" timestamp stored +# in metadata for all VM images. This mechanism functions in tandem with +# an out-of-band pruning operation to remove disused VM images. +meta_task: + name: "VM img. keepalive" + alias: meta + container: + cpu: 2 + memory: 2 + image: quay.io/libpod/imgts:$IMAGE_SUFFIX + env: + # Space-separated list of images used by this repository state + IMGNAMES: >- + ${FEDORA_CACHE_IMAGE_NAME} + ${PRIOR_FEDORA_CACHE_IMAGE_NAME} + ${UBUNTU_CACHE_IMAGE_NAME} + ${PRIOR_UBUNTU_CACHE_IMAGE_NAME} + BUILDID: "${CIRRUS_BUILD_ID}" + REPOREF: "${CIRRUS_REPO_NAME}" + GCPJSON: ENCRYPTED[3a198350077849c8df14b723c0f4c9fece9ebe6408d35982e7adf2105a33f8e0e166ed3ed614875a0887e1af2b8775f4] + GCPNAME: ENCRYPTED[2f9738ef295a706f66a13891b40e8eaa92a89e0e87faf8bed66c41eca72bf76cfd190a6f2d0e8444c631fdf15ed32ef6] + GCPPROJECT: libpod-218412 + clone_script: *noop + script: /usr/local/bin/entrypoint.sh -# Post message to IRC if everything passed PR testing +# Status aggregator for all tests. This task simply ensures a defined +# set of tasks all passed, and allows confirming that based on the status +# of this task. success_task: - - # This task is a required-pass in github settings, - # it blocks PRs from merging if a depends_on task fails - only_if: $CIRRUS_BRANCH != $DEST_BRANCH - - # ignores any dependent task conditions + name: "Total Success" + alias: success + # N/B: ALL tasks must be listed here, minus their '_task' suffix. depends_on: - - "gating" - - "vendor" - - "varlink_api" - - "build_each_commit" - - "build_without_cgo" - - "container_image_build" - - "meta" - - "testing" - - "rpmbuild" - - "special_testing_rootless" - - "special_testing_in_podman" - - "special_testing_cross" - - "special_testing_endpoint" - - "special_testing_bindings" - - "docs" - - "static_build" - - "darwin_build" - + - ext_svc_check + - automation + - smoke + - build + - validate + - bindings + - endpoint + - swagger + - vendor + - alt_build + - static_alt_build + - osx_alt_build + - docker-py_test + - unit_test + - local_integration_test + - remote_integration_test + - rootless_integration_test + - container_integration_test + - local_system_test + - remote_system_test + - rootless_system_test + - meta + container: *smallcontainer env: - CIRRUS_WORKING_DIR: "/usr/src/libpod" - SRCPATH: "$CIRRUS_WORKING_DIR" - EPOCH_TEST_COMMIT: "${CIRRUS_BASE_SHA}" # repo clone missing this data - - container: - # Note: Image has dual purpose, see contrib/gate/README.md - image: "quay.io/libpod/gate:master" - cpu: 1 - memory: 1 - - success_script: /bin/true - -# Build the static binary -static_build_task: + CTR_FQIN: ${FEDORA_CONTAINER_FQIN} + TEST_ENVIRON: container + clone_script: *noop + script: /bin/true + + +# When a new tag is pushed, confirm that the code and commits +# meet criteria for an official release. +release_task: + name: "Verify Release" + alias: release + only_if: $CIRRUS_TAG != '' depends_on: - - "gating" - - gce_instance: - image_name: "${FEDORA_CACHE_IMAGE_NAME}" - cpu: 8 - memory: 12 - disk: 200 - - init_script: | - set -ex - setenforce 0 - growpart /dev/sda 1 || true - resize2fs /dev/sda1 || true - yum -y install podman - - nix_cache: - folder: '.cache' - fingerprint_script: | - echo "nix-v1-$(sha1sum nix/nixpkgs.json | head -c 40)" - - build_script: | - set -ex - mkdir -p .cache - mv .cache /nix - if [[ -z $(ls -A /nix) ]]; then podman run --rm --privileged -ti -v /:/mnt nixos/nix cp -rfT /nix /mnt/nix; fi - podman run --rm --privileged -ti -v /nix:/nix -v ${PWD}:${PWD} -w ${PWD} nixos/nix nix --print-build-logs --option cores 8 --option max-jobs 8 build --file nix/ - - binaries_artifacts: - path: "result/bin/podman" - - save_cache_script: | - mv /nix .cache - chown -Rf $(whoami) .cache - - -darwin_build_task: + - success + gce_instance: *standardvm + env: + <<: *stdenvars + TEST_FLAVOR: release + gopath_cache: *ro_gopath_cache + clone_script: *noop # Comes from cache + setup_script: *setup + main_script: *main + always: *artifacts + +# When preparing to release a new version, this task may be manually +# activated at the PR stage to verify the code is in a proper state. +# Note: This cannot use a YAML alias on 'release_task' as of this +# comment, it is incompatible with 'trigger_type: manual' +release_test_task: + name: "Optional Release Test" + alias: release_test + only_if: $CIRRUS_PR != '' + trigger_type: manual depends_on: - - "gating" - osx_instance: - image: catalina-base - setup-script: - - brew install go - - brew install go-md2man - build-script: - - make podman-remote-darwin - - make install-podman-remote-darwin-docs - binaries_artifacts: - path: "bin/podman-remote-darwin" + - success + gce_instance: *standardvm + env: + <<: *stdenvars + TEST_FLAVOR: release + gopath_cache: *ro_gopath_cache + clone_script: *noop # Comes from cache + setup_script: *setup + main_script: *main + always: *artifacts diff --git a/.golangci.yml b/.golangci.yml index 4d5ff549c..b3466d8f9 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -11,20 +11,32 @@ run: - pkg/varlink - pkg/varlinkapi - docs/varlink + - vendor skip-files: - iopodman.go - swagger.go + modules-download-mode: readonly linters: enable-all: true disable: + # All these break for one reason or another - dupl - funlen - gochecknoglobals - gochecknoinits - goconst - gocyclo - - gosec + - golint - lll + - structcheck + - typecheck + - unconvert + - varcheck + - deadcode + - depguard + - errcheck + - gocritic + - gosec - maligned linters-settings: errcheck: @@ -169,7 +169,13 @@ golangci-lint: .gopathok varlink_generate .install.golangci-lint .PHONY: gofmt gofmt: ## Verify the source code gofmt - find . -name '*.go' ! -path './vendor/*' -exec gofmt -s -w {} \+ + find . -name '*.go' -type f \ + -not \( \ + -name '.golangci.yml' -o \ + -name 'Makefile' -o \ + -path './vendor/*' -prune -o \ + -path './contrib/*' -prune \ + \) -exec gofmt -d -e -s -w {} \+ git diff --exit-code .PHONY: test/checkseccomp/checkseccomp diff --git a/contrib/cirrus/README.md b/contrib/cirrus/README.md deleted file mode 100644 index 4056edb6a..000000000 --- a/contrib/cirrus/README.md +++ /dev/null @@ -1,150 +0,0 @@ - - -# Cirrus-CI - -Similar to other integrated github CI/CD services, Cirrus utilizes a simple -YAML-based configuration/description file: ``.cirrus.yml``. Ref: https://cirrus-ci.org/ - - -## Workflow - -All tasks execute in parallel, unless there are conditions or dependencies -which alter this behavior. Within each task, each script executes in sequence, -so long as any previous script exited successfully. The overall state of each -task (pass or fail) is set based on the exit status of the last script to execute. - -### ``gating`` Task - -***N/B: Steps below are performed by automation*** - -1. Launch a purpose-built container in Cirrus's community cluster. - For container image details, please see - [the contributors guide](https://github.com/containers/podman/blob/master/CONTRIBUTING.md#go-format-and-lint). - -3. ``validate``: Perform standard `make validate` source verification, - Should run for less than a minute or two. - -4. ``lint``: Execute regular `make lint` to check for any code cruft. - Should also run for less than a few minutes. - -5. ``vendor``: runs `make vendor-in-container` followed by `./hack/tree_status.sh` to check - whether the git tree is clean. The reasoning for that is to make sure that - the vendor.conf, the code and the vendored packages in ./vendor are in sync - at all times. - -### ``meta`` Task - -***N/B: Steps below are performed by automation*** - -1. Launch a container built from definition in ``./contrib/imgts``. - -2. Update VM Image metadata to help track usage across all automation. - -4. Always exits successfully unless there's a major problem. - - -### ``testing`` Task - -***N/B: Steps below are performed by automation*** - -1. After `gating` passes, spin up one VM per - `matrix: image_name` item. Once accessible, ``ssh`` - into each VM as the `root` user. - -2. ``setup_environment.sh``: Configure root's `.bash_profile` - for all subsequent scripts (each run in a new shell). Any - distribution-specific environment variables are also defined - here. For example, setting tags/flags to use compiling. - -5. ``integration_test.sh``: Execute integration-testing. This is - much more involved, and relies on access to external - resources like container images and code from other repositories. - Total execution time is capped at 2-hours (includes all the above) - but this script normally completes in less than an hour. - - -### ``special_testing_cross`` Task - -Confirm that cross-compile of podman-remote functions for both `windows` -and `darwin` targets. - - -### ``special_testing_cgroupv2`` Task - -Use the latest Fedora release with the required kernel options pre-set for -exercising cgroups v2 with Podman integration tests. Also depends on -having `SPECIALMODE` set to 'cgroupv2` - - -### `docs` Task - -Builds swagger API documentation YAML and uploads to google storage (an online -service for storing unstructured data) for both -PR's (for testing the process) and the master branch. For PR's -the YAML is uploaded into a [dedicated short-pruning cycle -bucket.](https://storage.googleapis.com/libpod-pr-releases/) for testing purposes -only. For the master branch, a [separate bucket is -used](https://storage.googleapis.com/libpod-master-releases) and provides the -content rendered on [the API Reference page](https://docs.podman.io/en/latest/_static/api.html) - -The online API reference is presented by javascript to the client. To prevent hijacking -of the client by malicious data, the [javascript utilises CORS](https://cloud.google.com/storage/docs/cross-origin). -This CORS metadata is served by `https://storage.googleapis.com` when configured correctly. -It will appear in [the request and response headers from the -client](https://cloud.google.com/storage/docs/configuring-cors#troubleshooting) when accessing -the API reference page. - -However, when the CORS metadata is missing or incorrectly configured, clients will receive an -error-message similar to: - - - -For documentation built by Read The Docs from the master branch, CORS metadata is -set on the `libpod-master-releases` storage bucket. Viewing or setting the CORS -metadata on the bucket requires having locally [installed and -configured the google-cloud SDK](https://cloud.google.com/sdk/docs). It also requires having -admin access to the google-storage bucket. Contact a project owner for help if you are -unsure of your permissions or need help resolving an error similar to the picture above. - -Assuming the SDK is installed, and you have the required admin access, the following command -will display the current CORS metadata: - -``` -gsutil cors get gs://libpod-master-releases -``` - -To function properly (allow client "trust" of content from `storage.googleapis.com`) the followiing -metadata JSON should be used. Following the JSON, is an example of the command used to set this -metadata on the libpod-master-releases bucket. For additional information about configuring CORS -please refer to [the google-storage documentation](https://cloud.google.com/storage/docs/configuring-cors). - -```JSON -[ - { - "origin": ["http://docs.podman.io", "https://docs.podman.io"], - "responseHeader": ["Content-Type"], - "method": ["GET"], - "maxAgeSeconds": 600 - } -] -``` - -``` -gsutil cors set /path/to/file.json gs://libpod-master-releases -``` - -***Note:*** The CORS metadata does _NOT_ change after the `docs` task uploads a new swagger YAML -file. Therefore, if it is not functioning or misconfigured, a person must have altered it or -changes were made to the referring site (e.g. `docs.podman.io`). - -## `$SPECIALMODE` - -Some tasks alter their behavior based on this value. A summary of supported -values follows: - -* `none`: Operate as normal, this is the default value if unspecified. -* `rootless`: Causes a random, ordinary user account to be created - and utilized for testing. -* `in_podman`: Causes testing to occur within a container executed by -* `windows`: See **darwin** -* `darwin`: Signals the ``special_testing_cross`` task to cross-compile the remote client. diff --git a/contrib/cirrus/add_second_partition.sh b/contrib/cirrus/add_second_partition.sh index d0407be86..322dd2512 100644 --- a/contrib/cirrus/add_second_partition.sh +++ b/contrib/cirrus/add_second_partition.sh @@ -3,6 +3,11 @@ # N/B: This script could mega f*!@up your disks if run by mistake. # it is left without the execute-bit on purpose! +set -eo pipefail + +# shellcheck source=./lib.sh +source $(dirname $0)/lib.sh + # $SLASH_DEVICE is the disk device to be f*xtuP SLASH_DEVICE="/dev/sda" # Always the case on GCP @@ -11,21 +16,18 @@ SLASH_DEVICE="/dev/sda" # Always the case on GCP NEW_PART_START="50%" NEW_PART_END="100%" -set -eo pipefail - -source $(dirname $0)/lib.sh if [[ ! -r "/root" ]] || [[ -r "/root/second_partition_ready" ]] then - echo "Warning: Ignoring attempted execution of $(basename $0)" + warn "Ignoring attempted execution of $(basename $0)" exit 0 fi -[[ -n "type -P parted" ]] || \ - die 2 "The parted command is required." +[[ -x "$(type -P parted)" ]] || \ + die "The parted command is required." [[ ! -b ${SLASH_DEVICE}2 ]] || \ - die 5 "Found unexpected block device ${SLASH_DEVICE}2" + die "Found unexpected block device ${SLASH_DEVICE}2" PPRINTCMD="parted --script ${SLASH_DEVICE} print" FINDMNTCMD="findmnt --source=${SLASH_DEVICE}1 --mountpoint=/ --canonicalize --evaluate --first-only --noheadings" @@ -34,28 +36,28 @@ trap "rm -f $TMPF" EXIT if $FINDMNTCMD | tee $TMPF | egrep -q "^/\s+${SLASH_DEVICE}1" then - echo "Repartitioning original partition table:" + msg "Repartitioning original partition table:" $PPRINTCMD else - die 6 "Unexpected output from '$FINDMNTCMD': $(<$TMPF)" + die "Unexpected output from '$FINDMNTCMD': $(<$TMPF)" fi echo "Adding partition offset within unpartitioned space." parted --script --align optimal /dev/sda unit % mkpart primary "" "" "$NEW_PART_START" "$NEW_PART_END" -echo "New partition table:" +msg "New partition table:" $PPRINTCMD -echo "Growing ${SLASH_DEVICE}1 meet start of ${SLASH_DEVICE}2" +msg "Growing ${SLASH_DEVICE}1 meet start of ${SLASH_DEVICE}2" growpart ${SLASH_DEVICE} 1 FSTYPE=$(findmnt --first-only --noheadings --output FSTYPE ${SLASH_DEVICE}1) echo "Expanding $FSTYPE filesystem on ${SLASH_DEVICE}1" case $FSTYPE in ext*) resize2fs ${SLASH_DEVICE}1 ;; - *) die 11 "Script $(basename $0) doesn't know how to resize a $FSTYPE filesystem." ;; + *) die "Script $(basename $0) doesn't know how to resize a $FSTYPE filesystem." ;; esac # Must happen last - signals completion to other tooling -echo "Recording newly available disk partition device into /root/second_partition_ready" +msg "Recording newly available disk partition device into /root/second_partition_ready" echo "${SLASH_DEVICE}2" > /root/second_partition_ready diff --git a/contrib/cirrus/apiv2_test.sh b/contrib/cirrus/apiv2_test.sh deleted file mode 100755 index dbee078b6..000000000 --- a/contrib/cirrus/apiv2_test.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env bash - -set -e - -source $(dirname $0)/lib.sh - -req_env_var GOSRC SCRIPT_BASE OS_RELEASE_ID OS_RELEASE_VER CONTAINER_RUNTIME VARLINK_LOG - -LOCAL_OR_REMOTE=local -if [[ "$RCLI" = "true" ]]; then - LOCAL_OR_REMOTE=remote -fi - -# Our name must be of the form xxxx_test or xxxx_test.sh, where xxxx is -# the test suite to run; currently (2019-05) the only option is 'integration' -# but pr2947 intends to add 'system'. -TESTSUITE=$(expr $(basename $0) : '\(.*\)_test') -if [[ -z $TESTSUITE ]]; then - die 1 "Script name is not of the form xxxx_test.sh" -fi - -cd "$GOSRC" - -case "$SPECIALMODE" in - in_podman) - ${CONTAINER_RUNTIME} run --rm --privileged --net=host --cgroupns=host \ - -v $GOSRC:$GOSRC:Z \ - --workdir $GOSRC \ - -e "CGROUP_MANAGER=cgroupfs" \ - -e "STORAGE_OPTIONS=--storage-driver=vfs" \ - -e "CRIO_ROOT=$GOSRC" \ - -e "PODMAN_BINARY=/usr/bin/podman" \ - -e "CONMON_BINARY=/usr/libexec/podman/conmon" \ - -e "DIST=$OS_RELEASE_ID" \ - -e "CONTAINER_RUNTIME=$CONTAINER_RUNTIME" \ - $IN_PODMAN_IMAGE bash $GOSRC/$SCRIPT_BASE/container_test.sh -b -i -t - ;; - rootless) - req_env_var ROOTLESS_USER - ssh $ROOTLESS_USER@localhost \ - -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ - -o CheckHostIP=no $GOSRC/$SCRIPT_BASE/rootless_test.sh ${TESTSUITE} ${LOCAL_OR_REMOTE} - ;; - endpoint) - make - make install PREFIX=/usr ETCDIR=/etc - make test-binaries - make endpoint - ;; - bindings) - make - make install PREFIX=/usr ETCDIR=/etc - export PATH=$PATH:`pwd`/hack - cd pkg/bindings/test && ginkgo -trace -noColor -debug -r - ;; - none) - make - make install PREFIX=/usr ETCDIR=/etc - make test-binaries - make .install.bats - make ${LOCAL_OR_REMOTE}${TESTSUITE} PODMAN_SERVER_LOG=$PODMAN_SERVER_LOG - ;; - *) - die 110 "Unsupported \$SPECIALMODE: $SPECIALMODE" -esac diff --git a/contrib/cirrus/build_release.sh b/contrib/cirrus/build_release.sh deleted file mode 100755 index 46fe9781f..000000000 --- a/contrib/cirrus/build_release.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash - -set -e - -source $(dirname $0)/lib.sh - -req_env_var RCLI OS_RELEASE_ID GOSRC - -cd $GOSRC - -if [[ "$RCLI" == "true" ]] && [[ -z "$CROSS_PLATFORM" ]] -then - CROSS_PLATFORM=linux -fi - -if [[ -n "$CROSS_PLATFORM" ]] -then - # Will fail if $CROSS_PLATFORM is unsupported cross-compile $GOOS value - make podman-remote-${CROSS_PLATFORM}-release - - echo "Compiling podman-remote release archive for ${CROSS_PLATFORM}" - if [[ "$CROSS_PLATFORM" == "windows" ]] - then - # TODO: Remove next line, part of VM images next time they're built. - dnf install -y libmsi1 msitools pandoc - make podman.msi - fi -else - echo "Compiling release archive for $OS_RELEASE_ID" - make podman-release -fi - -echo "Preserving build details for later use." -mv -v release.txt actual_release.txt # Another 'make' during testing could overwrite it diff --git a/contrib/cirrus/build_swagger.sh b/contrib/cirrus/build_swagger.sh deleted file mode 100755 index eb9288dcd..000000000 --- a/contrib/cirrus/build_swagger.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash - -set -e - -source $(dirname $0)/lib.sh - -# Building this is a PITA, just grab binary for use in automation -# Ref: https://goswagger.io/install.html#static-binary -download_url=$(curl -s https://api.github.com/repos/go-swagger/go-swagger/releases/latest | \ - jq -r '.assets[] | select(.name | contains("'"$(uname | tr '[:upper:]' '[:lower:]')"'_amd64")) | .browser_download_url') -curl -o /usr/local/bin/swagger -L'#' "$download_url" -chmod +x /usr/local/bin/swagger - -cd $GOSRC -make swagger -echo "Preserving build details for later use." -mv -v release.txt actual_release.txt # Another 'make' during testing could overwrite it diff --git a/contrib/cirrus/cirrus_yaml_test.py b/contrib/cirrus/cirrus_yaml_test.py index c2ff8e69e..39de0c7b7 100755 --- a/contrib/cirrus/cirrus_yaml_test.py +++ b/contrib/cirrus/cirrus_yaml_test.py @@ -26,6 +26,7 @@ class TestCaseBase(unittest.TestCase): class TestDependsOn(TestCaseBase): ALL_TASK_NAMES = None + SUCCESS_DEPS_EXCLUDE = set(['success', 'release', 'release_test']) def setUp(self): super().setUp() @@ -33,23 +34,29 @@ class TestDependsOn(TestCaseBase): for key, _ in self.CIRRUS_YAML.items() if key.endswith('_task')]) - def test_00_dicts(self): + def test_dicts(self): """Expected dictionaries are present and non-empty""" self.assertIn('success_task', self.CIRRUS_YAML) self.assertIn('success_task'.replace('_task', ''), self.ALL_TASK_NAMES) self.assertIn('depends_on', self.CIRRUS_YAML['success_task']) self.assertGreater(len(self.CIRRUS_YAML['success_task']['depends_on']), 0) - def test_01_depends(self): + def test_task(self): + """There is no task named 'task'""" + self.assertNotIn('task', self.ALL_TASK_NAMES) + + def test_depends(self): """Success task depends on all other tasks""" success_deps = set(self.CIRRUS_YAML['success_task']['depends_on']) - for task_name in self.ALL_TASK_NAMES - set(['success']): + for task_name in self.ALL_TASK_NAMES - self.SUCCESS_DEPS_EXCLUDE: with self.subTest(task_name=task_name): msg=('Please add "{0}" to the "depends_on" list in "success_task"' "".format(task_name)) self.assertIn(task_name, success_deps, msg=msg) - + def not_task(self): + """Ensure no task is named 'task'""" + self.assertNotIn('task', self.ALL_TASK_NAMES) if __name__ == "__main__": unittest.main() diff --git a/contrib/cirrus/container_test.sh b/contrib/cirrus/container_test.sh deleted file mode 100644 index 8ea66e63c..000000000 --- a/contrib/cirrus/container_test.sh +++ /dev/null @@ -1,142 +0,0 @@ -#!/usr/bin/env bash -set -xeo pipefail - -export GOPATH=/var/tmp/go -export PATH=$HOME/gopath/bin:$PATH:$GOPATH/bin -export GOSRC=$GOPATH/src/github.com/containers/podman - -DIST=${DIST:=""} -CONTAINER_RUNTIME=${DIST:=""} - -source /etc/os-release - -INTEGRATION_TEST_ENVS="" - -# For all distributions not Fedora, we need to skip USERNS tests -# for now. -if [ "${ID}" != "fedora" ] || [ "${CONTAINER_RUNTIME}" != "" ]; then - INTEGRATION_TEST_ENVS="SKIP_USERNS=1" -fi - -echo "$(date --rfc-3339=seconds) $(basename $0) started with '$*' and RCLI='${RCLI}'" - -pwd - -# -i install -# -b build -# -t integration test -# -u unit test -# -v validate - -build=0 -install=0 -integrationtest=0 -unittest=0 -validate=0 -options=0 -remote=0 -install_tools_made=0 - -while getopts "bituv" opt; do - case "$opt" in - b) build=1 - options=1 - ;; - i) install=1 - options=1 - ;; - t) integrationtest=1 - options=1 - ;; - u) unittest=1 - options=1 - ;; - v) validate=1 - options=1 - ;; - esac -done - -# The RCLI environment variable decides whether -# to test varlinke -if [[ "$RCLI" == "true" ]]; then - remote=1 -fi - -# If no options are passed, do everything -if [ $options -eq 0 ]; then - build=1 - install=1 - integrationtest=1 - unittest=1 - validate=1 -fi - -# Make Install tools function used by multiple sections below -make_install_tools () { - # Only make the install tools once - if [ $install_tools_made -eq 0 ]; then - make install.tools TAGS="${TAGS}" - fi - install_tools_made=1 -} - -CONTAINER_RUNTIME=${CONTAINER_RUNTIME:-none} - -if [ "${CONTAINER_RUNTIME}" == "none" ]; then - mkdir -p /$GOPATH/src/github.com/containers/ - mv /var/tmp/checkout $GOSRC - cd $GOSRC - pwd -fi - - -export TAGS="seccomp $($GOSRC/hack/btrfs_tag.sh) $($GOSRC/hack/libdm_tag.sh) $($GOSRC/hack/btrfs_installed_tag.sh) $($GOSRC/hack/selinux_tag.sh)" - -# Validate -if [ $validate -eq 1 ]; then - make_install_tools - # PAPR adds a merge commit, for testing, which fails the - # short-commit-subject validation test, so tell git-validate.sh to only check - # up to, but not including, the merge commit. - export GITVALIDATE_TIP=$(cd $GOSRC; git log -2 --pretty='%H' | tail -n 1) - make gofmt TAGS="${TAGS}" - - # Only check lint and gitvalidation on more recent - # distros with updated git and tooling - if [[ ${DIST} == "Fedora" ]]; then - HEAD=$GITVALIDATE_TIP make -C $GOSRC .gitvalidation TAGS="${TAGS}" - make lint - fi -fi - -# Unit tests -if [ $unittest -eq 1 ]; then - make localunit TAGS="${TAGS}" -fi - -# Make Podman -if [ $build -eq 1 ]; then - make_install_tools - make TAGS="${TAGS}" GOPATH=$GOPATH - make podman-remote TAGS="${TAGS}" GOPATH=$GOPATH -fi - -# Install Podman -if [ $install -eq 1 ]; then - make_install_tools - make TAGS="${TAGS}" install.bin PREFIX=/usr ETCDIR=/etc - make TAGS="${TAGS}" install.man PREFIX=/usr ETCDIR=/etc - make TAGS="${TAGS}" install.cni PREFIX=/usr ETCDIR=/etc - make TAGS="${TAGS}" install.systemd PREFIX=/usr ETCDIR=/etc -fi - -# Run integration tests -if [ $integrationtest -eq 1 ]; then - make TAGS="${TAGS}" test-binaries - make varlink_generate - make localintegration $INTEGRATION_TEST_ENVS - if [ $remote -eq 1 ]; then - make remoteintegration $INTEGRATION_TEST_ENVS - fi -fi diff --git a/contrib/cirrus/ext_svc_check.sh b/contrib/cirrus/ext_svc_check.sh new file mode 100755 index 000000000..b89e3d753 --- /dev/null +++ b/contrib/cirrus/ext_svc_check.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +set -eo pipefail + +# This script attempts basic confirmation of functional networking +# by connecting to a set of essential external servers and failing +# if any cannot be reached. It's intended for use early on in the +# podman CI system, to help prevent wasting time on tests that can't +# succeede due to some outage or another. + +# shellcheck source=./contrib/cirrus/lib.sh +source $(dirname $0)/lib.sh + +cat ${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/required_host_ports.txt | \ + while read host port + do + if [[ "$port" -eq "443" ]] + then + echo "SSL/TLS to $host:$port" + echo -n '' | \ + err_retry 9 1000 "" openssl s_client -quiet -no_ign_eof -connect $host:$port + else + echo "Connect to $host:$port" + err_retry 9 1000 1 nc -zv -w 13 $host $port + fi + done + +# TODO: Pull images required during testing into /dev/null + +# TODO: Refresh DNF package-cache into /dev/null diff --git a/contrib/cirrus/integration_test.sh b/contrib/cirrus/integration_test.sh deleted file mode 100755 index 636d67411..000000000 --- a/contrib/cirrus/integration_test.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env bash - -set -e - -source $(dirname $0)/lib.sh - -req_env_var GOSRC SCRIPT_BASE OS_RELEASE_ID OS_RELEASE_VER CONTAINER_RUNTIME VARLINK_LOG - -LOCAL_OR_REMOTE=local -if [[ "$RCLI" = "true" ]]; then - LOCAL_OR_REMOTE=remote -fi - -# Our name must be of the form xxxx_test or xxxx_test.sh, where xxxx is -# the test suite to run; currently (2019-05) the only option is 'integration' -# but pr2947 intends to add 'system'. -TESTSUITE=$(expr $(basename $0) : '\(.*\)_test') -if [[ -z $TESTSUITE ]]; then - die 1 "Script name ($basename $0) is not of the form xxxx_test.sh" -fi - -cd "$GOSRC" - -case "$SPECIALMODE" in - in_podman) - ${CONTAINER_RUNTIME} run --rm --privileged --net=host --cgroupns=host \ - -v $GOSRC:$GOSRC:Z \ - --workdir $GOSRC \ - -e "CGROUP_MANAGER=cgroupfs" \ - -e "STORAGE_OPTIONS=--storage-driver=vfs" \ - -e "CRIO_ROOT=$GOSRC" \ - -e "PODMAN_BINARY=/usr/bin/podman" \ - -e "CONMON_BINARY=/usr/libexec/podman/conmon" \ - -e "DIST=$OS_RELEASE_ID" \ - -e "CONTAINER_RUNTIME=$CONTAINER_RUNTIME" \ - $IN_PODMAN_IMAGE bash $GOSRC/$SCRIPT_BASE/container_test.sh -b -i -t - ;; - rootless) - req_env_var ROOTLESS_USER - ssh $ROOTLESS_USER@localhost \ - -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ - -o CheckHostIP=no $GOSRC/$SCRIPT_BASE/rootless_test.sh ${TESTSUITE} ${LOCAL_OR_REMOTE} - ;; - endpoint) - make - make install PREFIX=/usr ETCDIR=/etc - make test-binaries - make endpoint - ;; - bindings) - make - make install PREFIX=/usr ETCDIR=/etc - export PATH=$PATH:`pwd`/hack - cd pkg/bindings/test && ginkgo -trace -noColor -debug -r - ;; - none) - make - make install PREFIX=/usr ETCDIR=/etc - make test-binaries - make .install.bats - make ${LOCAL_OR_REMOTE}${TESTSUITE} PODMAN_SERVER_LOG=$PODMAN_SERVER_LOG - ;; - *) - die 110 "Unsupported \$SPECIALMODE: $SPECIALMODE" -esac diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh index 0dbb57ab3..b57aee1af 100644 --- a/contrib/cirrus/lib.sh +++ b/contrib/cirrus/lib.sh @@ -3,243 +3,161 @@ # Library of common, shared utility functions. This file is intended # to be sourced by other scripts, not called directly. -# Global details persist here -source /etc/environment # not always loaded under all circumstances - -# Automation environment doesn't automatically load for Ubuntu 18 -if [[ -r '/usr/share/automation/environment' ]]; then - source '/usr/share/automation/environment' +# BEGIN Global export of all variables +set -a + +# Due to differences across platforms and runtime execution environments, +# handling of the (otherwise) default shell setup is non-uniform. Rather +# than attempt to workaround differences, simply force-load/set required +# items every time this library is utilized. +source /etc/profile +source /etc/environment +USER="$(whoami)" +HOME="$(getent passwd $USER | cut -d : -f 6)" +# Some platforms set and make this read-only +[[ -n "$UID" ]] || \ + UID=$(getent passwd $USER | cut -d : -f 3) +GID=$(getent passwd $USER | cut -d : -f 4) + +# During VM Image build, the 'containers/automation' installation +# was performed. The final step of that installation sets the +# installation location in $AUTOMATION_LIB_PATH in /etc/environment +# or in the default shell profile. +# shellcheck disable=SC2154 +if [[ -n "$AUTOMATION_LIB_PATH" ]]; then + for libname in defaults anchors console_output utils; do + # There's no way shellcheck can process this location + # shellcheck disable=SC1090 + source $AUTOMATION_LIB_PATH/${libname}.sh + done +else + ( + echo "WARNING: It does not appear that containers/automation was installed." + echo " Functionality of most of this library will be negatively impacted" + echo " This ${BASH_SOURCE[0]} was loaded by ${BASH_SOURCE[1]}" + ) > /dev/stderr fi -# Under some contexts these values are not set, make sure they are. -export USER="$(whoami)" -export HOME="$(getent passwd $USER | cut -d : -f 6)" -[[ -n "$UID" ]] || export UID=$(getent passwd $USER | cut -d : -f 3) -export GID=$(getent passwd $USER | cut -d : -f 4) +OS_RELEASE_ID="$(source /etc/os-release; echo $ID)" +# GCE image-name compatible string representation of distribution _major_ version +OS_RELEASE_VER="$(source /etc/os-release; echo $VERSION_ID | cut -d '.' -f 1)" +# Combined to ease soe usage +OS_REL_VER="${OS_RELEASE_ID}-${OS_RELEASE_VER}" # Essential default paths, many are overridden when executing under Cirrus-CI -export GOPATH="${GOPATH:-/var/tmp/go}" +GOPATH="${GOPATH:-/var/tmp/go}" if type -P go &> /dev/null then - # required for go 1.12+ - export GOCACHE="${GOCACHE:-$HOME/.cache/go-build}" + # Cirrus-CI caches $GOPATH contents + export GOCACHE="${GOCACHE:-$GOPATH/cache/go-build}" # called processes like `make` and other tools need these vars. eval "export $(go env)" # Ensure compiled tooling is reachable - export PATH="$PATH:$GOPATH/bin" + PATH="$PATH:$GOPATH/bin:$HOME/.local/bin" fi -CIRRUS_WORKING_DIR="${CIRRUS_WORKING_DIR:-$GOPATH/src/github.com/containers/podman}" -export GOSRC="${GOSRC:-$CIRRUS_WORKING_DIR}" -export PATH="$HOME/bin:$GOPATH/bin:/usr/local/bin:$PATH" -export LD_LIBRARY_PATH="/usr/local/lib${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}" +CIRRUS_WORKING_DIR="${CIRRUS_WORKING_DIR:-$(realpath $(dirname ${BASH_SOURCE[0]})/../../)}" +GOSRC="${GOSRC:-$CIRRUS_WORKING_DIR}" +PATH="$HOME/bin:/usr/local/bin:$PATH" +LD_LIBRARY_PATH="/usr/local/lib${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}" + # Saves typing / in case location ever moves SCRIPT_BASE=${SCRIPT_BASE:-./contrib/cirrus} -# Important filepaths -SETUP_MARKER_FILEPATH="${SETUP_MARKER_FILEPATH:-/var/tmp/.setup_environment_sh_complete}" + # Downloaded, but not installed packages. PACKAGE_DOWNLOAD_DIR=/var/cache/download # Log remote-client system test varlink output here -export VARLINK_LOG=/var/tmp/varlink.log - -cd $GOSRC -if type -P git &> /dev/null && [[ -d "$GOSRC/.git" ]] -then - CIRRUS_CHANGE_IN_REPO=${CIRRUS_CHANGE_IN_REPO:-$(git show-ref --hash=8 HEAD || date +%s)} -else # pick something unique and obviously not from Cirrus - CIRRUS_CHANGE_IN_REPO=${CIRRUS_CHANGE_IN_REPO:-unknown_$(date +%s)} -fi +PODMAN_SERVER_LOG=$CIRRUS_WORKING_DIR/varlink.log # Defaults when not running under CI export CI="${CI:-false}" CIRRUS_CI="${CIRRUS_CI:-false}" DEST_BRANCH="${DEST_BRANCH:-master}" CONTINUOUS_INTEGRATION="${CONTINUOUS_INTEGRATION:-false}" -CIRRUS_REPO_NAME=${CIRRUS_REPO_NAME:-libpod} +CIRRUS_REPO_NAME=${CIRRUS_REPO_NAME:-podman} CIRRUS_BASE_SHA=${CIRRUS_BASE_SHA:-unknown$(date +%s)} # difficult to reliably discover CIRRUS_BUILD_ID=${CIRRUS_BUILD_ID:-$RANDOM$(date +%s)} # must be short and unique -OS_RELEASE_ID="$(source /etc/os-release; echo $ID)" -# GCE image-name compatible string representation of distribution _major_ version -OS_RELEASE_VER="$(source /etc/os-release; echo $VERSION_ID | cut -d '.' -f 1)" -# Combined to ease soe usage -OS_REL_VER="${OS_RELEASE_ID}-${OS_RELEASE_VER}" +# Needed for linting and code validation +EPOCH_TEST_COMMIT=${CIRRUS_BASE_SHA:-$CIRRUS_LAST_GREEN_CHANGE} -# IN_PODMAN container image -IN_PODMAN_IMAGE="quay.io/libpod/${OS_RELEASE_ID}_podman:$_BUILT_IMAGE_SUFFIX" -# Image for uploading releases -UPLDREL_IMAGE="quay.io/libpod/upldrel:master" - -# This is needed under some environments/contexts -SUDO='' -[[ "$UID" -eq 0 ]] || \ - SUDO='sudo -E' - -# Avoid getting stuck waiting for user input -export DEBIAN_FRONTEND="noninteractive" -SUDOAPTGET="$SUDO apt-get -qq --yes" -SUDOAPTADD="$SUDO add-apt-repository --yes" -# Regex that finds enabled periodic apt configuration items -PERIODIC_APT_RE='^(APT::Periodic::.+")1"\;' -# Short-cuts for retrying/timeout calls -LILTO="timeout_attempt_delay_command 120s 5 30s" -BIGTO="timeout_attempt_delay_command 300s 5 60s" - -# Safe env. vars. to transfer from root -> $ROOTLESS_USER (go env handled separately) -ROOTLESS_ENV_RE='(CIRRUS_.+)|(ROOTLESS_.+)|(.+_IMAGE.*)|(.+_BASE)|(.*DIRPATH)|(.*FILEPATH)|(SOURCE.*)|(DEPEND.*)|(.+_DEPS_.+)|(OS_REL.*)|(.+_ENV_RE)|(TRAVIS)|(CI.+)|(REMOTE.*)' +# Regex of env. vars. to explicitly pass when executing tests +# inside a container or as a rootless user +PASSTHROUGH_ENV_RE='(^CI.*)|(^CIRRUS)|(^DISTRO_NV)|(^GOPATH)|(^GOCACHE)|(^GOSRC)|(^SCRIPT_BASE)|(CGROUP_MANAGER)|(OCI_RUNTIME)|(^TEST.*)|(^PODBIN_NAME)|(^PRIV_NAME)|(^ALT_NAME)|(^ROOTLESS_USER)|(SKIP_USERNS)|(.*_NAME)|(.*_FQIN)' # Unsafe env. vars for display -SECRET_ENV_RE='(ACCOUNT)|(GC[EP]..+)|(SSH)' +SECRET_ENV_RE='(ACCOUNT)|(GC[EP]..+)|(SSH)|(PASSWORD)|(TOKEN)' -SPECIALMODE="${SPECIALMODE:-none}" -RCLI="${RCLI:-false}" -export CONTAINER_RUNTIME=${CONTAINER_RUNTIME:-podman} - -# When running as root, this may be empty or not, as a user, it MUST be set. -if [[ "$USER" == "root" ]] -then - ROOTLESS_USER="${ROOTLESS_USER:-}" -else - ROOTLESS_USER="${ROOTLESS_USER:-$USER}" -fi # Type of filesystem used for cgroups CG_FS_TYPE="$(stat -f -c %T /sys/fs/cgroup)" -# Pass in a list of one or more envariable names; exit non-zero with -# helpful error message if any value is empty -req_env_var() { - # Provide context. If invoked from function use its name; else script name - local caller=${FUNCNAME[1]} - if [[ -n "$caller" ]]; then - # Indicate that it's a function name - caller="$caller()" - else - # Not called from a function: use script name - caller=$(basename $0) - fi - - # Usage check - [[ -n "$1" ]] || die 1 "FATAL: req_env_var: invoked without arguments" - - # Each input arg is an envariable name, e.g. HOME PATH etc. Expand each. - # If any is empty, bail out and explain why. - for i; do - if [[ -z "${!i}" ]]; then - die 9 "FATAL: $caller requires \$$i to be non-empty" - fi +# Set to 1 in all podman container images +CONTAINER="${CONTAINER:-0}" + +# END Global export of all variables +set +a + +lilto() { err_retry 8 1000 "" "$@"; } # just over 4 minutes max +bigto() { err_retry 7 5670 "" "$@"; } # 12 minutes max + +# Print shell-escaped variable=value pairs, one per line, based on +# variable name matching a regex. This is intended to support +# passthrough of CI variables from host -> container or from root -> user. +# For all other vars. we rely on tooling to load this library from inside +# the container or as rootless user to pickup the remainder. +passthrough_envars(){ + local xchars + local envname + local envval + # Avoid values containing entirely punctuation|control|whitespace + xchars='[:punct:][:cntrl:][:space:]' + warn "Will pass env. vars. matching the following regex: + $PASSTHROUGH_ENV_RE" + for envname in $(awk 'BEGIN{for(v in ENVIRON) print v}' | \ + grep -Ev "SETUP_ENVIRONMENT" | \ + grep -Ev "$SECRET_ENV_RE" | \ + grep -E "$PASSTHROUGH_ENV_RE"); do + + envval="${!envname}" + [[ -n $(tr -d "$xchars" <<<"$envval") ]] || continue + + # Properly escape values to prevent injection + printf -- "$envname=%q\n" "$envval" done } -item_test() { - ITEM="$1" - shift - TEST_ARGS="$@" - req_env_var ITEM TEST_ARGS - - if ERR=$(test "$@" 2>&1) - then - echo "ok $ITEM" - return 0 - else - RET=$? - echo -n "not ok $ITEM: $TEST_ARGS" - if [[ -z "$ERR" ]] - then - echo "" - else # test command itself failed - echo -n ":" # space follows :'s in $ERR - echo "$ERR" | cut -d : -f 4- # omit filename, line number, and command - fi - return $RET - fi -} - -show_env_vars() { - echo "Showing selection of environment variable definitions:" - _ENV_VAR_NAMES=$(awk 'BEGIN{for(v in ENVIRON) print v}' | \ - egrep -v "(^PATH$)|(^BASH_FUNC)|(^[[:punct:][:space:]]+)|$SECRET_ENV_RE" | \ - sort -u) - for _env_var_name in $_ENV_VAR_NAMES - do - # Supports older BASH versions - printf " ${_env_var_name}=%q\n" "$(printenv $_env_var_name)" - done -} - -die() { - echo "************************************************" - echo ">>>>> ${2:-FATAL ERROR (but no message given!) in ${FUNCNAME[1]}()}" - echo "************************************************" - exit ${1:-1} -} - -warn() { - echo ">>>>> ${1:-WARNING (but no message given!) in ${FUNCNAME[1]}()}" > /dev/stderr -} - -bad_os_id_ver() { - echo "Unknown/Unsupported distro. $OS_RELEASE_ID and/or version $OS_RELEASE_VER for $(basename $0)" - exit 42 -} - -stub() { - echo "STUB: Pretending to do $1" -} - -timeout_attempt_delay_command() { - TIMEOUT=$1 - ATTEMPTS=$2 - DELAY=$3 - shift 3 - STDOUTERR=$(mktemp -p '' $(basename $0)_XXXXX) - req_env_var ATTEMPTS DELAY - echo "Retrying $ATTEMPTS times with a $DELAY delay, and $TIMEOUT timeout for command: $@" - for (( COUNT=1 ; COUNT <= $ATTEMPTS ; COUNT++ )) - do - echo "##### (attempt #$COUNT)" &>> "$STDOUTERR" - if timeout --foreground $TIMEOUT "$@" &>> "$STDOUTERR" - then - echo "##### (success after #$COUNT attempts)" &>> "$STDOUTERR" - break - else - echo "##### (failed with exit: $?)" &>> "$STDOUTERR" - sleep $DELAY - fi - done - cat "$STDOUTERR" - rm -f "$STDOUTERR" - if (( COUNT > $ATTEMPTS )) - then - echo "##### (exceeded $ATTEMPTS attempts)" - exit 125 - fi -} - setup_rootless() { - req_env_var ROOTLESS_USER GOPATH GOSRC SECRET_ENV_RE ROOTLESS_ENV_RE + req_env_vars ROOTLESS_USER GOPATH GOSRC SECRET_ENV_RE + + local rootless_uid + local rootless_gid + local env_var_val - # Only do this once + # Only do this once; established by setup_environment.sh + # shellcheck disable=SC2154 if passwd --status $ROOTLESS_USER then - echo "Updating $ROOTLESS_USER user permissions on possibly changed libpod code" + msg "Updating $ROOTLESS_USER user permissions on possibly changed libpod code" chown -R $ROOTLESS_USER:$ROOTLESS_USER "$GOPATH" "$GOSRC" return 0 fi - - cd $GOSRC + msg "************************************************************" + msg "Setting up rootless user '$ROOTLESS_USER'" + msg "************************************************************" + cd $GOSRC || exit 1 # Guarantee independence from specific values - ROOTLESS_UID=$[RANDOM+1000] - ROOTLESS_GID=$[RANDOM+1000] - echo "creating $ROOTLESS_UID:$ROOTLESS_GID $ROOTLESS_USER user" - groupadd -g $ROOTLESS_GID $ROOTLESS_USER - useradd -g $ROOTLESS_GID -u $ROOTLESS_UID --no-user-group --create-home $ROOTLESS_USER + rootless_uid=$[RANDOM+1000] + rootless_gid=$[RANDOM+1000] + msg "creating $rootless_uid:$rootless_gid $ROOTLESS_USER user" + groupadd -g $rootless_gid $ROOTLESS_USER + useradd -g $rootless_gid -u $rootless_uid --no-user-group --create-home $ROOTLESS_USER chown -R $ROOTLESS_USER:$ROOTLESS_USER "$GOPATH" "$GOSRC" - echo "creating ssh key pair for $USER" + msg "creating ssh key pair for $USER" [[ -r "$HOME/.ssh/id_rsa" ]] || \ ssh-keygen -P "" -f "$HOME/.ssh/id_rsa" - echo "Allowing ssh key for $ROOTLESS_USER" + msg "Allowing ssh key for $ROOTLESS_USER" (umask 077 && mkdir "/home/$ROOTLESS_USER/.ssh") chown -R $ROOTLESS_USER:$ROOTLESS_USER "/home/$ROOTLESS_USER/.ssh" install -o $ROOTLESS_USER -g $ROOTLESS_USER -m 0600 \ @@ -247,49 +165,38 @@ setup_rootless() { # Makes debugging easier cat /root/.ssh/authorized_keys >> "/home/$ROOTLESS_USER/.ssh/authorized_keys" - echo "Configuring subuid and subgid" + msg "Configuring subuid and subgid" grep -q "${ROOTLESS_USER}" /etc/subuid || \ - echo "${ROOTLESS_USER}:$[ROOTLESS_UID * 100]:65536" | \ + echo "${ROOTLESS_USER}:$[rootless_uid * 100]:65536" | \ tee -a /etc/subuid >> /etc/subgid # Env. vars set by Cirrus and setup_environment.sh must be explicitly # transferred to the test-user. - echo "Configuring rootless user's environment variables:" - echo "# Added by $GOSRC/$SCRIPT_PATH/lib.sh setup_rootless()" - _ENV_VAR_NAMES=$(awk 'BEGIN{for(v in ENVIRON) print v}' | \ - egrep -v "(^PATH$)|(^BASH_FUNC)|(^[[:punct:][:space:]]+)|$SECRET_ENV_RE" | \ - egrep "$ROOTLESS_ENV_RE" | \ - sort -u) - for _env_var_name in $_ENV_VAR_NAMES - do - # Works with older versions of bash - printf "${_env_var_name}=%q\n" "$(printenv $_env_var_name)" >> "/home/$ROOTLESS_USER/.bashrc" - done + msg "Configuring rootless user's environment variables:" + + ( + echo "# Added by ${BASH_SOURCE[0]} ${FUNCNAME[0]}()" + echo "export SETUP_ENVIRONMENT=1" + ) >> "/home/$ROOTLESS_USER/.bashrc" - echo "Ensure the systems ssh process is up and running within 5 minutes" + while read -r env_var_val; do + echo "export $env_var_val" >> "/home/$ROOTLESS_USER/.bashrc" + done <<<"$(passthrough_envars)" + chown $ROOTLESS_USER:$ROOTLESS_USER "/home/$ROOTLESS_USER/.bashrc" + cat "/home/$ROOTLESS_USER/.bashrc" | indent 2 + + msg "Ensure the systems ssh process is up and running within 5 minutes" systemctl start sshd - NOW=$(date +%s) - TIMEOUT=$(date --date '+5 minutes' +%s) - while [[ "$(date +%s)" -lt "$TIMEOUT" ]] - do - if timeout --foreground -k 1s 1s \ - ssh $ROOTLESS_USER@localhost \ - -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o CheckHostIP=no \ - true - then - break - else - sleep 2s - fi - done - [[ "$(date +%s)" -lt "$TIMEOUT" ]] || \ - die 11 "Timeout exceeded waiting for localhost ssh capability" + lilto ssh $ROOTLESS_USER@localhost \ + -o UserKnownHostsFile=/dev/null \ + -o StrictHostKeyChecking=no \ + -o CheckHostIP=no true } install_test_configs() { echo "Installing cni config, policy and registry config" - req_env_var GOSRC SCRIPT_BASE - cd $GOSRC + req_env_vars GOSRC SCRIPT_BASE + cd $GOSRC || exit 1 install -v -D -m 644 ./cni/87-podman-bridge.conflist /etc/cni/net.d/ # This config must always sort last in the list of networks (podman picks first one # as the default). This config prevents allocation of network address space used @@ -306,7 +213,7 @@ install_test_configs() { # can only run the compiled source version. remove_packaged_podman_files() { echo "Removing packaged podman files to prevent conflicts with source build and testing." - req_env_var OS_RELEASE_ID + req_env_vars OS_RELEASE_ID # If any binaries are resident they could cause unexpected pollution for unit in io.podman.service io.podman.socket @@ -324,9 +231,9 @@ remove_packaged_podman_files() { if [[ "$OS_RELEASE_ID" =~ "ubuntu" ]] then - LISTING_CMD="$SUDO dpkg-query -L podman" + LISTING_CMD="dpkg-query -L podman" else - LISTING_CMD="$SUDO rpm -ql podman" + LISTING_CMD="rpm -ql podman" fi # yum/dnf/dpkg may list system directories, only remove files @@ -334,29 +241,9 @@ remove_packaged_podman_files() { do # Sub-directories may contain unrelated/valuable stuff if [[ -d "$fullpath" ]]; then continue; fi - ooe.sh $SUDO rm -vf "$fullpath" + ooe.sh rm -vf "$fullpath" done # Be super extra sure and careful vs performant and completely safe - sync && echo 3 > /proc/sys/vm/drop_caches -} - -# The version of CRI-O and Kubernetes must always match -get_kubernetes_version(){ - # TODO: Look up the kube RPM/DEB version installed, or in $PACKAGE_DOWNLOAD_DIR - # and retrieve the major-minor version directly. - local KUBERNETES_VERSION="1.15" - echo "$KUBERNETES_VERSION" -} - -canonicalize_image_names() { - req_env_var IMGNAMES - echo "Adding all current base images to \$IMGNAMES for timestamp update" - export IMGNAMES="\ -$IMGNAMES -$UBUNTU_BASE_IMAGE -$PRIOR_UBUNTU_BASE_IMAGE -$FEDORA_BASE_IMAGE -$PRIOR_FEDORA_BASE_IMAGE -" + sync && echo 3 > /proc/sys/vm/drop_caches || true } diff --git a/contrib/cirrus/lib.sh.t b/contrib/cirrus/lib.sh.t deleted file mode 100755 index 643b5513d..000000000 --- a/contrib/cirrus/lib.sh.t +++ /dev/null @@ -1,121 +0,0 @@ -#!/usr/bin/env bash -# -# Unit tests for some functions in lib.sh -# -source $(dirname $0)/lib.sh - -# Iterator and return code; updated in test functions -testnum=0 -rc=0 - -function check_result { - testnum=$(expr $testnum + 1) - MSG=$(echo "$1" | tr -d '*>\012'|sed -e 's/^ \+//') - if [ "$MSG" = "$2" ]; then - echo "ok $testnum $(echo $3) = $(echo $MSG)" - else - echo "not ok $testnum $3" - echo "# expected: $2" - echo "# actual: $MSG" - rc=1 - fi -} - -############################################################################### -# tests for die() - -function test_die() { - local input_status=$1 - local input_msg=$2 - local expected_status=$3 - local expected_msg=$4 - - local msg - msg=$(die $input_status "$input_msg") - local status=$? - - check_result "$msg" "$expected_msg" "die $input_status $input_msg" -} - -test_die 1 "a message" 1 "a message" -test_die 2 "" 2 "FATAL ERROR (but no message given!) in test_die()" -test_die '' '' 1 "FATAL ERROR (but no message given!) in test_die()" - -############################################################################### -# tests for req_env_var() - -function test_rev() { - local input_args=$1 - local expected_status=$2 - local expected_msg=$3 - - # bash gotcha: doing 'local msg=...' on one line loses exit status - local msg - msg=$(req_env_var $input_args) - local status=$? - - check_result "$msg" "$expected_msg" "req_env_var $input_args" - check_result "$status" "$expected_status" "req_env_var $input_args (rc)" -} - -# error if called with no args -test_rev '' 1 'FATAL: req_env_var: invoked without arguments' - -# error if desired envariable is unset -unset FOO BAR -test_rev FOO 9 'FATAL: test_rev() requires $FOO to be non-empty' -test_rev BAR 9 'FATAL: test_rev() requires $BAR to be non-empty' -# OK if desired envariable was unset -FOO=1 -test_rev FOO 0 '' - -# OK if multiple vars are non-empty -FOO="stuff" -BAR="things" -ENV_VARS="FOO BAR" -test_rev "$ENV_VARS" 0 '' -unset BAR - -# ...but error if any single desired one is unset -test_rev "FOO BAR" 9 'FATAL: test_rev() requires $BAR to be non-empty' - -# ...and OK if all args are set -BAR=1 -test_rev "FOO BAR" 0 '' - -############################################################################### -# tests for item_test() - -function test_item_test { - local exp_msg=$1 - local exp_ret=$2 - local item=$3 - shift 3 - local test_args="$@" - local msg - msg=$(item_test "$item" "$@") - local status=$? - - check_result "$msg" "$exp_msg" "test_item $item $test_args" - check_result "$status" "$exp_ret" "test_item $item $test_args (actual rc $status)" -} - -# negative tests -test_item_test "FATAL: item_test() requires \$ITEM to be non-empty" 9 "" "" -test_item_test "FATAL: item_test() requires \$TEST_ARGS to be non-empty" 9 "foo" "" -test_item_test "not ok foo: -gt 5 ~= bar: too many arguments" 2 "foo" "-gt" "5" "~=" "bar" -test_item_test "not ok bar: a -ge 10: a: integer expression expected" 2 "bar" "a" "-ge" "10" -test_item_test "not ok basic logic: 0 -ne 0" 1 "basic logic" "0" "-ne" "0" - -# positive tests -test_item_test "ok snafu" 0 "snafu" "foo" "!=" "bar" -test_item_test "ok foobar" 0 "foobar" "one two three" "=" "one two three" -test_item_test "ok oh boy" 0 "oh boy" "line 1 -line2" "!=" "line 1 - -line2" -test_item_test "ok okay enough" 0 "okay enough" "line 1 -line2" "=" "line 1 -line2" - -exit $rc diff --git a/contrib/cirrus/logcollector.sh b/contrib/cirrus/logcollector.sh index fd5017b44..9dcc252c0 100755 --- a/contrib/cirrus/logcollector.sh +++ b/contrib/cirrus/logcollector.sh @@ -2,9 +2,10 @@ set -e +# shellcheck source=contrib/cirrus/lib.sh source $(dirname $0)/lib.sh -req_env_var CIRRUS_WORKING_DIR OS_RELEASE_ID RCLI +req_env_vars CIRRUS_WORKING_DIR OS_RELEASE_ID # Assume there are other log collection commands to follow - Don't # let one break another that may be useful, but also keep any @@ -33,14 +34,9 @@ case $1 in ginkgo) showrun cat $CIRRUS_WORKING_DIR/test/e2e/ginkgo-node-*.log ;; journal) showrun journalctl -b ;; podman) showrun ./bin/podman system info ;; - varlink) - if [[ "$RCLI" == "true" ]] - then - echo "(Trailing 100 lines of $VARLINK_LOG)" - showrun tail -100 $VARLINK_LOG - else - die 0 "\$RCLI is not 'true': $RCLI" - fi + server) + msg "(Trailing 100 lines of $PODMAN_SERVER_LOG)" + if [[ -r "$PODMAN_SERVER_LOG" ]]; then tail -100 $PODMAN_SERVER_LOG; fi ;; packages) # These names are common to Fedora and Ubuntu @@ -78,7 +74,7 @@ case $1 in echo "Kernel: " $(uname -r) echo "Cgroups: " $(stat -f -c %T /sys/fs/cgroup) # Any not-present packages will be listed as such - $PKG_LST_CMD ${PKG_NAMES[@]} | sort -u + $PKG_LST_CMD "${PKG_NAMES[@]}" | sort -u ;; - *) die 1 "Warning, $(basename $0) doesn't know how to handle the parameter '$1'" + *) die "Warning, $(basename $0) doesn't know how to handle the parameter '$1'" esac diff --git a/contrib/cirrus/networking.sh b/contrib/cirrus/networking.sh deleted file mode 100755 index 824d03e44..000000000 --- a/contrib/cirrus/networking.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash - -# This script attempts basic confirmation of functional networking -# by connecting to a set of essential external servers and failing -# if any cannot be reached. - -source $(dirname $0)/lib.sh - -while read host port -do - if [[ "$port" -eq "443" ]] - then - item_test "SSL/TLS to $host:$port" "$(echo -n '' | timeout 60 openssl s_client -quiet -no_ign_eof -connect $host:$port &> /dev/null; echo $?)" -eq "0" - else - item_test "Connect to $host:$port" "$(nc -zv -w 13 $host $port &> /dev/null; echo $?)" -eq 0 - fi -done < ${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/required_host_ports.txt diff --git a/contrib/cirrus/ooe.sh b/contrib/cirrus/ooe.sh deleted file mode 100755 index 0966b5ce8..000000000 --- a/contrib/cirrus/ooe.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash - -# This script executes a command while logging all output to a temporary -# file. If the command exits non-zero, then all output is sent to the console, -# before returning the exit code. If the script itself fails, the exit code 121 -# is returned. - -set -eo pipefail - -SCRIPT_BASEDIR="$(basename $0)" - -badusage() { - echo "Incorrect usage: $SCRIPT_BASEDIR) <command> [options]" > /dev/stderr - echo "ERROR: $1" - exit 121 -} - -COMMAND="$@" -[[ -n "$COMMAND" ]] || badusage "No command specified" - -OUTPUT_TMPFILE="$(mktemp -p '' ${SCRIPT_BASEDIR}_output_XXXX)" -output_on_error() { - RET=$? - set +e - if [[ "$RET" -ne "0" ]] - then - echo "---------------------------" - cat "$OUTPUT_TMPFILE" - echo "[$(date --iso-8601=second)] <exit $RET> $COMMAND" - fi - rm -f "$OUTPUT_TMPFILE" -} -trap "output_on_error" EXIT - -"$@" 2>&1 | while IFS='' read LINE # Preserve leading/trailing whitespace -do - # Every stdout and (copied) stderr line - echo "[$(date --iso-8601=second)] $LINE" -done >> "$OUTPUT_TMPFILE" diff --git a/contrib/cirrus/rootless_test.sh b/contrib/cirrus/rootless_test.sh deleted file mode 100755 index 31db18302..000000000 --- a/contrib/cirrus/rootless_test.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash - -set -e - -echo "$(date --rfc-3339=seconds) $(basename $0) started with '$*'" - -source $(dirname $0)/lib.sh - -if [[ "$UID" == "0" ]] -then - echo "$(basename $0): Error: Expected to be running as a regular user" - exit 1 -fi - -TESTSUITE=${1?Missing TESTSUITE argument (arg1)} -LOCAL_OR_REMOTE=${2?Missing LOCAL_OR_REMOTE argument (arg2)} - -# Ensure environment setup correctly -req_env_var GOSRC ROOTLESS_USER - -echo "." -echo "Hello, my name is $USER and I live in $PWD can I be your friend?" -echo "." - -show_env_vars - -set -x -cd "$GOSRC" -make -make varlink_generate -make test-binaries -make ${LOCAL_OR_REMOTE}${TESTSUITE} diff --git a/contrib/cirrus/runner.sh b/contrib/cirrus/runner.sh new file mode 100755 index 000000000..82e512de9 --- /dev/null +++ b/contrib/cirrus/runner.sh @@ -0,0 +1,233 @@ +#!/bin/bash + +set -eo pipefail + +# This script is intended to be called by automation or humans, +# from a specially configured environment. Depending on the contents +# of various variable, entirely different operations will be performed. + +# shellcheck source=contrib/cirrus/lib.sh +source $(dirname $0)/lib.sh + +exec_container() { + local var_val + local cmd + # Required to be defined by caller + # shellcheck disable=SC2154 + msg "Re-executing runner inside container: $CTR_FQIN" + msg "************************************************************" + + req_env_vars CTR_FQIN TEST_ENVIRON CONTAINER SECRET_ENV_RE + + # Line-separated arguments which include shell-escaped special characters + declare -a envargs + while read -r var_val; do + envargs+=("-e $var_val") + done <<<"$(passthrough_envars)" + + # VM Images and Container images are built using (nearly) identical operations. + set -x + # shellcheck disable=SC2154 + exec podman run --rm --privileged --net=host --cgroupns=host \ + -v /dev/fuse:/dev/fuse \ + -v "$GOPATH:$GOPATH:Z" \ + --workdir "$GOSRC" \ + -e "CONTAINER=1" \ + "${envargs[@]}" \ + $CTR_FQIN bash -c "$SCRIPT_BASE/setup_environment.sh && $SCRIPT_BASE/runner.sh" +} + +build_swagger() { + local download_url + # Building this is a PITA, just grab binary for use in automation + # Ref: https://goswagger.io/install.html#static-binary + download_url=$(\ + curl -s https://api.github.com/repos/go-swagger/go-swagger/releases/latest | \ + jq -r '.assets[] | select(.name | contains("linux_amd64")) | .browser_download_url') + curl -o /usr/local/bin/swagger -L'#' "$download_url" + chmod +x /usr/local/bin/swagger + + cd $GOSRC + make swagger +} + +altbuild() { + req_env_vars ALT_NAME + # Defined in .cirrus.yml + # shellcheck disable=SC2154 + msg "Performing alternate build: $ALT_NAME" + msg "************************************************************" + cd $GOSRC + case "$ALT_NAME" in + *Each*) + git fetch origin + make build-all-new-commits GIT_BASE_BRANCH=origin/$DEST_BRANCH + ;; + *Windows*) + make podman-remote-windows-release + make podman.msi + ;; + *Without*) + make build-no-cgo + ;; + *varlink-API) + export SUGGESTION='remove API.md, then "make varlink_api_generate" and commit changes.' + make varlink_api_generate BUILDTAGS="varlink" + ./hack/tree_status.sh + ;; + *varlink-binaries) + make clean BUILDTAGS="varlink" binaries + ;; + *RPM*) + make -f ./.copr/Makefile + rpmbuild --rebuild ./podman-*.src.rpm + ;; + *Static*) + req_env_vars CTR_FQIN + [[ "$UID" -eq 0 ]] || \ + die "Static build must execute nixos container as root on host" + mkdir -p /var/cache/nix + podman run -i --rm -v /var/cache/nix:/mnt/nix:Z \ + $CTR_FQIN cp -rfT /nix /mnt/nix + podman run -i --rm -v /var/cache/nix:/nix:Z \ + -v $PWD:$PWD:Z -w $PWD $CTR_FQIN \ + nix --print-build-logs --option cores 4 --option max-jobs 4 \ + build --file ./nix/ + # result symlink is absolute from container perspective :( + cp /var/cache/$(readlink result)/bin/podman ./ # for cirrus-ci artifact + rm result # makes cirrus puke + ;; + *) + die "Unknown/Unsupported \$$ALT_NAME '$ALT_NAME'" + esac +} + +integration_outputfilter() { + # Use similar format as human-friendly task name from .cirrus.yml + # shellcheck disable=SC2154 + output_name="$TEST_FLAVOR-$PODBIN_NAME-$DISTRO_NV-$PRIV_NAME-$TEST_ENVIRON" + # Requires stdin and stderr combined! + cat - \ + |& awk --file "${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/timestamp.awk" \ + |& "${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/logformatter" "$output_name" +} + +# Handle local|remote integration|system testing in a uniform way +dotest() { + local testsuite="$1" + req_env_vars testsuite CONTAINER TEST_ENVIRON PRIV_NAME + + # shellcheck disable=SC2154 + if ((CONTAINER==0)) && [[ "$TEST_ENVIRON" == "container" ]]; then + exec_container # does not return + fi; + + # shellcheck disable=SC2154 + if [[ "$PRIV_NAME" == "rootless" ]] && [[ "$UID" -eq 0 ]]; then + req_env_vars ROOTLESS_USER + msg "Re-executing runner through ssh as user '$ROOTLESS_USER'" + msg "************************************************************" + set -x + exec ssh $ROOTLESS_USER@localhost \ + -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ + -o CheckHostIP=no $GOSRC/$SCRIPT_BASE/runner.sh + # does not return + fi + + output_filter="cat" # no filter + if [[ "$testsuite" == "integration" ]]; then + output_filter=integration_outputfilter + fi + + # containers/automation sets this to 0 for it's dbg() function + # but the e2e integration tests are also sensitive to it. + unset DEBUG + + # shellcheck disable=SC2154 + case "$PODBIN_NAME" in + podman) + # ginkgo doesn't play nicely with C Go + make local${testsuite} \ + |& "$output_filter" + ;; + remote) + make remote${testsuite} PODMAN_SERVER_LOG=$PODMAN_SERVER_LOG \ + |& "$output_filter" + ;; + esac +} + +msg "************************************************************" +# Required to be defined by caller +# shellcheck disable=SC2154 +msg "Runner executing $TEST_FLAVOR $PODBIN_NAME-tests as $PRIV_NAME on $DISTRO_NV($OS_REL_VER)" +if ((CONTAINER)); then + # shellcheck disable=SC2154 + msg "Current environment container image: $CTR_FQIN" +else + # shellcheck disable=SC2154 + msg "Current environment VM image: $VM_IMAGE_NAME" +fi +msg "************************************************************" + +((${SETUP_ENVIRONMENT:-0})) || \ + die "Expecting setup_environment.sh to have completed successfully" + +cd "${GOSRC}/" + +case "$TEST_FLAVOR" in + ext_svc) $SCRIPT_BASE/ext_svc_check.sh ;; + smoke) + make gofmt + make .gitvalidation + ;; + automation) + $SCRIPT_BASE/cirrus_yaml_test.py + req_env_vars CI DEST_BRANCH IMAGE_SUFFIX TEST_FLAVOR TEST_ENVIRON \ + PODBIN_NAME PRIV_NAME DISTRO_NV CONTAINER USER HOME \ + UID GID AUTOMATION_LIB_PATH SCRIPT_BASE OS_RELEASE_ID \ + OS_RELEASE_VER CG_FS_TYPE + bigto ooe.sh dnf install -y ShellCheck # small/quick addition + $SCRIPT_BASE/shellcheck.sh + ;; + altbuild) altbuild ;; + build) + make podman-release + make podman-remote-linux-release + ;; + validate) + # Confirm compiile via prior task + cache + bin/podman --version + bin/podman-remote --version + make validate # Some items require a build + ;; + bindings) + # shellcheck disable=SC2155 + export PATH=$PATH:$GOSRC/hack + cd pkg/bindings/test && ginkgo -trace -noColor -debug -r + ;; + endpoint) + make test-binaries + make endpoint + ;; + swagger) + build_swagger + # Cirrus-CI Artifact instruction expects file here + cp -v $GOSRC/pkg/api/swagger.yaml $GOSRC/ + ;; + vendor) + make vendor + ./hack/tree_status.sh + ;; + docker-py) msg "This is docker-py stub, it is only a stub" ;; + unit) make localunit ;; + int) dotest integration ;; + sys) dotest system ;; + release) + if bin/podman info |& grep -Eq -- '-dev'; then + die "Releases must never contain '-dev' in output of 'podman info'" + fi + ;; + *) + die "Unknown/Unsupported \$TEST_FLAVOR=$TEST_FLAVOR" ;; +esac diff --git a/contrib/cirrus/setup_container_environment.sh b/contrib/cirrus/setup_container_environment.sh deleted file mode 100755 index 72542df17..000000000 --- a/contrib/cirrus/setup_container_environment.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash -set -e - -source $(dirname $0)/lib.sh - -req_env_var GOSRC OS_RELEASE_ID CONTAINER_RUNTIME - -# Since CRIU 3.11 has been pushed to Fedora 28 the checkpoint/restore -# test cases are actually run. As CRIU uses iptables to lock and unlock -# the network during checkpoint and restore it needs the following two -# modules loaded. -modprobe ip6table_nat || : -modprobe iptable_nat || : - -# Pull the test image -${CONTAINER_RUNTIME} pull ${IN_PODMAN_IMAGE} diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh index b406d7b5c..c064b6840 100755 --- a/contrib/cirrus/setup_environment.sh +++ b/contrib/cirrus/setup_environment.sh @@ -2,119 +2,187 @@ set -e +# shellcheck source=./contrib/cirrus/lib.sh source $(dirname $0)/lib.sh -req_env_var USER HOME GOSRC SCRIPT_BASE SETUP_MARKER_FILEPATH +die_unknown() { + local var_name="$1" + req_env_vars var_name + local var_value="${!var_name}" + die "Unknown/unsupported \$$var_name '$var_value'" +} -# Ensure this script only executes successfully once and always logs ending timestamp -if [[ -e "$SETUP_MARKER_FILEPATH" ]]; then - show_env_vars - exit 0 -fi +msg "************************************************************" +msg "Setting up runtime environment" +msg "************************************************************" +show_env_vars -exithandler() { - RET=$? - echo "." - echo "$(basename $0) exit status: $RET" - [[ "$RET" -eq "0" ]] && date +%s >> "$SETUP_MARKER_FILEPATH" - show_env_vars - [[ "$RET" -eq "0" ]] || warn "Non-zero exit caused by error ABOVE env. var. display." -} -trap exithandler EXIT +req_env_vars USER HOME GOSRC SCRIPT_BASE TEST_FLAVOR TEST_ENVIRON \ + PODBIN_NAME PRIV_NAME DISTRO_NV # Verify basic dependencies for depbin in go rsync unzip sha256sum curl make python3 git do if ! type -P "$depbin" &> /dev/null then - echo "***** WARNING: $depbin binary not found in $PATH *****" + warn "$depbin binary not found in $PATH" fi done -# Sometimes environment setup needs to vary between distros -# Note: This should only be used for environment variables, and temporary workarounds. +# This is a possible manual maintenance gaff, check to be sure everything matches. +# shellcheck disable=SC2154 +[[ "$DISTRO_NV" == "$OS_REL_VER" ]] || \ + die "Automation spec. '$DISTRO_NV'; actual host '$OS_REL_VER'" + +# Only allow this script to execute once +if ((${SETUP_ENVIRONMENT:-0})); then + # Comes from automation library + # shellcheck disable=SC2154 + warn "Not executing $SCRIPT_FILENAME again" + exit 0 +fi + cd "${GOSRC}/" -case "${OS_RELEASE_ID}" in - ubuntu) - ;; - fedora) - # All SELinux distros need this for systemd-in-a-container - setsebool container_manage_cgroup true - if [[ "$ADD_SECOND_PARTITION" == "true" ]]; then - bash "$SCRIPT_BASE/add_second_partition.sh" +# Defined by lib.sh: Does the host support cgroups v1 or v2 +case "$CG_FS_TYPE" in + tmpfs) + if ((CONTAINER==0)); then + warn "Forcing testing with runc instead of crun" + if [[ "$OS_RELEASE_ID" == "ubuntu" ]]; then + echo "export OCI_RUNTIME=/usr/lib/cri-o-runc/sbin/runc" >> /etc/environment + else + echo "export OCI_RUNTIME=/usr/bin/runc" >> /etc/environment + fi fi - - warn "Forcing systemd cgroup manager" - X=$(echo "export CGROUP_MANAGER=systemd" | \ - tee -a /etc/environment) && eval "$X" && echo "$X" ;; - centos) # Current VM is an image-builder-image no local podman/testing - echo "No further setup required for VM image building" - exit 0 + cgroup2fs) + if ((CONTAINER==0)); then + # This is necessary since we've built/installed from source, + # which uses runc as the default. + warn "Forcing testing with crun instead of runc" + echo "export OCI_RUNTIME=/usr/bin/crun" >> /etc/environment + fi ;; - *) bad_os_id_ver ;; + *) die_unknown CG_FS_TYPE esac -# Reload to incorporate any changes from above -source "$SCRIPT_BASE/lib.sh" +# Required to be defined by caller: Which distribution are we testing on +# shellcheck disable=SC2154 +case "$DISTRO_NV" in + ubuntu*) ;; + fedora*) + if ((CONTAINER==0)); then # Not yet running inside a container + msg "Configuring / Expanding host storage." + # VM is setup to allow flexibility in testing alternate storage. + # For general use, simply make use of all available space. + ooe.sh bash "$SCRIPT_BASE/add_second_partition.sh" -case "$CG_FS_TYPE" in - tmpfs) - warn "Forcing testing with runc instead of crun" - # On ubuntu, the default runc is usually not new enough. - if [[ "$OS_RELEASE_ID" == "ubuntu" ]]; then - X=$(echo "export OCI_RUNTIME=/usr/lib/cri-o-runc/sbin/runc" | \ - tee -a /etc/environment) && eval "$X" && echo "$X" + # All SELinux distros need this for systemd-in-a-container + msg "Enabling container_manage_cgroup" + setsebool container_manage_cgroup true + fi + ;; + *) die_unknown DISTRO_NV +esac + +# Required to be defined by caller: The environment where primary testing happens +# shellcheck disable=SC2154 +case "$TEST_ENVIRON" in + host) + if [[ "$OS_RELEASE_ID" == "fedora" ]]; then + # The e2e tests wrongly guess `--cgroup-manager cgroupfs` + msg "Forcing CGROUP_MANAGER=systemd" + _cgm="export CGROUP_MANAGER=systemd" + echo "$_cgm" >> /etc/environment + source /etc/environment + fi + ;; + container) + if ((CONTAINER==0)); then # not yet inside a container + msg "Force loading iptables modules" + # Since CRIU 3.11, uses iptables to lock and unlock + # the network during checkpoint and restore. Needs + # the following two modules loaded on the host. + modprobe ip6table_nat || : + modprobe iptable_nat || : else - X=$(echo "export OCI_RUNTIME=/usr/bin/runc" | \ - tee -a /etc/environment) && eval "$X" && echo "$X" + # The e2e tests wrongly guess `--cgroup-manager systemd` + msg "Forcing CGROUP_MANAGER=cgroupfs" + _cgm="export CGROUP_MANAGER=cgroupfs" + echo "$_cgm" >> /etc/environment + source /etc/environment fi ;; - cgroup2fs) - # This is necessary since we've built/installed from source, which uses runc as the default. - warn "Forcing testing with crun instead of runc" - X=$(echo "export OCI_RUNTIME=/usr/bin/crun" | \ - tee -a /etc/environment) && eval "$X" && echo "$X" + *) die_unknown TEST_ENVIRON +esac + +# Required to be defined by caller: Are we testing as root or a regular user +# shellcheck disable=SC2154 +case "$PRIV_NAME" in + root) + if [[ "$TEST_ENVIRON" == "container" ]] && ((container)); then + # There's no practical way to detect userns w/in a container + # affected/related tests are sensitive to this variable. + _suns='export SKIP_USERNS=1' + echo "$_suns" >> /etc/environment + source /etc/environment + fi ;; - *) - die 110 "Unsure how to handle cgroup filesystem type '$CG_FS_TYPE'" + rootless) + _ru="export ROOTLESS_USER='${ROOTLESS_USER:-some${RANDOM}dude}'" + echo "$_ru" >> /etc/environment + source /etc/environment + setup_rootless ;; + *) die_unknown PRIV_NAME esac -# Must execute before possible setup_rootless() -make install.tools +# Required to be defined by caller: Are we testing podman or podman-remote client +# shellcheck disable=SC2154 +case "$PODBIN_NAME" in + podman) ;; + remote) ;; + *) die_unknown PODBIN_NAME +esac -case "$SPECIALMODE" in - none) - [[ -n "$CROSS_PLATFORM" ]] || \ - remove_packaged_podman_files +# Required to be defined by caller: The primary type of testing that will be performed +# shellcheck disable=SC2154 +case "$TEST_FLAVOR" in + ext_svc) ;; + smoke) ;& + validate) + # For some reason, this is also needed for validation + make .install.pre-commit ;; + automation) ;; + altbuild) + # Defined in .cirrus.yml + # shellcheck disable=SC2154 + if [[ "$ALT_NAME" =~ RPM ]]; then + bigto dnf install -y glibc-minimal-langpack rpm-build + fi + ;& + docker-py) ;& + build) make clean ;; + unit) ;; + int) ;& + sys) ;& + bindings) ;& + swagger) ;& endpoint) - remove_packaged_podman_files - ;; - bindings) - remove_packaged_podman_files - ;; - rootless) - # Only do this once, even if ROOTLESS_USER (somehow) changes - if ! grep -q 'ROOTLESS_USER' /etc/environment - then - X=$(echo "export ROOTLESS_USER='${ROOTLESS_USER:-some${RANDOM}dude}'" | \ - tee -a /etc/environment) && eval "$X" && echo "$X" - X=$(echo "export SPECIALMODE='${SPECIALMODE}'" | \ - tee -a /etc/environment) && eval "$X" && echo "$X" - X=$(echo "export RCLI='${RCLI}'" | \ - tee -a /etc/environment) && eval "$X" && echo "$X" - setup_rootless + # Use existing host bits when testing is to happen inside a container + # since this script will run again in that environment. + # shellcheck disable=SC2154 + if ((CONTAINER==0)) && [[ "$TEST_ENVIRON" == "host" ]]; then + remove_packaged_podman_files + make install PREFIX=/usr ETCDIR=/etc fi - remove_packaged_podman_files - ;; - in_podman) # Assumed to be Fedora - $SCRIPT_BASE/setup_container_environment.sh ;; - *) - die 111 "Unsupported \$SPECIALMODE: $SPECIALMODE" + vendor) make clean ;; + release) ;; + *) die_unknown TEST_FLAVOR esac -install_test_configs +# Must be the very last command. Establishes successful setup. +echo 'export SETUP_ENVIRONMENT=1' >> /etc/environment diff --git a/contrib/cirrus/shellcheck.sh b/contrib/cirrus/shellcheck.sh new file mode 100755 index 000000000..edf8248d3 --- /dev/null +++ b/contrib/cirrus/shellcheck.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -eo pipefail + +# shellcheck source=./contrib/cirrus/lib.sh +source $(dirname $0)/lib.sh + +cd $CIRRUS_WORKING_DIR +shellcheck --color=always --format=tty \ + --shell=bash --external-sources \ + --enable add-default-case,avoid-nullary-conditions,check-unassigned-uppercase \ + --exclude SC2046,SC2034,SC2090,SC2064 \ + --wiki-link-count=0 --severity=warning \ + $SCRIPT_BASE/*.sh + +echo "Shellcheck: PASS" diff --git a/contrib/cirrus/system_test.sh b/contrib/cirrus/system_test.sh deleted file mode 100755 index dbee078b6..000000000 --- a/contrib/cirrus/system_test.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env bash - -set -e - -source $(dirname $0)/lib.sh - -req_env_var GOSRC SCRIPT_BASE OS_RELEASE_ID OS_RELEASE_VER CONTAINER_RUNTIME VARLINK_LOG - -LOCAL_OR_REMOTE=local -if [[ "$RCLI" = "true" ]]; then - LOCAL_OR_REMOTE=remote -fi - -# Our name must be of the form xxxx_test or xxxx_test.sh, where xxxx is -# the test suite to run; currently (2019-05) the only option is 'integration' -# but pr2947 intends to add 'system'. -TESTSUITE=$(expr $(basename $0) : '\(.*\)_test') -if [[ -z $TESTSUITE ]]; then - die 1 "Script name is not of the form xxxx_test.sh" -fi - -cd "$GOSRC" - -case "$SPECIALMODE" in - in_podman) - ${CONTAINER_RUNTIME} run --rm --privileged --net=host --cgroupns=host \ - -v $GOSRC:$GOSRC:Z \ - --workdir $GOSRC \ - -e "CGROUP_MANAGER=cgroupfs" \ - -e "STORAGE_OPTIONS=--storage-driver=vfs" \ - -e "CRIO_ROOT=$GOSRC" \ - -e "PODMAN_BINARY=/usr/bin/podman" \ - -e "CONMON_BINARY=/usr/libexec/podman/conmon" \ - -e "DIST=$OS_RELEASE_ID" \ - -e "CONTAINER_RUNTIME=$CONTAINER_RUNTIME" \ - $IN_PODMAN_IMAGE bash $GOSRC/$SCRIPT_BASE/container_test.sh -b -i -t - ;; - rootless) - req_env_var ROOTLESS_USER - ssh $ROOTLESS_USER@localhost \ - -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ - -o CheckHostIP=no $GOSRC/$SCRIPT_BASE/rootless_test.sh ${TESTSUITE} ${LOCAL_OR_REMOTE} - ;; - endpoint) - make - make install PREFIX=/usr ETCDIR=/etc - make test-binaries - make endpoint - ;; - bindings) - make - make install PREFIX=/usr ETCDIR=/etc - export PATH=$PATH:`pwd`/hack - cd pkg/bindings/test && ginkgo -trace -noColor -debug -r - ;; - none) - make - make install PREFIX=/usr ETCDIR=/etc - make test-binaries - make .install.bats - make ${LOCAL_OR_REMOTE}${TESTSUITE} PODMAN_SERVER_LOG=$PODMAN_SERVER_LOG - ;; - *) - die 110 "Unsupported \$SPECIALMODE: $SPECIALMODE" -esac diff --git a/contrib/cirrus/unit_test.sh b/contrib/cirrus/unit_test.sh deleted file mode 100755 index 17a618a1c..000000000 --- a/contrib/cirrus/unit_test.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash - -set -e - -source $(dirname $0)/lib.sh - -req_env_var GOSRC - -cd "$GOSRC" -make install.tools -make localunit - -case "$SPECIALMODE" in - in_podman) ;& - bindings) ;& - rootless) ;& - none) - make - ;; - *) - die 109 "Unsupported \$SPECIAL_MODE: $SPECIALMODE" -esac diff --git a/contrib/cirrus/update_meta.sh b/contrib/cirrus/update_meta.sh deleted file mode 100755 index 6e4a473e9..000000000 --- a/contrib/cirrus/update_meta.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash - -source $(dirname $0)/lib.sh - -# N/B: This script is expected to wrap $ENTRYPOINT when executing under the -# 'meta' Cirrus task on the libpod repo. -ENTRYPOINT=/usr/local/bin/entrypoint.sh - -req_env_var IMGNAMES BUILDID REPOREF GCPJSON GCPNAME GCPPROJECT CIRRUS_CI - -[[ -x "$ENTRYPOINT" ]] || \ - die 2 "Expecting to find an installed entrypoint script $ENTRYPOINT." - -# A better way of checking isn't compatible with old but functional images -# in-use by other repos. -grep -q 'compute images update' "$ENTRYPOINT" || \ - die 3 "Expecting to be running inside a specific imgts container image" - -canonicalize_image_names - -# Executing inside a container; proper hand-off for process control -exec $ENTRYPOINT diff --git a/contrib/cirrus/upload_release_archive.sh b/contrib/cirrus/upload_release_archive.sh deleted file mode 100755 index 2e2f4ddde..000000000 --- a/contrib/cirrus/upload_release_archive.sh +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/env bash - -set -eo pipefail - -source $(dirname $0)/lib.sh - -req_env_var CI UPLDREL_IMAGE CIRRUS_BUILD_ID GOSRC RELEASE_GCPJSON RELEASE_GCPNAME RELEASE_GCPROJECT - -[[ "$CI" == "true" ]] || \ - die 56 "$0 must be run under Cirrus-CI to function" - -SWAGGER_FILEPATH="pkg/api/swagger.yaml" - -# We store "releases" for each PR, mostly to validate the process is functional -unset PR_OR_BRANCH BUCKET -if [[ -n "$CIRRUS_PR" ]] -then - PR_OR_BRANCH="pr$CIRRUS_PR" - BUCKET="libpod-pr-releases" -elif [[ -n "$CIRRUS_BRANCH" ]] -then - # Only release binaries for docs - if [[ $CIRRUS_TASK_NAME =~ "docs" ]] - then - PR_OR_BRANCH="$CIRRUS_BRANCH" - BUCKET="libpod-$CIRRUS_BRANCH-releases" - else - warn "" "Skipping release processing for non-docs task." - exit 0 - fi -else - die 1 "Expecting either \$CIRRUS_PR or \$CIRRUS_BRANCH to be non-empty." -fi - -# Functional local podman required for uploading -echo "Verifying a local, functional podman, building one if necessary." -[[ -n "$(type -P podman)" ]] || \ - make install PREFIX=/usr || \ - die 57 "$0 requires working podman binary on path to function" - -TMPF=$(mktemp -p '' $(basename $0)_XXXX.json) -trap "rm -f $TMPF" EXIT -set +x -echo "$RELEASE_GCPJSON" > "$TMPF" -[[ "$OS_RELEASE_ID" == "ubuntu" ]] || \ - chcon -t container_file_t "$TMPF" -unset RELEASE_GCPJSON - -cd $GOSRC -for filename in $(ls -1 $SWAGGER_FILEPATH) -do - unset EXT - EXT=$(echo "$filename" | sed -r -e 's/.+\.(.+$)/\1/g') - if [[ -z "$EXT" ]] || [[ "$EXT" == "$filename" ]] - then - echo "Warning: Not processing $filename (invalid extension '$EXT')" - continue - fi - if [[ "$EXT" =~ "gz" ]] - then - EXT="tar.gz" - fi - - if [[ $filename == $SWAGGER_FILEPATH ]] - then - # Support other tools referencing branch and/or version-specific refs. - TO_FILENAME="swagger-${RELEASE_VERSION}-${PR_OR_BRANCH}.yaml" - # For doc. ref. this must always be a static filename, e.g. swagger-latest-master.yaml - ALSO_FILENAME="swagger-latest-${PR_OR_BRANCH}.yaml" - else - die "Uploading non-docs files has been disabled" - fi - - [[ "$OS_RELEASE_ID" == "ubuntu" ]] || \ - chcon -t container_file_t "$filename" - - echo "Running podman ... $UPLDREL_IMAGE for $filename -> $TO_FILENAME" - podman run -i --rm \ - -e "GCPNAME=$RELEASE_GCPNAME" \ - -e "GCPPROJECT=$RELEASE_GCPROJECT" \ - -e "GCPJSON_FILEPATH=$TMPF" \ - -e "FROM_FILEPATH=/tmp/$filename" \ - -e "TO_FILENAME=$TO_FILENAME" \ - -e "ALSO_FILENAME=$ALSO_FILENAME" \ - -e "PR_OR_BRANCH=$PR_OR_BRANCH" \ - -e "BUCKET=$BUCKET" \ - -v "$TMPF:$TMPF:ro" \ - -v "$(realpath $GOSRC/$filename):/tmp/$filename:ro" \ - $UPLDREL_IMAGE -done |