summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorChris Evich <cevich@redhat.com>2020-06-25 17:38:33 -0400
committerChris Evich <cevich@redhat.com>2020-10-02 11:53:04 -0400
commit2c9084e2245834094e14e9105e64b6062d70a0f1 (patch)
treee2e5c0be702cdae5919489fd9a1ed1b6f87d135a
parentb58980a43ccfcef5134274c3963bf68cc51a4983 (diff)
downloadpodman-2c9084e2245834094e14e9105e64b6062d70a0f1.tar.gz
podman-2c9084e2245834094e14e9105e64b6062d70a0f1.tar.bz2
podman-2c9084e2245834094e14e9105e64b6062d70a0f1.zip
Cirrus: Implement podman automation 2.0
Reimplement CI-automation to remove accumulated technical-debt and optimize workflow. The task-dependency graph designed goal was to shorten it's depth and increase width (i.e. more parallelism). A reduction in redundant building (and 3rd party module download) was also realized by caching `$GOPATH` and `$GOCACHE` early on. This cache is then reused in favor of a fresh clone of the repository (when possible). Note: The system tests typically execute MUCH faster than the integration tests. However, contrary to a fail-fast/fail-early principal, they are executed last. This was implemented due to debug-ability related concerns/preferences of the primary (golang-centric) project developers. Signed-off-by: Chris Evich <cevich@redhat.com>
-rw-r--r--.cirrus.yml1098
-rw-r--r--.golangci.yml14
-rw-r--r--Makefile8
-rw-r--r--contrib/cirrus/README.md150
-rw-r--r--contrib/cirrus/add_second_partition.sh28
-rwxr-xr-xcontrib/cirrus/apiv2_test.sh65
-rwxr-xr-xcontrib/cirrus/build_release.sh34
-rwxr-xr-xcontrib/cirrus/build_swagger.sh17
-rwxr-xr-xcontrib/cirrus/cirrus_yaml_test.py15
-rw-r--r--contrib/cirrus/container_test.sh142
-rwxr-xr-xcontrib/cirrus/ext_svc_check.sh30
-rwxr-xr-xcontrib/cirrus/integration_test.sh65
-rw-r--r--contrib/cirrus/lib.sh379
-rwxr-xr-xcontrib/cirrus/lib.sh.t121
-rwxr-xr-xcontrib/cirrus/logcollector.sh18
-rwxr-xr-xcontrib/cirrus/networking.sh17
-rwxr-xr-xcontrib/cirrus/ooe.sh39
-rwxr-xr-xcontrib/cirrus/rootless_test.sh32
-rwxr-xr-xcontrib/cirrus/runner.sh233
-rwxr-xr-xcontrib/cirrus/setup_container_environment.sh16
-rwxr-xr-xcontrib/cirrus/setup_environment.sh230
-rwxr-xr-xcontrib/cirrus/shellcheck.sh16
-rwxr-xr-xcontrib/cirrus/system_test.sh65
-rwxr-xr-xcontrib/cirrus/unit_test.sh22
-rwxr-xr-xcontrib/cirrus/update_meta.sh22
-rwxr-xr-xcontrib/cirrus/upload_release_archive.sh90
26 files changed, 1179 insertions, 1787 deletions
diff --git a/.cirrus.yml b/.cirrus.yml
index 6f8cd0a30..75f095fe0 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -1,13 +1,5 @@
---
-# Only github users with write-access can define or use encrypted variables
-# Ref: https://cirrus-ci.org/guide/writing-tasks/#encrypted-variables
-# more specifically: https://cirrus-ci.com/settings/repository/6707778565701632
-gcp_credentials: ENCRYPTED[a28959877b2c9c36f151781b0a05407218cda646c7d047fc556e42f55e097e897ab63ee78369dae141dcf0b46a9d0cdd]
-
-# Default timeout for each task
-timeout_in: 120m
-
# Main collection of env. vars to set for all tasks and scripts.
env:
####
@@ -16,603 +8,643 @@ env:
# Name of the ultimate destination branch for this CI run, PR or post-merge.
DEST_BRANCH: "master"
# Overrides default location (/tmp/cirrus) for repo clone
- GOPATH: "/var/tmp/go"
+ GOPATH: &gopath "/var/tmp/go"
GOBIN: "${GOPATH}/bin"
- GOSRC: "/var/tmp/go/src/github.com/containers/podman"
- CIRRUS_WORKING_DIR: "/var/tmp/go/src/github.com/containers/podman"
+ GOCACHE: "${GOPATH}/cache"
+ GOSRC: &gosrc "/var/tmp/go/src/github.com/containers/podman"
+ CIRRUS_WORKING_DIR: *gosrc
# The default is 'sh' if unspecified
CIRRUS_SHELL: "/bin/bash"
# Save a little typing (path relative to $CIRRUS_WORKING_DIR)
SCRIPT_BASE: "./contrib/cirrus"
- # Command to prefix output lines with timing information
- # (can't do inline awk script, Cirrus-CI or YAML mangles quoting)
- TIMESTAMP: "awk --file ${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/timestamp.awk"
-
- # HTMLify ginkgo and bats logs
- LOGFORMAT: "${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/logformatter"
####
#### Cache-image names to test with (double-quotes around names are critical)
- ###
+ ####
FEDORA_NAME: "fedora-32"
PRIOR_FEDORA_NAME: "fedora-31"
UBUNTU_NAME: "ubuntu-20"
PRIOR_UBUNTU_NAME: "ubuntu-19"
- _BUILT_IMAGE_SUFFIX: "c4948709391728640"
- FEDORA_CACHE_IMAGE_NAME: "fedora-${_BUILT_IMAGE_SUFFIX}"
- PRIOR_FEDORA_CACHE_IMAGE_NAME: "prior-fedora-${_BUILT_IMAGE_SUFFIX}"
- UBUNTU_CACHE_IMAGE_NAME: "ubuntu-${_BUILT_IMAGE_SUFFIX}"
- PRIOR_UBUNTU_CACHE_IMAGE_NAME: "prior-ubuntu-${_BUILT_IMAGE_SUFFIX}"
+ # Google-cloud VM Images
+ IMAGE_SUFFIX: "c4948709391728640"
+ FEDORA_CACHE_IMAGE_NAME: "fedora-${IMAGE_SUFFIX}"
+ PRIOR_FEDORA_CACHE_IMAGE_NAME: "prior-fedora-${IMAGE_SUFFIX}"
+ UBUNTU_CACHE_IMAGE_NAME: "ubuntu-${IMAGE_SUFFIX}"
+ PRIOR_UBUNTU_CACHE_IMAGE_NAME: "prior-ubuntu-${IMAGE_SUFFIX}"
- ####
- #### Default to NOT operating in any special-case testing mode
- ####
- SPECIALMODE: "none" # don't do anything special
- RCLI: 'false' # don't test remote client by default
- ADD_SECOND_PARTITION: 'false' # will certainly fail inside containers
- MOD_CONTAINERS_CONF: 'true' # Update containers.conf runtime if required by OS environment
+ # Container FQIN's
+ FEDORA_CONTAINER_FQIN: "quay.io/libpod/fedora_podman:${IMAGE_SUFFIX}"
+ PRIOR_FEDORA_CONTAINER_FQIN: "quay.io/libpod/prior-fedora_podman:${IMAGE_SUFFIX}"
+ UBUNTU_CONTAINER_FQIN: "quay.io/libpod/ubuntu_podman:${IMAGE_SUFFIX}"
+ PRIOR_UBUNTU_CONTAINER_FQIN: "quay.io/libpod/prior-ubuntu_podman:${IMAGE_SUFFIX}"
####
- #### Credentials and other secret-sauces, decrypted at runtime when authorized.
+ #### Control variables that determine what to run and how to run it.
+ #### N/B: Required ALL of these are set for every single task.
####
- # Name where this repositories VM images are stored
- GCP_PROJECT_ID: libpod-218412
- # Service-account client_email - needed to build images
- SERVICE_ACCOUNT: ENCRYPTED[702a8e07e27a6faf7988fcddcc068c2ef2bb182a5aa671f5ccb7fbbfb891c823aa4a7856fb17240766845dbd68bd3f90]
- # Service account username part of client_email - for ssh'ing into VMs
- GCE_SSH_USERNAME: 'cirrus-ci'
-
-# Default VM to use unless set or modified by task
-gce_instance:
- image_project: $GCP_PROJECT_ID
- zone: "us-central1-a" # Required by Cirrus for the time being
- cpu: 2
- memory: "4Gb"
- disk: 200
- # A matrix could be used here, for now just one VM
- image_name: "${FEDORA_CACHE_IMAGE_NAME}"
-
-
-# Every *_task runs in parallel in separate VMsd. The name prefix only for reference
-# in WebUI, and will be followed by matrix details. This task gates all others with
-# quick format, lint, and unit tests on the standard platform.
-gating_task:
-
- # Only run this on PRs, never during post-merge testing (for speed).
- only_if: $CIRRUS_BRANCH != $DEST_BRANCH
+ TEST_FLAVOR: # int, sys, ext_svc, smoke, automation, etc.
+ TEST_ENVIRON: host # 'host' or 'container'
+ PODBIN_NAME: podman # 'podman' or 'remote'
+ PRIV_NAME: root # 'root' or 'rootless'
+ DISTRO_NV: $FEDORA_NAME # any {PRIOR_,}{FEDORA,UBUNTU}_NAME value
+ VM_IMAGE_NAME: # One of the "Google-cloud VM Images" (above)
+ CTR_FQIN: # One of the "Container FQIN's" (above)
- env:
- CIRRUS_WORKING_DIR: "/usr/src/libpod"
- SRCPATH: "$CIRRUS_WORKING_DIR"
- # Runs within Cirrus's "community cluster"
- container:
- # Note: Image has dual purpose, see contrib/gate/README.md
- # The entrypoint.sh script ensures a prestine copy of $SRCPATH is
- # available at $GOSRC before executing make instructions.
- image: "quay.io/libpod/gate:master"
- cpu: 8
- memory: 12
+# Default timeout for each task
+timeout_in: 60m
- timeout_in: 20m
- # Custom cloning is required to satisfy lint/validation needs
- clone_script: |
+gcp_credentials: ENCRYPTED[a28959877b2c9c36f151781b0a05407218cda646c7d047fc556e42f55e097e897ab63ee78369dae141dcf0b46a9d0cdd]
+
+
+# Attempt to prevent flakes by confirming all required external/3rd-party
+# services are available and functional.
+ext_svc_check_task:
+ alias: 'ext_svc_check' # int. ref. name - required for depends_on reference
+ name: "Ext. services" # Displayed Title - has no other significance
+ # Default/small container image to execute tasks with
+ container: &smallcontainer
+ image: ${CTR_FQIN}
+ # Resources are limited across ALL currently executing tasks
+ # ref: https://cirrus-ci.org/guide/linux/#linux-containers
+ cpu: 2
+ memory: 2
+ env:
+ TEST_FLAVOR: ext_svc
+ CTR_FQIN: ${FEDORA_CONTAINER_FQIN}
+ setup_script: &setup
+ - 'cd $GOSRC/$SCRIPT_BASE || exit 1'
+ - './setup_environment.sh'
+ main_script: &main
+ - 'cd $GOSRC/$SCRIPT_BASE || exit 1'
+ - './runner.sh'
+
+
+# Execute some quick checks to confirm this YAML file and all
+# automation-related shell scripts are sane.
+automation_task:
+ alias: 'automation'
+ name: "Check Automation"
+ container: *smallcontainer
+ env:
+ TEST_FLAVOR: automation
+ CTR_FQIN: ${FEDORA_CONTAINER_FQIN}
+ TEST_ENVIRON: container
+ setup_script: *setup
+ main_script: *main
+
+
+# This task use to be called 'gating', however that name is being
+# used downstream for release testing. Renamed this to avoid confusion.
+# All it does is run basic golang formatting and commit validation checks.
+smoke_task:
+ alias: 'smoke'
+ name: "Smoke Test"
+ container: &bigcontainer
+ image: ${CTR_FQIN}
+ # Leave some resources for smallcontainer
+ cpu: 6
+ memory: 22
+ env:
+ TEST_FLAVOR: 'smoke'
+ CTR_FQIN: "${FEDORA_CONTAINER_FQIN}"
+ TEST_ENVIRON: container
+ # This clone script is also used to initially populate gopath_cache (below)
+ clone_script: &full_clone |
+ cd /
+ rm -rf $CIRRUS_WORKING_DIR
+ mkdir -p $CIRRUS_WORKING_DIR
git clone --recursive --branch=$DEST_BRANCH https://x-access-token:${CIRRUS_REPO_CLONE_TOKEN}@github.com/${CIRRUS_REPO_FULL_NAME}.git $CIRRUS_WORKING_DIR
+ cd $CIRRUS_WORKING_DIR
git remote update origin
if [[ -n "$CIRRUS_PR" ]]; then # running for a PR
git fetch origin pull/$CIRRUS_PR/head:pull/$CIRRUS_PR
git checkout pull/$CIRRUS_PR
+ else
+ git reset --hard $CIRRUS_CHANGE_IN_REPO
fi
- git reset --hard $CIRRUS_CHANGE_IN_REPO
-
- # Don't bother going further if something is down
- networking_script: '${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/networking.sh'
-
- gate_script:
- # N/B: entrypoint.sh resets $GOSRC (same as make clean)
- - '/usr/local/bin/entrypoint.sh install.tools |& ${TIMESTAMP}'
- - '/usr/local/bin/entrypoint.sh validate |& ${TIMESTAMP}'
-
- # This task builds Podman with different buildtags to ensure the build does
- # not break. It also verifies all sub-commands have man pages.
- build_script:
- - '/usr/local/bin/entrypoint.sh podman |& ${TIMESTAMP}'
- - 'cd $GOSRC && ./hack/podman-commands.sh |& ${TIMESTAMP}'
- # N/B: need 'clean' so some committed files are re-generated.
- - '/usr/local/bin/entrypoint.sh clean podman-remote |& ${TIMESTAMP}'
- - '/usr/local/bin/entrypoint.sh clean podman xref_helpmsgs_manpages BUILDTAGS="exclude_graphdriver_devicemapper selinux seccomp" |& ${TIMESTAMP}'
- - '/usr/local/bin/entrypoint.sh clean BUILDTAGS="varlink" binaries |& ${TIMESTAMP}'
- - '/usr/local/bin/entrypoint.sh local-cross |& ${TIMESTAMP}'
-
- # Verify some aspects of ci/related scripts
- ci_script:
- - '${GOSRC}/${SCRIPT_BASE}/lib.sh.t |& ${TIMESTAMP}'
- - '${GOSRC}/${SCRIPT_BASE}/cirrus_yaml_test.py |& ${TIMESTAMP}'
-
- # Verify expected bash environment (-o pipefail)
- pipefail_enabledscript: 'if /bin/false | /bin/true; then echo "pipefail fault" && exit 72; fi'
-
-
-# Ensure these container images can build
-container_image_build_task:
- alias: 'container_image_build'
- name: "build gate image $DEST_BRANCH branch"
+ cd $CIRRUS_WORKING_DIR
+ make install.tools
+ setup_script: *setup
+ main_script: *main
+
+
+# N/B: This task is critical. It builds all binaries and release archives
+# for the project, using all primary OS platforms and versions. Assuming
+# the builds are successful, a cache is stored of the entire `$GOPATH`
+# contents. For all subsequent tasks, the _BUILD_CACHE_HANDLE value
+# is used as a key to reuse this cache, saving both time and money.
+# The only exceptions are tasks which only run inside a container, they
+# will not have access the cache and therefore must rely on cloning the
+# repository.
+build_task:
+ alias: 'build'
+ name: 'Build for $DISTRO_NV'
+ gce_instance: &standardvm
+ image_project: libpod-218412
+ zone: "us-central1-a"
+ cpu: 2
+ memory: "4Gb"
+ # Required to be 200gig, do not modify - has i/o performance impact
+ # according to gcloud CLI tool warning messages.
+ disk: 200
+ image_name: "${VM_IMAGE_NAME}" # from stdenvars
+ matrix: &platform_axis
+ # Ref: https://cirrus-ci.org/guide/writing-tasks/#matrix-modification
+ - env: &stdenvars
+ DISTRO_NV: ${FEDORA_NAME}
+ # Not used here, is used in other tasks
+ VM_IMAGE_NAME: ${FEDORA_CACHE_IMAGE_NAME}
+ CTR_FQIN: ${FEDORA_CONTAINER_FQIN}
+ # ID for re-use of build output
+ _BUILD_CACHE_HANDLE: ${FEDORA_NAME}-build-${CIRRUS_BUILD_ID}
+ - env:
+ DISTRO_NV: ${PRIOR_FEDORA_NAME}
+ VM_IMAGE_NAME: ${PRIOR_FEDORA_CACHE_IMAGE_NAME}
+ CTR_FQIN: ${PRIOR_FEDORA_CONTAINER_FQIN}
+ _BUILD_CACHE_HANDLE: ${PRIOR_FEDORA_NAME}-build-${CIRRUS_BUILD_ID}
+ - env:
+ DISTRO_NV: ${UBUNTU_NAME}
+ VM_IMAGE_NAME: ${UBUNTU_CACHE_IMAGE_NAME}
+ CTR_FQIN: ${UBUNTU_CONTAINER_FQIN}
+ _BUILD_CACHE_HANDLE: ${UBUNTU_NAME}-build-${CIRRUS_BUILD_ID}
+ - env:
+ DISTRO_NV: ${PRIOR_UBUNTU_NAME}
+ VM_IMAGE_NAME: ${PRIOR_UBUNTU_CACHE_IMAGE_NAME}
+ CTR_FQIN: ${PRIOR_UBUNTU_CONTAINER_FQIN}
+ _BUILD_CACHE_HANDLE: ${PRIOR_UBUNTU_NAME}-build-${CIRRUS_BUILD_ID}
+ env:
+ TEST_FLAVOR: build
+ # Ref: https://cirrus-ci.org/guide/writing-tasks/#cache-instruction
+ gopath_cache: &gopath_cache
+ folder: *gopath # Required hard-coded path, no variables.
+ fingerprint_script: echo "$_BUILD_CACHE_HANDLE"
+ # Cheat: Clone here when cache is empty, guaranteeing consistency.
+ populate_script: *full_clone
+ # A normal clone would invalidate useful cache
+ clone_script: &noop mkdir -p $CIRRUS_WORKING_DIR
+ setup_script: *setup
+ main_script: *main
+ always: &artifacts
+ gosrc_artifacts:
+ path: ./* # Grab everything in top-level $GOSRC
+ type: application/octet-stream
+ binary_artifacts:
+ path: ./bin/*
+ type: application/octet-stream
+ # Required for `contrib/cirrus/logformatter` to work properly
+ html_artifacts:
+ path: ./*.html
+ type: text/html
+
+# Confirm the result of building on at least one platform appears sane.
+# This confirms the binaries can be executed, checks --help vs docs, and
+# other essential post-build validation checks.
+validate_task:
+ name: "Validate $DISTRO_NV Build"
+ alias: validate
depends_on:
- - "gating"
-
- # Only run for PRs, quay.io will automatically build after branch-push
- only_if: $CIRRUS_BRANCH != $DEST_BRANCH
-
- container:
- dockerfile: contrib/gate/Dockerfile
-
- script: make install.remote
-
-
-# This task checks to make sure that we can still build an rpm from the
-# source code using contrib/rpm/podman.spec.in
-rpmbuild_task:
-
- only_if: >-
- $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*' &&
- $CIRRUS_BRANCH != $DEST_BRANCH
-
+ - ext_svc_check
+ - automation
+ - smoke
+ - build
+ # golangci-lint is a very, very hungry beast.
+ gce_instance: &bigvm
+ <<: *standardvm
+ cpu: 8
+ memory: "16Gb"
+ env:
+ <<: *stdenvars
+ TEST_FLAVOR: validate
+ gopath_cache: &ro_gopath_cache
+ <<: *gopath_cache
+ reupload_on_changes: false
+ clone_script: *noop
+ setup_script: *setup
+ main_script: *main
+ always: *artifacts
+
+# Exercise the "libpod" API with a small set of common
+# operations to ensure they are functional.
+bindings_task:
+ name: "Test Bindings"
+ alias: bindings
depends_on:
- - "gating"
+ - build
+ gce_instance: *standardvm
env:
- CIRRUS_WORKING_DIR: "/usr/src/libpod"
-
- container:
- image: quay.io/libpod/rpmbuild:$DEST_BRANCH
- cpu: 2
- memory: 4
-
- rpmbuild_script:
- - 'make -C ${CIRRUS_WORKING_DIR} -f ${CIRRUS_WORKING_DIR}/.copr/Makefile'
- - 'rpmbuild --rebuild ${CIRRUS_WORKING_DIR}/podman-*.src.rpm'
-
-# This task runs `make vendor` followed by ./hack/tree_status.sh to check
-# whether the git tree is clean. The reasoning for that is to make sure
-# that the vendor.conf, the code and the vendored packages in ./vendor are
-# in sync at all times.
+ <<: *stdenvars
+ TEST_FLAVOR: bindings
+ gopath_cache: *ro_gopath_cache
+ clone_script: *noop # Comes from cache
+ setup_script: *setup
+ main_script: *main
+ always: *artifacts
+
+
+# Build the "libpod" API documentation `swagger.yaml` for eventual
+# publishing along side the official podman documentation.
+swagger_task:
+ name: "Test Swagger"
+ alias: swagger
+ depends_on:
+ - build
+ container: *smallcontainer
+ env:
+ <<: *stdenvars
+ TEST_FLAVOR: swagger
+ TEST_ENVIRON: container
+ CTR_FQIN: ${FEDORA_CONTAINER_FQIN}
+ clone_script: *full_clone # build-cache not available to container tasks
+ setup_script: *setup
+ main_script: *main
+ always: *artifacts
+
+
+endpoint_task:
+ name: "Test Endpoint"
+ alias: endpoint
+ depends_on:
+ - build
+ container: *smallcontainer
+ env:
+ <<: *stdenvars
+ TEST_FLAVOR: endpoint
+ TEST_ENVIRON: container
+ CTR_FQIN: ${FEDORA_CONTAINER_FQIN}
+ clone_script: *full_clone # build-cache not available to container tasks
+ setup_script: *setup
+ main_script: *main
+ always: *artifacts
+
+
+# Check that all included go modules from other sources match
+# what is expected in `vendor/modules.txt` vs `go.mod`.
vendor_task:
-
- only_if: $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*'
-
+ name: "Test Vendoring"
+ alias: vendor
depends_on:
- - "gating"
-
+ - build
+ container: *smallcontainer
env:
- CIRRUS_WORKING_DIR: "/var/tmp/go/src/github.com/containers/podman"
- GOPATH: "/var/tmp/go"
- GOSRC: "$CIRRUS_WORKING_DIR"
-
- # Runs within Cirrus's "community cluster"
- container:
- image: docker.io/library/golang:1.13
- cpu: 4
- memory: 12
-
- timeout_in: 30m
-
- vendor_script:
- - 'cd ${CIRRUS_WORKING_DIR} && make vendor'
- - 'cd ${CIRRUS_WORKING_DIR} && ./hack/tree_status.sh'
-
-
-# This task runs `make varlink_api_generate` followed by ./hack/tree_status.sh to check
-# whether the git tree is clean.
-varlink_api_task:
-
- only_if: $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*'
-
+ <<: *stdenvars
+ TEST_FLAVOR: vendor
+ TEST_ENVIRON: container
+ CTR_FQIN: ${FEDORA_CONTAINER_FQIN}
+ clone_script: *full_clone # build-cache not available to container tasks
+ setup_script: *setup
+ main_script: *main
+ always: *artifacts
+
+
+# There are several other important variations of podman which
+# must always build successfully. Most of them are handled in
+# this task, though a few need dedicated tasks which follow.
+alt_build_task:
+ name: "$ALT_NAME"
+ alias: alt_build
depends_on:
- - "gating"
-
+ - build
env:
- CIRRUS_WORKING_DIR: "/usr/src/libpod"
- SRCPATH: "$CIRRUS_WORKING_DIR"
- EPOCH_TEST_COMMIT: "${CIRRUS_BASE_SHA}" # repo clone missing this data
- # Used by tree_status.sh
- SUGGESTION: 'remove API.md, then "make varlink_api_generate" and commit changes.'
-
- # Runs within Cirrus's "community cluster"
- container:
- # Note: Image has dual purpose, see contrib/gate/README.md
- image: "quay.io/libpod/gate:master"
- cpu: 4
- memory: 12
-
- timeout_in: 10m
-
- api_md_script:
- - '/usr/local/bin/entrypoint.sh BUILDTAGS="varlink" varlink_api_generate |& ${TIMESTAMP}'
- - 'cd ${GOSRC} && ./hack/tree_status.sh |& ${TIMESTAMP}'
-
-
-build_each_commit_task:
-
+ <<: *stdenvars
+ TEST_FLAVOR: "altbuild"
+ gce_instance: *standardvm
+ matrix:
+ - env:
+ ALT_NAME: 'Build Each Commit'
+ - env:
+ ALT_NAME: 'Windows Cross'
+ - env:
+ ALT_NAME: 'Build Without CGO'
+ - env:
+ ALT_NAME: 'Build varlink-API'
+ - env:
+ ALT_NAME: 'Test build RPM'
+ - env:
+ ALT_NAME: 'Build varlink-binaries'
+ setup_script: *setup
+ main_script: *main
+ always: *artifacts
+
+# Confirm building a statically-linked binary is successful
+static_alt_build_task:
+ name: "Static Build"
+ alias: static_alt_build
depends_on:
- - "gating"
- - "vendor"
- - "varlink_api"
-
- only_if: >-
- $CIRRUS_BRANCH != $DEST_BRANCH &&
- $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*'
-
- gce_instance:
- cpu: 8
- memory: "8Gb"
-
+ - build
+ # Community-maintained task, may fail on occasion. If so, uncomment
+ # the next line and file an issue with details about the failure.
+ # allow_failures: $CI == $CI
+ gce_instance: *bigvm
env:
- MOD_CONTAINERS_CONF: 'false'
-
- timeout_in: 30m
-
- setup_environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}'
- build_each_commit_script:
- # set -x by default, no need to spew contents of lib.sh
- - 'source $SCRIPT_BASE/lib.sh &> /dev/null'
- - 'git fetch --depth 50 origin $DEST_BRANCH |& ${TIMESTAMP}'
- - 'make build-all-new-commits GIT_BASE_BRANCH=origin/$DEST_BRANCH |& ${TIMESTAMP}'
-
+ <<: *stdenvars
+ TEST_FLAVOR: "altbuild"
+ # gce_instance variation prevents this being included in alt_build_task
+ ALT_NAME: 'Static build'
+ # Do not use 'latest', fixed-version tag for runtime stability.
+ CTR_FQIN: "docker.io/nixos/nix:2.3.6"
+ # This is critical, it helps to avoid a very lengthy process of
+ # statically building every dependency needed to build podman.
+ # Assuming the dependency and build description hasn't changed,
+ # this cache ensures only the static podman binary is built.
+ nix_cache:
+ folder: '/var/cache/nix'
+ fingerprint_script: cat nix/*
+ setup_script: *setup
+ main_script: *main
+ always: *artifacts
-build_without_cgo_task:
+# Confirm building the remote client, natively on a Mac OS-X VM.
+osx_alt_build_task:
+ name: "OSX Cross"
+ alias: osx_alt_build
depends_on:
- - "gating"
- - "vendor"
- - "varlink_api"
-
- only_if: >-
- $CIRRUS_BRANCH != $DEST_BRANCH &&
- $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*'
-
- gce_instance:
- cpu: 8
- memory: "8Gb"
-
+ - build
env:
- MOD_CONTAINERS_CONF: 'false'
-
- timeout_in: 30m
-
- setup_environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}'
- build_without_cgo_script:
- - 'source $SCRIPT_BASE/lib.sh'
- - 'make build-no-cgo'
-
-
-# Update metadata on VM images referenced by this repository state
-meta_task:
-
- container:
- image: "quay.io/libpod/imgts:${_BUILT_IMAGE_SUFFIX}"
- cpu: 1
- memory: 1
-
- env: &meta_env_vars
- # Space-separated list of images used by this repository state
- IMGNAMES: >-
- ${FEDORA_CACHE_IMAGE_NAME}
- ${PRIOR_FEDORA_CACHE_IMAGE_NAME}
- ${UBUNTU_CACHE_IMAGE_NAME}
- ${PRIOR_UBUNTU_CACHE_IMAGE_NAME}
- BUILDID: "${CIRRUS_BUILD_ID}"
- REPOREF: "${CIRRUS_CHANGE_IN_REPO}"
- GCPJSON: ENCRYPTED[3a198350077849c8df14b723c0f4c9fece9ebe6408d35982e7adf2105a33f8e0e166ed3ed614875a0887e1af2b8775f4]
- # needed for output-masking purposes
- GCPNAME: ENCRYPTED[2f9738ef295a706f66a13891b40e8eaa92a89e0e87faf8bed66c41eca72bf76cfd190a6f2d0e8444c631fdf15ed32ef6]
- GCPPROJECT: $GCP_PROJECT_ID
-
- timeout_in: 10m
-
- # Cirrus-CI ignores entrypoint defined in image
- script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/update_meta.sh |& ${TIMESTAMP}'
-
+ <<: *stdenvars
+ # OSX platform variation prevents this being included in alt_build_task
+ TEST_FLAVOR: "altbuild"
+ ALT_NAME: 'OSX Cross'
+ osx_instance:
+ image: 'catalina-base'
+ script:
+ - brew install go
+ - brew install go-md2man
+ - make podman-remote-darwin
+ - make install-podman-remote-darwin-docs
+ always: *artifacts
-# This task does the unit and integration testing for every platform
-testing_task:
- alias: "testing"
+# This task is a stub: In the future it will be used to verify
+# podman is compatible with the docker python-module.
+docker-py_test_task:
+ name: Docker-py Compat.
+ alias: docker-py_test
depends_on:
- - "gating"
- - "vendor"
- - "varlink_api"
- - "build_each_commit"
- - "build_without_cgo"
- - "container_image_build"
-
- # Only test build cache-images, if that's what's requested
- only_if: $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*'
-
- matrix:
- - name: "test ${FEDORA_NAME}"
- gce_instance:
- image_name: "${FEDORA_CACHE_IMAGE_NAME}"
- - name: "test ${PRIOR_FEDORA_NAME}"
- gce_instance:
- image_name: "${PRIOR_FEDORA_CACHE_IMAGE_NAME}"
- - name: "test ${UBUNTU_NAME}"
- gce_instance:
- image_name: "${UBUNTU_CACHE_IMAGE_NAME}"
- - name: "test ${PRIOR_UBUNTU_NAME}"
- gce_instance:
- image_name: "${PRIOR_UBUNTU_CACHE_IMAGE_NAME}"
-
+ - build
+ container: *smallcontainer
+ env:
+ <<: *stdenvars
+ TEST_FLAVOR: docker-py
+ TEST_ENVIRON: container
+ gopath_cache: *ro_gopath_cache
+ clone_script: *full_clone # build-cache not available to container tasks
+ setup_script: *setup
+ main_script: *main
+ always: *artifacts
+
+# Does exactly what it says, execute the podman unit-tests on all primary
+# platforms and release versions.
+unit_test_task:
+ name: "Unit tests on $DISTRO_NV"
+ alias: unit_test
+ depends_on:
+ - validate
+ matrix: *platform_axis
+ gce_instance: *standardvm
env:
- ADD_SECOND_PARTITION: 'true'
- matrix:
- - RCLI: 'true'
- - RCLI: 'false'
-
- networking_script: '${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/networking.sh'
- setup_environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}'
- unit_test_script: '$SCRIPT_BASE/unit_test.sh |& ${TIMESTAMP}'
- integration_test_script: '$SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP} | ${LOGFORMAT} integration_test'
- system_test_script: '$SCRIPT_BASE/system_test.sh |& ${TIMESTAMP} | ${LOGFORMAT} system_test'
- apiv2_test_script: '$SCRIPT_BASE/apiv2_test.sh |& ${TIMESTAMP} | ${LOGFORMAT} apiv2_test'
-
- # When examining a particular run, provide convenient access to release files.
- tar_artifacts:
- path: "*.tar.gz"
- type: "application/x-tar"
-
- always: &standardlogs
+ TEST_FLAVOR: unit
+ clone_script: *noop # Comes from cache
+ gopath_cache: *ro_gopath_cache
+ setup_script: *setup
+ main_script: *main
+ always: *artifacts
+
+# Execute the podman integration tests on all primary platforms and release
+# versions, as root, without involving the podman-remote client.
+local_integration_test_task: &local_integration_test_task
+ # Integration-test task name convention:
+ # <int.|sys.> <podman|remote> <Distro NV> <root|rootless>
+ name: &std_name_fmt "$TEST_FLAVOR $PODBIN_NAME $DISTRO_NV $PRIV_NAME $TEST_ENVIRON"
+ alias: local_integration_test
+ depends_on:
+ - unit_test
+ matrix: *platform_axis
+ gce_instance: *standardvm
+ timeout_in: 90m
+ env:
+ TEST_FLAVOR: int
+ clone_script: *noop # Comes from cache
+ gopath_cache: *ro_gopath_cache
+ setup_script: *setup
+ main_script: *main
+ always: &logs_artifacts
+ <<: *artifacts
package_versions_script: '$SCRIPT_BASE/logcollector.sh packages'
ginkgo_node_logs_script: '$SCRIPT_BASE/logcollector.sh ginkgo'
df_script: '$SCRIPT_BASE/logcollector.sh df'
audit_log_script: '$SCRIPT_BASE/logcollector.sh audit'
journal_script: '$SCRIPT_BASE/logcollector.sh journal'
- varlink_script: '$SCRIPT_BASE/logcollector.sh varlink'
podman_system_info_script: '$SCRIPT_BASE/logcollector.sh podman'
- html_artifacts:
- path: "*.log.html"
- type: "text/html"
-
-# This task executes tests under unique environments/conditions
-special_testing_rootless_task:
-
- depends_on:
- - "gating"
- - "varlink_api"
- - "vendor"
- - "build_each_commit"
- - "build_without_cgo"
-
- only_if: $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*'
+# Nearly identical to `local_integration_test` except all operations
+# are performed through the podman-remote client vs a podman "server"
+# running on the same host.
+remote_integration_test_task:
+ <<: *local_integration_test_task
+ alias: remote_integration_test
env:
- ADD_SECOND_PARTITION: 'true'
- SPECIALMODE: 'rootless' # See docs
- matrix:
- - RCLI: 'true'
- - RCLI: 'false'
-
- networking_script: '${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/networking.sh'
- setup_environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}'
- integration_test_script: '$SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP} | ${LOGFORMAT} integration_test'
- system_test_script: '$SCRIPT_BASE/system_test.sh |& ${TIMESTAMP} | ${LOGFORMAT} system_test'
- apiv2_test_script: '$SCRIPT_BASE/apiv2_test.sh |& ${TIMESTAMP} | ${LOGFORMAT} apiv2_test'
-
- always:
- <<: *standardlogs
+ TEST_FLAVOR: int
+ PODBIN_NAME: remote
-special_testing_in_podman_task:
-
- alias: "special_testing_in_podman"
+# Run the complete set of integration tests from inside a container.
+# This verifies all/most operations function with "podman-in-podman".
+container_integration_test_task:
+ name: *std_name_fmt
+ alias: container_integration_test
depends_on:
- - "gating"
- - "varlink_api"
- - "vendor"
- - "build_each_commit"
- - "build_without_cgo"
-
- only_if: $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*'
-
- matrix:
- - name: "in-podman ${PRIOR_FEDORA_NAME}"
- gce_instance:
- image_name: "${PRIOR_FEDORA_CACHE_IMAGE_NAME}"
- - name: "in-podman ${FEDORA_NAME}"
- gce_instance:
- image_name: "${FEDORA_CACHE_IMAGE_NAME}"
-
+ - unit_test
+ matrix: &fedora_vm_axis
+ - env:
+ DISTRO_NV: ${FEDORA_NAME}
+ _BUILD_CACHE_HANDLE: ${FEDORA_NAME}-build-${CIRRUS_BUILD_ID}
+ VM_IMAGE_NAME: ${FEDORA_CACHE_IMAGE_NAME}
+ CTR_FQIN: ${FEDORA_CONTAINER_FQIN}
+ - env:
+ DISTRO_NV: ${PRIOR_FEDORA_NAME}
+ _BUILD_CACHE_HANDLE: ${PRIOR_FEDORA_NAME}-build-${CIRRUS_BUILD_ID}
+ VM_IMAGE_NAME: ${PRIOR_FEDORA_CACHE_IMAGE_NAME}
+ CTR_FQIN: ${PRIOR_FEDORA_CONTAINER_FQIN}
+ gce_instance: *standardvm
+ timeout_in: 90m
env:
- ADD_SECOND_PARTITION: 'true'
- MOD_CONTAINERS_CONF: 'false' # Use existing/native setup
- SPECIALMODE: 'in_podman' # See docs
-
- networking_script: '${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/networking.sh'
- setup_environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}'
- integration_test_script: '$SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP} | ${LOGFORMAT} integration_test'
-
- always:
- <<: *standardlogs
-
-
-special_testing_cross_task:
-
- alias: "special_testing_cross"
+ TEST_FLAVOR: int
+ TEST_ENVIRON: container
+ clone_script: *noop # Comes from cache
+ gopath_cache: *ro_gopath_cache
+ setup_script: *setup
+ main_script: *main
+ always: *logs_artifacts
+
+# Execute most integration tests as a regular (non-root) user.
+rootless_integration_test_task:
+ name: *std_name_fmt
+ alias: rootless_integration_test
depends_on:
- - "gating"
- - "varlink_api"
- - "vendor"
-
- only_if: $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*'
-
- matrix:
- - name: 'cross-platform: windows'
- env:
- CROSS_PLATFORM: 'windows'
- - name: 'cross-platform: darwin'
- env:
- CROSS_PLATFORM: 'darwin'
-
- timeout_in: 20m
-
- networking_script: '${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/networking.sh'
- setup_environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}'
- build_release_script: '$SCRIPT_BASE/build_release.sh |& ${TIMESTAMP}'
-
- # When examining a particular run, provide convenient access to release files.
- zip_artifacts:
- path: "*.zip"
- type: "application/zip"
-
- msi_artifacts:
- path: "*.msi"
- type: "application/octet-stream"
-
-
-special_testing_bindings_task:
-
+ - unit_test
+ matrix: *fedora_vm_axis
+ gce_instance: *standardvm
+ timeout_in: 90m
+ env:
+ TEST_FLAVOR: int
+ PRIV_NAME: rootless
+ clone_script: *noop # Comes from cache
+ gopath_cache: *ro_gopath_cache
+ setup_script: *setup
+ main_script: *main
+ always: *logs_artifacts
+
+
+# Always run subsequent to integration tests. While parallelism is lost
+# with runtime, debugging system-test failures can be more challenging
+# for some golang developers. Otherwise the following tasks run across
+# the same matrix as the integration-tests (above).
+local_system_test_task: &local_system_test_task
+ name: *std_name_fmt
+ alias: local_system_test
depends_on:
- - "gating"
- - "varlink_api"
- - "vendor"
-
- only_if: $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*'
-
+ - local_integration_test
+ matrix: *platform_axis
+ gce_instance: *standardvm
env:
- SPECIALMODE: 'bindings' # See docs
- ADD_SECOND_PARTITION: 'true' # More root fs space is required
-
- timeout_in: 40m
-
- setup_environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}'
- integration_test_script: '$SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP} | ${LOGFORMAT} integration_test'
-
- always:
- <<: *standardlogs
+ TEST_FLAVOR: sys
+ clone_script: *noop # Comes from cache
+ gopath_cache: *ro_gopath_cache
+ setup_script: *setup
+ main_script: *main
+ always: *logs_artifacts
-special_testing_endpoint_task:
-
+remote_system_test_task:
+ <<: *local_system_test_task
+ alias: remote_system_test
depends_on:
- - "gating"
- - "varlink_api"
- - "vendor"
-
- only_if: $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*'
-
+ - remote_integration_test
env:
- SPECIALMODE: 'endpoint' # See docs
-
- timeout_in: 20m
-
- setup_environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}'
- integration_test_script: '$SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP} | ${LOGFORMAT} integration_test'
- always:
- <<: *standardlogs
-
+ TEST_FLAVOR: sys
+ PODBIN_NAME: remote
-docs_task:
+rootless_system_test_task:
+ name: *std_name_fmt
+ alias: rootless_system_test
depends_on:
- - "gating"
+ - rootless_integration_test
+ matrix: *fedora_vm_axis
+ gce_instance: *standardvm
env:
- RELEASE_GCPJSON: ENCRYPTED[927dc01e755eaddb4242b0845cf86c9098d1e3dffac38c70aefb1487fd8b4fe6dd6ae627b3bffafaba70e2c63172664e]
- RELEASE_GCPNAME: ENCRYPTED[c145e9c16b6fb88d476944a454bf4c1ccc84bb4ecaca73bdd28bdacef0dfa7959ebc8171a27b2e4064d66093b2cdba49]
- RELEASE_GCPROJECT: $GCP_PROJECT_ID
-
- script:
- - "$SCRIPT_BASE/build_swagger.sh |& ${TIMESTAMP}"
- - "$SCRIPT_BASE/upload_release_archive.sh |& ${TIMESTAMP}"
+ TEST_FLAVOR: sys
+ PRIV_NAME: rootless
+ PODBIN_NAME: remote
+ clone_script: *noop # Comes from cache
+ gopath_cache: *ro_gopath_cache
+ setup_script: *setup
+ main_script: *main
+ always: *logs_artifacts
+
+# This task is critical. It updates the "last-used by" timestamp stored
+# in metadata for all VM images. This mechanism functions in tandem with
+# an out-of-band pruning operation to remove disused VM images.
+meta_task:
+ name: "VM img. keepalive"
+ alias: meta
+ container:
+ cpu: 2
+ memory: 2
+ image: quay.io/libpod/imgts:$IMAGE_SUFFIX
+ env:
+ # Space-separated list of images used by this repository state
+ IMGNAMES: >-
+ ${FEDORA_CACHE_IMAGE_NAME}
+ ${PRIOR_FEDORA_CACHE_IMAGE_NAME}
+ ${UBUNTU_CACHE_IMAGE_NAME}
+ ${PRIOR_UBUNTU_CACHE_IMAGE_NAME}
+ BUILDID: "${CIRRUS_BUILD_ID}"
+ REPOREF: "${CIRRUS_REPO_NAME}"
+ GCPJSON: ENCRYPTED[3a198350077849c8df14b723c0f4c9fece9ebe6408d35982e7adf2105a33f8e0e166ed3ed614875a0887e1af2b8775f4]
+ GCPNAME: ENCRYPTED[2f9738ef295a706f66a13891b40e8eaa92a89e0e87faf8bed66c41eca72bf76cfd190a6f2d0e8444c631fdf15ed32ef6]
+ GCPPROJECT: libpod-218412
+ clone_script: *noop
+ script: /usr/local/bin/entrypoint.sh
-# Post message to IRC if everything passed PR testing
+# Status aggregator for all tests. This task simply ensures a defined
+# set of tasks all passed, and allows confirming that based on the status
+# of this task.
success_task:
-
- # This task is a required-pass in github settings,
- # it blocks PRs from merging if a depends_on task fails
- only_if: $CIRRUS_BRANCH != $DEST_BRANCH
-
- # ignores any dependent task conditions
+ name: "Total Success"
+ alias: success
+ # N/B: ALL tasks must be listed here, minus their '_task' suffix.
depends_on:
- - "gating"
- - "vendor"
- - "varlink_api"
- - "build_each_commit"
- - "build_without_cgo"
- - "container_image_build"
- - "meta"
- - "testing"
- - "rpmbuild"
- - "special_testing_rootless"
- - "special_testing_in_podman"
- - "special_testing_cross"
- - "special_testing_endpoint"
- - "special_testing_bindings"
- - "docs"
- - "static_build"
- - "darwin_build"
-
+ - ext_svc_check
+ - automation
+ - smoke
+ - build
+ - validate
+ - bindings
+ - endpoint
+ - swagger
+ - vendor
+ - alt_build
+ - static_alt_build
+ - osx_alt_build
+ - docker-py_test
+ - unit_test
+ - local_integration_test
+ - remote_integration_test
+ - rootless_integration_test
+ - container_integration_test
+ - local_system_test
+ - remote_system_test
+ - rootless_system_test
+ - meta
+ container: *smallcontainer
env:
- CIRRUS_WORKING_DIR: "/usr/src/libpod"
- SRCPATH: "$CIRRUS_WORKING_DIR"
- EPOCH_TEST_COMMIT: "${CIRRUS_BASE_SHA}" # repo clone missing this data
-
- container:
- # Note: Image has dual purpose, see contrib/gate/README.md
- image: "quay.io/libpod/gate:master"
- cpu: 1
- memory: 1
-
- success_script: /bin/true
-
-# Build the static binary
-static_build_task:
+ CTR_FQIN: ${FEDORA_CONTAINER_FQIN}
+ TEST_ENVIRON: container
+ clone_script: *noop
+ script: /bin/true
+
+
+# When a new tag is pushed, confirm that the code and commits
+# meet criteria for an official release.
+release_task:
+ name: "Verify Release"
+ alias: release
+ only_if: $CIRRUS_TAG != ''
depends_on:
- - "gating"
-
- gce_instance:
- image_name: "${FEDORA_CACHE_IMAGE_NAME}"
- cpu: 8
- memory: 12
- disk: 200
-
- init_script: |
- set -ex
- setenforce 0
- growpart /dev/sda 1 || true
- resize2fs /dev/sda1 || true
- yum -y install podman
-
- nix_cache:
- folder: '.cache'
- fingerprint_script: |
- echo "nix-v1-$(sha1sum nix/nixpkgs.json | head -c 40)"
-
- build_script: |
- set -ex
- mkdir -p .cache
- mv .cache /nix
- if [[ -z $(ls -A /nix) ]]; then podman run --rm --privileged -ti -v /:/mnt nixos/nix cp -rfT /nix /mnt/nix; fi
- podman run --rm --privileged -ti -v /nix:/nix -v ${PWD}:${PWD} -w ${PWD} nixos/nix nix --print-build-logs --option cores 8 --option max-jobs 8 build --file nix/
-
- binaries_artifacts:
- path: "result/bin/podman"
-
- save_cache_script: |
- mv /nix .cache
- chown -Rf $(whoami) .cache
-
-
-darwin_build_task:
+ - success
+ gce_instance: *standardvm
+ env:
+ <<: *stdenvars
+ TEST_FLAVOR: release
+ gopath_cache: *ro_gopath_cache
+ clone_script: *noop # Comes from cache
+ setup_script: *setup
+ main_script: *main
+ always: *artifacts
+
+# When preparing to release a new version, this task may be manually
+# activated at the PR stage to verify the code is in a proper state.
+# Note: This cannot use a YAML alias on 'release_task' as of this
+# comment, it is incompatible with 'trigger_type: manual'
+release_test_task:
+ name: "Optional Release Test"
+ alias: release_test
+ only_if: $CIRRUS_PR != ''
+ trigger_type: manual
depends_on:
- - "gating"
- osx_instance:
- image: catalina-base
- setup-script:
- - brew install go
- - brew install go-md2man
- build-script:
- - make podman-remote-darwin
- - make install-podman-remote-darwin-docs
- binaries_artifacts:
- path: "bin/podman-remote-darwin"
+ - success
+ gce_instance: *standardvm
+ env:
+ <<: *stdenvars
+ TEST_FLAVOR: release
+ gopath_cache: *ro_gopath_cache
+ clone_script: *noop # Comes from cache
+ setup_script: *setup
+ main_script: *main
+ always: *artifacts
diff --git a/.golangci.yml b/.golangci.yml
index 4d5ff549c..b3466d8f9 100644
--- a/.golangci.yml
+++ b/.golangci.yml
@@ -11,20 +11,32 @@ run:
- pkg/varlink
- pkg/varlinkapi
- docs/varlink
+ - vendor
skip-files:
- iopodman.go
- swagger.go
+ modules-download-mode: readonly
linters:
enable-all: true
disable:
+ # All these break for one reason or another
- dupl
- funlen
- gochecknoglobals
- gochecknoinits
- goconst
- gocyclo
- - gosec
+ - golint
- lll
+ - structcheck
+ - typecheck
+ - unconvert
+ - varcheck
+ - deadcode
+ - depguard
+ - errcheck
+ - gocritic
+ - gosec
- maligned
linters-settings:
errcheck:
diff --git a/Makefile b/Makefile
index 99a32eb13..89bf8707e 100644
--- a/Makefile
+++ b/Makefile
@@ -169,7 +169,13 @@ golangci-lint: .gopathok varlink_generate .install.golangci-lint
.PHONY: gofmt
gofmt: ## Verify the source code gofmt
- find . -name '*.go' ! -path './vendor/*' -exec gofmt -s -w {} \+
+ find . -name '*.go' -type f \
+ -not \( \
+ -name '.golangci.yml' -o \
+ -name 'Makefile' -o \
+ -path './vendor/*' -prune -o \
+ -path './contrib/*' -prune \
+ \) -exec gofmt -d -e -s -w {} \+
git diff --exit-code
.PHONY: test/checkseccomp/checkseccomp
diff --git a/contrib/cirrus/README.md b/contrib/cirrus/README.md
deleted file mode 100644
index 4056edb6a..000000000
--- a/contrib/cirrus/README.md
+++ /dev/null
@@ -1,150 +0,0 @@
-![PODMAN logo](../../logo/podman-logo-source.svg)
-
-# Cirrus-CI
-
-Similar to other integrated github CI/CD services, Cirrus utilizes a simple
-YAML-based configuration/description file: ``.cirrus.yml``. Ref: https://cirrus-ci.org/
-
-
-## Workflow
-
-All tasks execute in parallel, unless there are conditions or dependencies
-which alter this behavior. Within each task, each script executes in sequence,
-so long as any previous script exited successfully. The overall state of each
-task (pass or fail) is set based on the exit status of the last script to execute.
-
-### ``gating`` Task
-
-***N/B: Steps below are performed by automation***
-
-1. Launch a purpose-built container in Cirrus's community cluster.
- For container image details, please see
- [the contributors guide](https://github.com/containers/podman/blob/master/CONTRIBUTING.md#go-format-and-lint).
-
-3. ``validate``: Perform standard `make validate` source verification,
- Should run for less than a minute or two.
-
-4. ``lint``: Execute regular `make lint` to check for any code cruft.
- Should also run for less than a few minutes.
-
-5. ``vendor``: runs `make vendor-in-container` followed by `./hack/tree_status.sh` to check
- whether the git tree is clean. The reasoning for that is to make sure that
- the vendor.conf, the code and the vendored packages in ./vendor are in sync
- at all times.
-
-### ``meta`` Task
-
-***N/B: Steps below are performed by automation***
-
-1. Launch a container built from definition in ``./contrib/imgts``.
-
-2. Update VM Image metadata to help track usage across all automation.
-
-4. Always exits successfully unless there's a major problem.
-
-
-### ``testing`` Task
-
-***N/B: Steps below are performed by automation***
-
-1. After `gating` passes, spin up one VM per
- `matrix: image_name` item. Once accessible, ``ssh``
- into each VM as the `root` user.
-
-2. ``setup_environment.sh``: Configure root's `.bash_profile`
- for all subsequent scripts (each run in a new shell). Any
- distribution-specific environment variables are also defined
- here. For example, setting tags/flags to use compiling.
-
-5. ``integration_test.sh``: Execute integration-testing. This is
- much more involved, and relies on access to external
- resources like container images and code from other repositories.
- Total execution time is capped at 2-hours (includes all the above)
- but this script normally completes in less than an hour.
-
-
-### ``special_testing_cross`` Task
-
-Confirm that cross-compile of podman-remote functions for both `windows`
-and `darwin` targets.
-
-
-### ``special_testing_cgroupv2`` Task
-
-Use the latest Fedora release with the required kernel options pre-set for
-exercising cgroups v2 with Podman integration tests. Also depends on
-having `SPECIALMODE` set to 'cgroupv2`
-
-
-### `docs` Task
-
-Builds swagger API documentation YAML and uploads to google storage (an online
-service for storing unstructured data) for both
-PR's (for testing the process) and the master branch. For PR's
-the YAML is uploaded into a [dedicated short-pruning cycle
-bucket.](https://storage.googleapis.com/libpod-pr-releases/) for testing purposes
-only. For the master branch, a [separate bucket is
-used](https://storage.googleapis.com/libpod-master-releases) and provides the
-content rendered on [the API Reference page](https://docs.podman.io/en/latest/_static/api.html)
-
-The online API reference is presented by javascript to the client. To prevent hijacking
-of the client by malicious data, the [javascript utilises CORS](https://cloud.google.com/storage/docs/cross-origin).
-This CORS metadata is served by `https://storage.googleapis.com` when configured correctly.
-It will appear in [the request and response headers from the
-client](https://cloud.google.com/storage/docs/configuring-cors#troubleshooting) when accessing
-the API reference page.
-
-However, when the CORS metadata is missing or incorrectly configured, clients will receive an
-error-message similar to:
-
-![Javascript Stack Trace Image](swagger_stack_trace.png)
-
-For documentation built by Read The Docs from the master branch, CORS metadata is
-set on the `libpod-master-releases` storage bucket. Viewing or setting the CORS
-metadata on the bucket requires having locally [installed and
-configured the google-cloud SDK](https://cloud.google.com/sdk/docs). It also requires having
-admin access to the google-storage bucket. Contact a project owner for help if you are
-unsure of your permissions or need help resolving an error similar to the picture above.
-
-Assuming the SDK is installed, and you have the required admin access, the following command
-will display the current CORS metadata:
-
-```
-gsutil cors get gs://libpod-master-releases
-```
-
-To function properly (allow client "trust" of content from `storage.googleapis.com`) the followiing
-metadata JSON should be used. Following the JSON, is an example of the command used to set this
-metadata on the libpod-master-releases bucket. For additional information about configuring CORS
-please refer to [the google-storage documentation](https://cloud.google.com/storage/docs/configuring-cors).
-
-```JSON
-[
- {
- "origin": ["http://docs.podman.io", "https://docs.podman.io"],
- "responseHeader": ["Content-Type"],
- "method": ["GET"],
- "maxAgeSeconds": 600
- }
-]
-```
-
-```
-gsutil cors set /path/to/file.json gs://libpod-master-releases
-```
-
-***Note:*** The CORS metadata does _NOT_ change after the `docs` task uploads a new swagger YAML
-file. Therefore, if it is not functioning or misconfigured, a person must have altered it or
-changes were made to the referring site (e.g. `docs.podman.io`).
-
-## `$SPECIALMODE`
-
-Some tasks alter their behavior based on this value. A summary of supported
-values follows:
-
-* `none`: Operate as normal, this is the default value if unspecified.
-* `rootless`: Causes a random, ordinary user account to be created
- and utilized for testing.
-* `in_podman`: Causes testing to occur within a container executed by
-* `windows`: See **darwin**
-* `darwin`: Signals the ``special_testing_cross`` task to cross-compile the remote client.
diff --git a/contrib/cirrus/add_second_partition.sh b/contrib/cirrus/add_second_partition.sh
index d0407be86..322dd2512 100644
--- a/contrib/cirrus/add_second_partition.sh
+++ b/contrib/cirrus/add_second_partition.sh
@@ -3,6 +3,11 @@
# N/B: This script could mega f*!@up your disks if run by mistake.
# it is left without the execute-bit on purpose!
+set -eo pipefail
+
+# shellcheck source=./lib.sh
+source $(dirname $0)/lib.sh
+
# $SLASH_DEVICE is the disk device to be f*xtuP
SLASH_DEVICE="/dev/sda" # Always the case on GCP
@@ -11,21 +16,18 @@ SLASH_DEVICE="/dev/sda" # Always the case on GCP
NEW_PART_START="50%"
NEW_PART_END="100%"
-set -eo pipefail
-
-source $(dirname $0)/lib.sh
if [[ ! -r "/root" ]] || [[ -r "/root/second_partition_ready" ]]
then
- echo "Warning: Ignoring attempted execution of $(basename $0)"
+ warn "Ignoring attempted execution of $(basename $0)"
exit 0
fi
-[[ -n "type -P parted" ]] || \
- die 2 "The parted command is required."
+[[ -x "$(type -P parted)" ]] || \
+ die "The parted command is required."
[[ ! -b ${SLASH_DEVICE}2 ]] || \
- die 5 "Found unexpected block device ${SLASH_DEVICE}2"
+ die "Found unexpected block device ${SLASH_DEVICE}2"
PPRINTCMD="parted --script ${SLASH_DEVICE} print"
FINDMNTCMD="findmnt --source=${SLASH_DEVICE}1 --mountpoint=/ --canonicalize --evaluate --first-only --noheadings"
@@ -34,28 +36,28 @@ trap "rm -f $TMPF" EXIT
if $FINDMNTCMD | tee $TMPF | egrep -q "^/\s+${SLASH_DEVICE}1"
then
- echo "Repartitioning original partition table:"
+ msg "Repartitioning original partition table:"
$PPRINTCMD
else
- die 6 "Unexpected output from '$FINDMNTCMD': $(<$TMPF)"
+ die "Unexpected output from '$FINDMNTCMD': $(<$TMPF)"
fi
echo "Adding partition offset within unpartitioned space."
parted --script --align optimal /dev/sda unit % mkpart primary "" "" "$NEW_PART_START" "$NEW_PART_END"
-echo "New partition table:"
+msg "New partition table:"
$PPRINTCMD
-echo "Growing ${SLASH_DEVICE}1 meet start of ${SLASH_DEVICE}2"
+msg "Growing ${SLASH_DEVICE}1 meet start of ${SLASH_DEVICE}2"
growpart ${SLASH_DEVICE} 1
FSTYPE=$(findmnt --first-only --noheadings --output FSTYPE ${SLASH_DEVICE}1)
echo "Expanding $FSTYPE filesystem on ${SLASH_DEVICE}1"
case $FSTYPE in
ext*) resize2fs ${SLASH_DEVICE}1 ;;
- *) die 11 "Script $(basename $0) doesn't know how to resize a $FSTYPE filesystem." ;;
+ *) die "Script $(basename $0) doesn't know how to resize a $FSTYPE filesystem." ;;
esac
# Must happen last - signals completion to other tooling
-echo "Recording newly available disk partition device into /root/second_partition_ready"
+msg "Recording newly available disk partition device into /root/second_partition_ready"
echo "${SLASH_DEVICE}2" > /root/second_partition_ready
diff --git a/contrib/cirrus/apiv2_test.sh b/contrib/cirrus/apiv2_test.sh
deleted file mode 100755
index dbee078b6..000000000
--- a/contrib/cirrus/apiv2_test.sh
+++ /dev/null
@@ -1,65 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-source $(dirname $0)/lib.sh
-
-req_env_var GOSRC SCRIPT_BASE OS_RELEASE_ID OS_RELEASE_VER CONTAINER_RUNTIME VARLINK_LOG
-
-LOCAL_OR_REMOTE=local
-if [[ "$RCLI" = "true" ]]; then
- LOCAL_OR_REMOTE=remote
-fi
-
-# Our name must be of the form xxxx_test or xxxx_test.sh, where xxxx is
-# the test suite to run; currently (2019-05) the only option is 'integration'
-# but pr2947 intends to add 'system'.
-TESTSUITE=$(expr $(basename $0) : '\(.*\)_test')
-if [[ -z $TESTSUITE ]]; then
- die 1 "Script name is not of the form xxxx_test.sh"
-fi
-
-cd "$GOSRC"
-
-case "$SPECIALMODE" in
- in_podman)
- ${CONTAINER_RUNTIME} run --rm --privileged --net=host --cgroupns=host \
- -v $GOSRC:$GOSRC:Z \
- --workdir $GOSRC \
- -e "CGROUP_MANAGER=cgroupfs" \
- -e "STORAGE_OPTIONS=--storage-driver=vfs" \
- -e "CRIO_ROOT=$GOSRC" \
- -e "PODMAN_BINARY=/usr/bin/podman" \
- -e "CONMON_BINARY=/usr/libexec/podman/conmon" \
- -e "DIST=$OS_RELEASE_ID" \
- -e "CONTAINER_RUNTIME=$CONTAINER_RUNTIME" \
- $IN_PODMAN_IMAGE bash $GOSRC/$SCRIPT_BASE/container_test.sh -b -i -t
- ;;
- rootless)
- req_env_var ROOTLESS_USER
- ssh $ROOTLESS_USER@localhost \
- -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \
- -o CheckHostIP=no $GOSRC/$SCRIPT_BASE/rootless_test.sh ${TESTSUITE} ${LOCAL_OR_REMOTE}
- ;;
- endpoint)
- make
- make install PREFIX=/usr ETCDIR=/etc
- make test-binaries
- make endpoint
- ;;
- bindings)
- make
- make install PREFIX=/usr ETCDIR=/etc
- export PATH=$PATH:`pwd`/hack
- cd pkg/bindings/test && ginkgo -trace -noColor -debug -r
- ;;
- none)
- make
- make install PREFIX=/usr ETCDIR=/etc
- make test-binaries
- make .install.bats
- make ${LOCAL_OR_REMOTE}${TESTSUITE} PODMAN_SERVER_LOG=$PODMAN_SERVER_LOG
- ;;
- *)
- die 110 "Unsupported \$SPECIALMODE: $SPECIALMODE"
-esac
diff --git a/contrib/cirrus/build_release.sh b/contrib/cirrus/build_release.sh
deleted file mode 100755
index 46fe9781f..000000000
--- a/contrib/cirrus/build_release.sh
+++ /dev/null
@@ -1,34 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-source $(dirname $0)/lib.sh
-
-req_env_var RCLI OS_RELEASE_ID GOSRC
-
-cd $GOSRC
-
-if [[ "$RCLI" == "true" ]] && [[ -z "$CROSS_PLATFORM" ]]
-then
- CROSS_PLATFORM=linux
-fi
-
-if [[ -n "$CROSS_PLATFORM" ]]
-then
- # Will fail if $CROSS_PLATFORM is unsupported cross-compile $GOOS value
- make podman-remote-${CROSS_PLATFORM}-release
-
- echo "Compiling podman-remote release archive for ${CROSS_PLATFORM}"
- if [[ "$CROSS_PLATFORM" == "windows" ]]
- then
- # TODO: Remove next line, part of VM images next time they're built.
- dnf install -y libmsi1 msitools pandoc
- make podman.msi
- fi
-else
- echo "Compiling release archive for $OS_RELEASE_ID"
- make podman-release
-fi
-
-echo "Preserving build details for later use."
-mv -v release.txt actual_release.txt # Another 'make' during testing could overwrite it
diff --git a/contrib/cirrus/build_swagger.sh b/contrib/cirrus/build_swagger.sh
deleted file mode 100755
index eb9288dcd..000000000
--- a/contrib/cirrus/build_swagger.sh
+++ /dev/null
@@ -1,17 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-source $(dirname $0)/lib.sh
-
-# Building this is a PITA, just grab binary for use in automation
-# Ref: https://goswagger.io/install.html#static-binary
-download_url=$(curl -s https://api.github.com/repos/go-swagger/go-swagger/releases/latest | \
- jq -r '.assets[] | select(.name | contains("'"$(uname | tr '[:upper:]' '[:lower:]')"'_amd64")) | .browser_download_url')
-curl -o /usr/local/bin/swagger -L'#' "$download_url"
-chmod +x /usr/local/bin/swagger
-
-cd $GOSRC
-make swagger
-echo "Preserving build details for later use."
-mv -v release.txt actual_release.txt # Another 'make' during testing could overwrite it
diff --git a/contrib/cirrus/cirrus_yaml_test.py b/contrib/cirrus/cirrus_yaml_test.py
index c2ff8e69e..39de0c7b7 100755
--- a/contrib/cirrus/cirrus_yaml_test.py
+++ b/contrib/cirrus/cirrus_yaml_test.py
@@ -26,6 +26,7 @@ class TestCaseBase(unittest.TestCase):
class TestDependsOn(TestCaseBase):
ALL_TASK_NAMES = None
+ SUCCESS_DEPS_EXCLUDE = set(['success', 'release', 'release_test'])
def setUp(self):
super().setUp()
@@ -33,23 +34,29 @@ class TestDependsOn(TestCaseBase):
for key, _ in self.CIRRUS_YAML.items()
if key.endswith('_task')])
- def test_00_dicts(self):
+ def test_dicts(self):
"""Expected dictionaries are present and non-empty"""
self.assertIn('success_task', self.CIRRUS_YAML)
self.assertIn('success_task'.replace('_task', ''), self.ALL_TASK_NAMES)
self.assertIn('depends_on', self.CIRRUS_YAML['success_task'])
self.assertGreater(len(self.CIRRUS_YAML['success_task']['depends_on']), 0)
- def test_01_depends(self):
+ def test_task(self):
+ """There is no task named 'task'"""
+ self.assertNotIn('task', self.ALL_TASK_NAMES)
+
+ def test_depends(self):
"""Success task depends on all other tasks"""
success_deps = set(self.CIRRUS_YAML['success_task']['depends_on'])
- for task_name in self.ALL_TASK_NAMES - set(['success']):
+ for task_name in self.ALL_TASK_NAMES - self.SUCCESS_DEPS_EXCLUDE:
with self.subTest(task_name=task_name):
msg=('Please add "{0}" to the "depends_on" list in "success_task"'
"".format(task_name))
self.assertIn(task_name, success_deps, msg=msg)
-
+ def not_task(self):
+ """Ensure no task is named 'task'"""
+ self.assertNotIn('task', self.ALL_TASK_NAMES)
if __name__ == "__main__":
unittest.main()
diff --git a/contrib/cirrus/container_test.sh b/contrib/cirrus/container_test.sh
deleted file mode 100644
index 8ea66e63c..000000000
--- a/contrib/cirrus/container_test.sh
+++ /dev/null
@@ -1,142 +0,0 @@
-#!/usr/bin/env bash
-set -xeo pipefail
-
-export GOPATH=/var/tmp/go
-export PATH=$HOME/gopath/bin:$PATH:$GOPATH/bin
-export GOSRC=$GOPATH/src/github.com/containers/podman
-
-DIST=${DIST:=""}
-CONTAINER_RUNTIME=${DIST:=""}
-
-source /etc/os-release
-
-INTEGRATION_TEST_ENVS=""
-
-# For all distributions not Fedora, we need to skip USERNS tests
-# for now.
-if [ "${ID}" != "fedora" ] || [ "${CONTAINER_RUNTIME}" != "" ]; then
- INTEGRATION_TEST_ENVS="SKIP_USERNS=1"
-fi
-
-echo "$(date --rfc-3339=seconds) $(basename $0) started with '$*' and RCLI='${RCLI}'"
-
-pwd
-
-# -i install
-# -b build
-# -t integration test
-# -u unit test
-# -v validate
-
-build=0
-install=0
-integrationtest=0
-unittest=0
-validate=0
-options=0
-remote=0
-install_tools_made=0
-
-while getopts "bituv" opt; do
- case "$opt" in
- b) build=1
- options=1
- ;;
- i) install=1
- options=1
- ;;
- t) integrationtest=1
- options=1
- ;;
- u) unittest=1
- options=1
- ;;
- v) validate=1
- options=1
- ;;
- esac
-done
-
-# The RCLI environment variable decides whether
-# to test varlinke
-if [[ "$RCLI" == "true" ]]; then
- remote=1
-fi
-
-# If no options are passed, do everything
-if [ $options -eq 0 ]; then
- build=1
- install=1
- integrationtest=1
- unittest=1
- validate=1
-fi
-
-# Make Install tools function used by multiple sections below
-make_install_tools () {
- # Only make the install tools once
- if [ $install_tools_made -eq 0 ]; then
- make install.tools TAGS="${TAGS}"
- fi
- install_tools_made=1
-}
-
-CONTAINER_RUNTIME=${CONTAINER_RUNTIME:-none}
-
-if [ "${CONTAINER_RUNTIME}" == "none" ]; then
- mkdir -p /$GOPATH/src/github.com/containers/
- mv /var/tmp/checkout $GOSRC
- cd $GOSRC
- pwd
-fi
-
-
-export TAGS="seccomp $($GOSRC/hack/btrfs_tag.sh) $($GOSRC/hack/libdm_tag.sh) $($GOSRC/hack/btrfs_installed_tag.sh) $($GOSRC/hack/selinux_tag.sh)"
-
-# Validate
-if [ $validate -eq 1 ]; then
- make_install_tools
- # PAPR adds a merge commit, for testing, which fails the
- # short-commit-subject validation test, so tell git-validate.sh to only check
- # up to, but not including, the merge commit.
- export GITVALIDATE_TIP=$(cd $GOSRC; git log -2 --pretty='%H' | tail -n 1)
- make gofmt TAGS="${TAGS}"
-
- # Only check lint and gitvalidation on more recent
- # distros with updated git and tooling
- if [[ ${DIST} == "Fedora" ]]; then
- HEAD=$GITVALIDATE_TIP make -C $GOSRC .gitvalidation TAGS="${TAGS}"
- make lint
- fi
-fi
-
-# Unit tests
-if [ $unittest -eq 1 ]; then
- make localunit TAGS="${TAGS}"
-fi
-
-# Make Podman
-if [ $build -eq 1 ]; then
- make_install_tools
- make TAGS="${TAGS}" GOPATH=$GOPATH
- make podman-remote TAGS="${TAGS}" GOPATH=$GOPATH
-fi
-
-# Install Podman
-if [ $install -eq 1 ]; then
- make_install_tools
- make TAGS="${TAGS}" install.bin PREFIX=/usr ETCDIR=/etc
- make TAGS="${TAGS}" install.man PREFIX=/usr ETCDIR=/etc
- make TAGS="${TAGS}" install.cni PREFIX=/usr ETCDIR=/etc
- make TAGS="${TAGS}" install.systemd PREFIX=/usr ETCDIR=/etc
-fi
-
-# Run integration tests
-if [ $integrationtest -eq 1 ]; then
- make TAGS="${TAGS}" test-binaries
- make varlink_generate
- make localintegration $INTEGRATION_TEST_ENVS
- if [ $remote -eq 1 ]; then
- make remoteintegration $INTEGRATION_TEST_ENVS
- fi
-fi
diff --git a/contrib/cirrus/ext_svc_check.sh b/contrib/cirrus/ext_svc_check.sh
new file mode 100755
index 000000000..b89e3d753
--- /dev/null
+++ b/contrib/cirrus/ext_svc_check.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+set -eo pipefail
+
+# This script attempts basic confirmation of functional networking
+# by connecting to a set of essential external servers and failing
+# if any cannot be reached. It's intended for use early on in the
+# podman CI system, to help prevent wasting time on tests that can't
+# succeede due to some outage or another.
+
+# shellcheck source=./contrib/cirrus/lib.sh
+source $(dirname $0)/lib.sh
+
+cat ${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/required_host_ports.txt | \
+ while read host port
+ do
+ if [[ "$port" -eq "443" ]]
+ then
+ echo "SSL/TLS to $host:$port"
+ echo -n '' | \
+ err_retry 9 1000 "" openssl s_client -quiet -no_ign_eof -connect $host:$port
+ else
+ echo "Connect to $host:$port"
+ err_retry 9 1000 1 nc -zv -w 13 $host $port
+ fi
+ done
+
+# TODO: Pull images required during testing into /dev/null
+
+# TODO: Refresh DNF package-cache into /dev/null
diff --git a/contrib/cirrus/integration_test.sh b/contrib/cirrus/integration_test.sh
deleted file mode 100755
index 636d67411..000000000
--- a/contrib/cirrus/integration_test.sh
+++ /dev/null
@@ -1,65 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-source $(dirname $0)/lib.sh
-
-req_env_var GOSRC SCRIPT_BASE OS_RELEASE_ID OS_RELEASE_VER CONTAINER_RUNTIME VARLINK_LOG
-
-LOCAL_OR_REMOTE=local
-if [[ "$RCLI" = "true" ]]; then
- LOCAL_OR_REMOTE=remote
-fi
-
-# Our name must be of the form xxxx_test or xxxx_test.sh, where xxxx is
-# the test suite to run; currently (2019-05) the only option is 'integration'
-# but pr2947 intends to add 'system'.
-TESTSUITE=$(expr $(basename $0) : '\(.*\)_test')
-if [[ -z $TESTSUITE ]]; then
- die 1 "Script name ($basename $0) is not of the form xxxx_test.sh"
-fi
-
-cd "$GOSRC"
-
-case "$SPECIALMODE" in
- in_podman)
- ${CONTAINER_RUNTIME} run --rm --privileged --net=host --cgroupns=host \
- -v $GOSRC:$GOSRC:Z \
- --workdir $GOSRC \
- -e "CGROUP_MANAGER=cgroupfs" \
- -e "STORAGE_OPTIONS=--storage-driver=vfs" \
- -e "CRIO_ROOT=$GOSRC" \
- -e "PODMAN_BINARY=/usr/bin/podman" \
- -e "CONMON_BINARY=/usr/libexec/podman/conmon" \
- -e "DIST=$OS_RELEASE_ID" \
- -e "CONTAINER_RUNTIME=$CONTAINER_RUNTIME" \
- $IN_PODMAN_IMAGE bash $GOSRC/$SCRIPT_BASE/container_test.sh -b -i -t
- ;;
- rootless)
- req_env_var ROOTLESS_USER
- ssh $ROOTLESS_USER@localhost \
- -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \
- -o CheckHostIP=no $GOSRC/$SCRIPT_BASE/rootless_test.sh ${TESTSUITE} ${LOCAL_OR_REMOTE}
- ;;
- endpoint)
- make
- make install PREFIX=/usr ETCDIR=/etc
- make test-binaries
- make endpoint
- ;;
- bindings)
- make
- make install PREFIX=/usr ETCDIR=/etc
- export PATH=$PATH:`pwd`/hack
- cd pkg/bindings/test && ginkgo -trace -noColor -debug -r
- ;;
- none)
- make
- make install PREFIX=/usr ETCDIR=/etc
- make test-binaries
- make .install.bats
- make ${LOCAL_OR_REMOTE}${TESTSUITE} PODMAN_SERVER_LOG=$PODMAN_SERVER_LOG
- ;;
- *)
- die 110 "Unsupported \$SPECIALMODE: $SPECIALMODE"
-esac
diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh
index 0dbb57ab3..b57aee1af 100644
--- a/contrib/cirrus/lib.sh
+++ b/contrib/cirrus/lib.sh
@@ -3,243 +3,161 @@
# Library of common, shared utility functions. This file is intended
# to be sourced by other scripts, not called directly.
-# Global details persist here
-source /etc/environment # not always loaded under all circumstances
-
-# Automation environment doesn't automatically load for Ubuntu 18
-if [[ -r '/usr/share/automation/environment' ]]; then
- source '/usr/share/automation/environment'
+# BEGIN Global export of all variables
+set -a
+
+# Due to differences across platforms and runtime execution environments,
+# handling of the (otherwise) default shell setup is non-uniform. Rather
+# than attempt to workaround differences, simply force-load/set required
+# items every time this library is utilized.
+source /etc/profile
+source /etc/environment
+USER="$(whoami)"
+HOME="$(getent passwd $USER | cut -d : -f 6)"
+# Some platforms set and make this read-only
+[[ -n "$UID" ]] || \
+ UID=$(getent passwd $USER | cut -d : -f 3)
+GID=$(getent passwd $USER | cut -d : -f 4)
+
+# During VM Image build, the 'containers/automation' installation
+# was performed. The final step of that installation sets the
+# installation location in $AUTOMATION_LIB_PATH in /etc/environment
+# or in the default shell profile.
+# shellcheck disable=SC2154
+if [[ -n "$AUTOMATION_LIB_PATH" ]]; then
+ for libname in defaults anchors console_output utils; do
+ # There's no way shellcheck can process this location
+ # shellcheck disable=SC1090
+ source $AUTOMATION_LIB_PATH/${libname}.sh
+ done
+else
+ (
+ echo "WARNING: It does not appear that containers/automation was installed."
+ echo " Functionality of most of this library will be negatively impacted"
+ echo " This ${BASH_SOURCE[0]} was loaded by ${BASH_SOURCE[1]}"
+ ) > /dev/stderr
fi
-# Under some contexts these values are not set, make sure they are.
-export USER="$(whoami)"
-export HOME="$(getent passwd $USER | cut -d : -f 6)"
-[[ -n "$UID" ]] || export UID=$(getent passwd $USER | cut -d : -f 3)
-export GID=$(getent passwd $USER | cut -d : -f 4)
+OS_RELEASE_ID="$(source /etc/os-release; echo $ID)"
+# GCE image-name compatible string representation of distribution _major_ version
+OS_RELEASE_VER="$(source /etc/os-release; echo $VERSION_ID | cut -d '.' -f 1)"
+# Combined to ease soe usage
+OS_REL_VER="${OS_RELEASE_ID}-${OS_RELEASE_VER}"
# Essential default paths, many are overridden when executing under Cirrus-CI
-export GOPATH="${GOPATH:-/var/tmp/go}"
+GOPATH="${GOPATH:-/var/tmp/go}"
if type -P go &> /dev/null
then
- # required for go 1.12+
- export GOCACHE="${GOCACHE:-$HOME/.cache/go-build}"
+ # Cirrus-CI caches $GOPATH contents
+ export GOCACHE="${GOCACHE:-$GOPATH/cache/go-build}"
# called processes like `make` and other tools need these vars.
eval "export $(go env)"
# Ensure compiled tooling is reachable
- export PATH="$PATH:$GOPATH/bin"
+ PATH="$PATH:$GOPATH/bin:$HOME/.local/bin"
fi
-CIRRUS_WORKING_DIR="${CIRRUS_WORKING_DIR:-$GOPATH/src/github.com/containers/podman}"
-export GOSRC="${GOSRC:-$CIRRUS_WORKING_DIR}"
-export PATH="$HOME/bin:$GOPATH/bin:/usr/local/bin:$PATH"
-export LD_LIBRARY_PATH="/usr/local/lib${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}"
+CIRRUS_WORKING_DIR="${CIRRUS_WORKING_DIR:-$(realpath $(dirname ${BASH_SOURCE[0]})/../../)}"
+GOSRC="${GOSRC:-$CIRRUS_WORKING_DIR}"
+PATH="$HOME/bin:/usr/local/bin:$PATH"
+LD_LIBRARY_PATH="/usr/local/lib${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}"
+
# Saves typing / in case location ever moves
SCRIPT_BASE=${SCRIPT_BASE:-./contrib/cirrus}
-# Important filepaths
-SETUP_MARKER_FILEPATH="${SETUP_MARKER_FILEPATH:-/var/tmp/.setup_environment_sh_complete}"
+
# Downloaded, but not installed packages.
PACKAGE_DOWNLOAD_DIR=/var/cache/download
# Log remote-client system test varlink output here
-export VARLINK_LOG=/var/tmp/varlink.log
-
-cd $GOSRC
-if type -P git &> /dev/null && [[ -d "$GOSRC/.git" ]]
-then
- CIRRUS_CHANGE_IN_REPO=${CIRRUS_CHANGE_IN_REPO:-$(git show-ref --hash=8 HEAD || date +%s)}
-else # pick something unique and obviously not from Cirrus
- CIRRUS_CHANGE_IN_REPO=${CIRRUS_CHANGE_IN_REPO:-unknown_$(date +%s)}
-fi
+PODMAN_SERVER_LOG=$CIRRUS_WORKING_DIR/varlink.log
# Defaults when not running under CI
export CI="${CI:-false}"
CIRRUS_CI="${CIRRUS_CI:-false}"
DEST_BRANCH="${DEST_BRANCH:-master}"
CONTINUOUS_INTEGRATION="${CONTINUOUS_INTEGRATION:-false}"
-CIRRUS_REPO_NAME=${CIRRUS_REPO_NAME:-libpod}
+CIRRUS_REPO_NAME=${CIRRUS_REPO_NAME:-podman}
CIRRUS_BASE_SHA=${CIRRUS_BASE_SHA:-unknown$(date +%s)} # difficult to reliably discover
CIRRUS_BUILD_ID=${CIRRUS_BUILD_ID:-$RANDOM$(date +%s)} # must be short and unique
-OS_RELEASE_ID="$(source /etc/os-release; echo $ID)"
-# GCE image-name compatible string representation of distribution _major_ version
-OS_RELEASE_VER="$(source /etc/os-release; echo $VERSION_ID | cut -d '.' -f 1)"
-# Combined to ease soe usage
-OS_REL_VER="${OS_RELEASE_ID}-${OS_RELEASE_VER}"
+# Needed for linting and code validation
+EPOCH_TEST_COMMIT=${CIRRUS_BASE_SHA:-$CIRRUS_LAST_GREEN_CHANGE}
-# IN_PODMAN container image
-IN_PODMAN_IMAGE="quay.io/libpod/${OS_RELEASE_ID}_podman:$_BUILT_IMAGE_SUFFIX"
-# Image for uploading releases
-UPLDREL_IMAGE="quay.io/libpod/upldrel:master"
-
-# This is needed under some environments/contexts
-SUDO=''
-[[ "$UID" -eq 0 ]] || \
- SUDO='sudo -E'
-
-# Avoid getting stuck waiting for user input
-export DEBIAN_FRONTEND="noninteractive"
-SUDOAPTGET="$SUDO apt-get -qq --yes"
-SUDOAPTADD="$SUDO add-apt-repository --yes"
-# Regex that finds enabled periodic apt configuration items
-PERIODIC_APT_RE='^(APT::Periodic::.+")1"\;'
-# Short-cuts for retrying/timeout calls
-LILTO="timeout_attempt_delay_command 120s 5 30s"
-BIGTO="timeout_attempt_delay_command 300s 5 60s"
-
-# Safe env. vars. to transfer from root -> $ROOTLESS_USER (go env handled separately)
-ROOTLESS_ENV_RE='(CIRRUS_.+)|(ROOTLESS_.+)|(.+_IMAGE.*)|(.+_BASE)|(.*DIRPATH)|(.*FILEPATH)|(SOURCE.*)|(DEPEND.*)|(.+_DEPS_.+)|(OS_REL.*)|(.+_ENV_RE)|(TRAVIS)|(CI.+)|(REMOTE.*)'
+# Regex of env. vars. to explicitly pass when executing tests
+# inside a container or as a rootless user
+PASSTHROUGH_ENV_RE='(^CI.*)|(^CIRRUS)|(^DISTRO_NV)|(^GOPATH)|(^GOCACHE)|(^GOSRC)|(^SCRIPT_BASE)|(CGROUP_MANAGER)|(OCI_RUNTIME)|(^TEST.*)|(^PODBIN_NAME)|(^PRIV_NAME)|(^ALT_NAME)|(^ROOTLESS_USER)|(SKIP_USERNS)|(.*_NAME)|(.*_FQIN)'
# Unsafe env. vars for display
-SECRET_ENV_RE='(ACCOUNT)|(GC[EP]..+)|(SSH)'
+SECRET_ENV_RE='(ACCOUNT)|(GC[EP]..+)|(SSH)|(PASSWORD)|(TOKEN)'
-SPECIALMODE="${SPECIALMODE:-none}"
-RCLI="${RCLI:-false}"
-export CONTAINER_RUNTIME=${CONTAINER_RUNTIME:-podman}
-
-# When running as root, this may be empty or not, as a user, it MUST be set.
-if [[ "$USER" == "root" ]]
-then
- ROOTLESS_USER="${ROOTLESS_USER:-}"
-else
- ROOTLESS_USER="${ROOTLESS_USER:-$USER}"
-fi
# Type of filesystem used for cgroups
CG_FS_TYPE="$(stat -f -c %T /sys/fs/cgroup)"
-# Pass in a list of one or more envariable names; exit non-zero with
-# helpful error message if any value is empty
-req_env_var() {
- # Provide context. If invoked from function use its name; else script name
- local caller=${FUNCNAME[1]}
- if [[ -n "$caller" ]]; then
- # Indicate that it's a function name
- caller="$caller()"
- else
- # Not called from a function: use script name
- caller=$(basename $0)
- fi
-
- # Usage check
- [[ -n "$1" ]] || die 1 "FATAL: req_env_var: invoked without arguments"
-
- # Each input arg is an envariable name, e.g. HOME PATH etc. Expand each.
- # If any is empty, bail out and explain why.
- for i; do
- if [[ -z "${!i}" ]]; then
- die 9 "FATAL: $caller requires \$$i to be non-empty"
- fi
+# Set to 1 in all podman container images
+CONTAINER="${CONTAINER:-0}"
+
+# END Global export of all variables
+set +a
+
+lilto() { err_retry 8 1000 "" "$@"; } # just over 4 minutes max
+bigto() { err_retry 7 5670 "" "$@"; } # 12 minutes max
+
+# Print shell-escaped variable=value pairs, one per line, based on
+# variable name matching a regex. This is intended to support
+# passthrough of CI variables from host -> container or from root -> user.
+# For all other vars. we rely on tooling to load this library from inside
+# the container or as rootless user to pickup the remainder.
+passthrough_envars(){
+ local xchars
+ local envname
+ local envval
+ # Avoid values containing entirely punctuation|control|whitespace
+ xchars='[:punct:][:cntrl:][:space:]'
+ warn "Will pass env. vars. matching the following regex:
+ $PASSTHROUGH_ENV_RE"
+ for envname in $(awk 'BEGIN{for(v in ENVIRON) print v}' | \
+ grep -Ev "SETUP_ENVIRONMENT" | \
+ grep -Ev "$SECRET_ENV_RE" | \
+ grep -E "$PASSTHROUGH_ENV_RE"); do
+
+ envval="${!envname}"
+ [[ -n $(tr -d "$xchars" <<<"$envval") ]] || continue
+
+ # Properly escape values to prevent injection
+ printf -- "$envname=%q\n" "$envval"
done
}
-item_test() {
- ITEM="$1"
- shift
- TEST_ARGS="$@"
- req_env_var ITEM TEST_ARGS
-
- if ERR=$(test "$@" 2>&1)
- then
- echo "ok $ITEM"
- return 0
- else
- RET=$?
- echo -n "not ok $ITEM: $TEST_ARGS"
- if [[ -z "$ERR" ]]
- then
- echo ""
- else # test command itself failed
- echo -n ":" # space follows :'s in $ERR
- echo "$ERR" | cut -d : -f 4- # omit filename, line number, and command
- fi
- return $RET
- fi
-}
-
-show_env_vars() {
- echo "Showing selection of environment variable definitions:"
- _ENV_VAR_NAMES=$(awk 'BEGIN{for(v in ENVIRON) print v}' | \
- egrep -v "(^PATH$)|(^BASH_FUNC)|(^[[:punct:][:space:]]+)|$SECRET_ENV_RE" | \
- sort -u)
- for _env_var_name in $_ENV_VAR_NAMES
- do
- # Supports older BASH versions
- printf " ${_env_var_name}=%q\n" "$(printenv $_env_var_name)"
- done
-}
-
-die() {
- echo "************************************************"
- echo ">>>>> ${2:-FATAL ERROR (but no message given!) in ${FUNCNAME[1]}()}"
- echo "************************************************"
- exit ${1:-1}
-}
-
-warn() {
- echo ">>>>> ${1:-WARNING (but no message given!) in ${FUNCNAME[1]}()}" > /dev/stderr
-}
-
-bad_os_id_ver() {
- echo "Unknown/Unsupported distro. $OS_RELEASE_ID and/or version $OS_RELEASE_VER for $(basename $0)"
- exit 42
-}
-
-stub() {
- echo "STUB: Pretending to do $1"
-}
-
-timeout_attempt_delay_command() {
- TIMEOUT=$1
- ATTEMPTS=$2
- DELAY=$3
- shift 3
- STDOUTERR=$(mktemp -p '' $(basename $0)_XXXXX)
- req_env_var ATTEMPTS DELAY
- echo "Retrying $ATTEMPTS times with a $DELAY delay, and $TIMEOUT timeout for command: $@"
- for (( COUNT=1 ; COUNT <= $ATTEMPTS ; COUNT++ ))
- do
- echo "##### (attempt #$COUNT)" &>> "$STDOUTERR"
- if timeout --foreground $TIMEOUT "$@" &>> "$STDOUTERR"
- then
- echo "##### (success after #$COUNT attempts)" &>> "$STDOUTERR"
- break
- else
- echo "##### (failed with exit: $?)" &>> "$STDOUTERR"
- sleep $DELAY
- fi
- done
- cat "$STDOUTERR"
- rm -f "$STDOUTERR"
- if (( COUNT > $ATTEMPTS ))
- then
- echo "##### (exceeded $ATTEMPTS attempts)"
- exit 125
- fi
-}
-
setup_rootless() {
- req_env_var ROOTLESS_USER GOPATH GOSRC SECRET_ENV_RE ROOTLESS_ENV_RE
+ req_env_vars ROOTLESS_USER GOPATH GOSRC SECRET_ENV_RE
+
+ local rootless_uid
+ local rootless_gid
+ local env_var_val
- # Only do this once
+ # Only do this once; established by setup_environment.sh
+ # shellcheck disable=SC2154
if passwd --status $ROOTLESS_USER
then
- echo "Updating $ROOTLESS_USER user permissions on possibly changed libpod code"
+ msg "Updating $ROOTLESS_USER user permissions on possibly changed libpod code"
chown -R $ROOTLESS_USER:$ROOTLESS_USER "$GOPATH" "$GOSRC"
return 0
fi
-
- cd $GOSRC
+ msg "************************************************************"
+ msg "Setting up rootless user '$ROOTLESS_USER'"
+ msg "************************************************************"
+ cd $GOSRC || exit 1
# Guarantee independence from specific values
- ROOTLESS_UID=$[RANDOM+1000]
- ROOTLESS_GID=$[RANDOM+1000]
- echo "creating $ROOTLESS_UID:$ROOTLESS_GID $ROOTLESS_USER user"
- groupadd -g $ROOTLESS_GID $ROOTLESS_USER
- useradd -g $ROOTLESS_GID -u $ROOTLESS_UID --no-user-group --create-home $ROOTLESS_USER
+ rootless_uid=$[RANDOM+1000]
+ rootless_gid=$[RANDOM+1000]
+ msg "creating $rootless_uid:$rootless_gid $ROOTLESS_USER user"
+ groupadd -g $rootless_gid $ROOTLESS_USER
+ useradd -g $rootless_gid -u $rootless_uid --no-user-group --create-home $ROOTLESS_USER
chown -R $ROOTLESS_USER:$ROOTLESS_USER "$GOPATH" "$GOSRC"
- echo "creating ssh key pair for $USER"
+ msg "creating ssh key pair for $USER"
[[ -r "$HOME/.ssh/id_rsa" ]] || \
ssh-keygen -P "" -f "$HOME/.ssh/id_rsa"
- echo "Allowing ssh key for $ROOTLESS_USER"
+ msg "Allowing ssh key for $ROOTLESS_USER"
(umask 077 && mkdir "/home/$ROOTLESS_USER/.ssh")
chown -R $ROOTLESS_USER:$ROOTLESS_USER "/home/$ROOTLESS_USER/.ssh"
install -o $ROOTLESS_USER -g $ROOTLESS_USER -m 0600 \
@@ -247,49 +165,38 @@ setup_rootless() {
# Makes debugging easier
cat /root/.ssh/authorized_keys >> "/home/$ROOTLESS_USER/.ssh/authorized_keys"
- echo "Configuring subuid and subgid"
+ msg "Configuring subuid and subgid"
grep -q "${ROOTLESS_USER}" /etc/subuid || \
- echo "${ROOTLESS_USER}:$[ROOTLESS_UID * 100]:65536" | \
+ echo "${ROOTLESS_USER}:$[rootless_uid * 100]:65536" | \
tee -a /etc/subuid >> /etc/subgid
# Env. vars set by Cirrus and setup_environment.sh must be explicitly
# transferred to the test-user.
- echo "Configuring rootless user's environment variables:"
- echo "# Added by $GOSRC/$SCRIPT_PATH/lib.sh setup_rootless()"
- _ENV_VAR_NAMES=$(awk 'BEGIN{for(v in ENVIRON) print v}' | \
- egrep -v "(^PATH$)|(^BASH_FUNC)|(^[[:punct:][:space:]]+)|$SECRET_ENV_RE" | \
- egrep "$ROOTLESS_ENV_RE" | \
- sort -u)
- for _env_var_name in $_ENV_VAR_NAMES
- do
- # Works with older versions of bash
- printf "${_env_var_name}=%q\n" "$(printenv $_env_var_name)" >> "/home/$ROOTLESS_USER/.bashrc"
- done
+ msg "Configuring rootless user's environment variables:"
+
+ (
+ echo "# Added by ${BASH_SOURCE[0]} ${FUNCNAME[0]}()"
+ echo "export SETUP_ENVIRONMENT=1"
+ ) >> "/home/$ROOTLESS_USER/.bashrc"
- echo "Ensure the systems ssh process is up and running within 5 minutes"
+ while read -r env_var_val; do
+ echo "export $env_var_val" >> "/home/$ROOTLESS_USER/.bashrc"
+ done <<<"$(passthrough_envars)"
+ chown $ROOTLESS_USER:$ROOTLESS_USER "/home/$ROOTLESS_USER/.bashrc"
+ cat "/home/$ROOTLESS_USER/.bashrc" | indent 2
+
+ msg "Ensure the systems ssh process is up and running within 5 minutes"
systemctl start sshd
- NOW=$(date +%s)
- TIMEOUT=$(date --date '+5 minutes' +%s)
- while [[ "$(date +%s)" -lt "$TIMEOUT" ]]
- do
- if timeout --foreground -k 1s 1s \
- ssh $ROOTLESS_USER@localhost \
- -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o CheckHostIP=no \
- true
- then
- break
- else
- sleep 2s
- fi
- done
- [[ "$(date +%s)" -lt "$TIMEOUT" ]] || \
- die 11 "Timeout exceeded waiting for localhost ssh capability"
+ lilto ssh $ROOTLESS_USER@localhost \
+ -o UserKnownHostsFile=/dev/null \
+ -o StrictHostKeyChecking=no \
+ -o CheckHostIP=no true
}
install_test_configs() {
echo "Installing cni config, policy and registry config"
- req_env_var GOSRC SCRIPT_BASE
- cd $GOSRC
+ req_env_vars GOSRC SCRIPT_BASE
+ cd $GOSRC || exit 1
install -v -D -m 644 ./cni/87-podman-bridge.conflist /etc/cni/net.d/
# This config must always sort last in the list of networks (podman picks first one
# as the default). This config prevents allocation of network address space used
@@ -306,7 +213,7 @@ install_test_configs() {
# can only run the compiled source version.
remove_packaged_podman_files() {
echo "Removing packaged podman files to prevent conflicts with source build and testing."
- req_env_var OS_RELEASE_ID
+ req_env_vars OS_RELEASE_ID
# If any binaries are resident they could cause unexpected pollution
for unit in io.podman.service io.podman.socket
@@ -324,9 +231,9 @@ remove_packaged_podman_files() {
if [[ "$OS_RELEASE_ID" =~ "ubuntu" ]]
then
- LISTING_CMD="$SUDO dpkg-query -L podman"
+ LISTING_CMD="dpkg-query -L podman"
else
- LISTING_CMD="$SUDO rpm -ql podman"
+ LISTING_CMD="rpm -ql podman"
fi
# yum/dnf/dpkg may list system directories, only remove files
@@ -334,29 +241,9 @@ remove_packaged_podman_files() {
do
# Sub-directories may contain unrelated/valuable stuff
if [[ -d "$fullpath" ]]; then continue; fi
- ooe.sh $SUDO rm -vf "$fullpath"
+ ooe.sh rm -vf "$fullpath"
done
# Be super extra sure and careful vs performant and completely safe
- sync && echo 3 > /proc/sys/vm/drop_caches
-}
-
-# The version of CRI-O and Kubernetes must always match
-get_kubernetes_version(){
- # TODO: Look up the kube RPM/DEB version installed, or in $PACKAGE_DOWNLOAD_DIR
- # and retrieve the major-minor version directly.
- local KUBERNETES_VERSION="1.15"
- echo "$KUBERNETES_VERSION"
-}
-
-canonicalize_image_names() {
- req_env_var IMGNAMES
- echo "Adding all current base images to \$IMGNAMES for timestamp update"
- export IMGNAMES="\
-$IMGNAMES
-$UBUNTU_BASE_IMAGE
-$PRIOR_UBUNTU_BASE_IMAGE
-$FEDORA_BASE_IMAGE
-$PRIOR_FEDORA_BASE_IMAGE
-"
+ sync && echo 3 > /proc/sys/vm/drop_caches || true
}
diff --git a/contrib/cirrus/lib.sh.t b/contrib/cirrus/lib.sh.t
deleted file mode 100755
index 643b5513d..000000000
--- a/contrib/cirrus/lib.sh.t
+++ /dev/null
@@ -1,121 +0,0 @@
-#!/usr/bin/env bash
-#
-# Unit tests for some functions in lib.sh
-#
-source $(dirname $0)/lib.sh
-
-# Iterator and return code; updated in test functions
-testnum=0
-rc=0
-
-function check_result {
- testnum=$(expr $testnum + 1)
- MSG=$(echo "$1" | tr -d '*>\012'|sed -e 's/^ \+//')
- if [ "$MSG" = "$2" ]; then
- echo "ok $testnum $(echo $3) = $(echo $MSG)"
- else
- echo "not ok $testnum $3"
- echo "# expected: $2"
- echo "# actual: $MSG"
- rc=1
- fi
-}
-
-###############################################################################
-# tests for die()
-
-function test_die() {
- local input_status=$1
- local input_msg=$2
- local expected_status=$3
- local expected_msg=$4
-
- local msg
- msg=$(die $input_status "$input_msg")
- local status=$?
-
- check_result "$msg" "$expected_msg" "die $input_status $input_msg"
-}
-
-test_die 1 "a message" 1 "a message"
-test_die 2 "" 2 "FATAL ERROR (but no message given!) in test_die()"
-test_die '' '' 1 "FATAL ERROR (but no message given!) in test_die()"
-
-###############################################################################
-# tests for req_env_var()
-
-function test_rev() {
- local input_args=$1
- local expected_status=$2
- local expected_msg=$3
-
- # bash gotcha: doing 'local msg=...' on one line loses exit status
- local msg
- msg=$(req_env_var $input_args)
- local status=$?
-
- check_result "$msg" "$expected_msg" "req_env_var $input_args"
- check_result "$status" "$expected_status" "req_env_var $input_args (rc)"
-}
-
-# error if called with no args
-test_rev '' 1 'FATAL: req_env_var: invoked without arguments'
-
-# error if desired envariable is unset
-unset FOO BAR
-test_rev FOO 9 'FATAL: test_rev() requires $FOO to be non-empty'
-test_rev BAR 9 'FATAL: test_rev() requires $BAR to be non-empty'
-# OK if desired envariable was unset
-FOO=1
-test_rev FOO 0 ''
-
-# OK if multiple vars are non-empty
-FOO="stuff"
-BAR="things"
-ENV_VARS="FOO BAR"
-test_rev "$ENV_VARS" 0 ''
-unset BAR
-
-# ...but error if any single desired one is unset
-test_rev "FOO BAR" 9 'FATAL: test_rev() requires $BAR to be non-empty'
-
-# ...and OK if all args are set
-BAR=1
-test_rev "FOO BAR" 0 ''
-
-###############################################################################
-# tests for item_test()
-
-function test_item_test {
- local exp_msg=$1
- local exp_ret=$2
- local item=$3
- shift 3
- local test_args="$@"
- local msg
- msg=$(item_test "$item" "$@")
- local status=$?
-
- check_result "$msg" "$exp_msg" "test_item $item $test_args"
- check_result "$status" "$exp_ret" "test_item $item $test_args (actual rc $status)"
-}
-
-# negative tests
-test_item_test "FATAL: item_test() requires \$ITEM to be non-empty" 9 "" ""
-test_item_test "FATAL: item_test() requires \$TEST_ARGS to be non-empty" 9 "foo" ""
-test_item_test "not ok foo: -gt 5 ~= bar: too many arguments" 2 "foo" "-gt" "5" "~=" "bar"
-test_item_test "not ok bar: a -ge 10: a: integer expression expected" 2 "bar" "a" "-ge" "10"
-test_item_test "not ok basic logic: 0 -ne 0" 1 "basic logic" "0" "-ne" "0"
-
-# positive tests
-test_item_test "ok snafu" 0 "snafu" "foo" "!=" "bar"
-test_item_test "ok foobar" 0 "foobar" "one two three" "=" "one two three"
-test_item_test "ok oh boy" 0 "oh boy" "line 1
-line2" "!=" "line 1
-
-line2"
-test_item_test "ok okay enough" 0 "okay enough" "line 1
-line2" "=" "line 1
-line2"
-
-exit $rc
diff --git a/contrib/cirrus/logcollector.sh b/contrib/cirrus/logcollector.sh
index fd5017b44..9dcc252c0 100755
--- a/contrib/cirrus/logcollector.sh
+++ b/contrib/cirrus/logcollector.sh
@@ -2,9 +2,10 @@
set -e
+# shellcheck source=contrib/cirrus/lib.sh
source $(dirname $0)/lib.sh
-req_env_var CIRRUS_WORKING_DIR OS_RELEASE_ID RCLI
+req_env_vars CIRRUS_WORKING_DIR OS_RELEASE_ID
# Assume there are other log collection commands to follow - Don't
# let one break another that may be useful, but also keep any
@@ -33,14 +34,9 @@ case $1 in
ginkgo) showrun cat $CIRRUS_WORKING_DIR/test/e2e/ginkgo-node-*.log ;;
journal) showrun journalctl -b ;;
podman) showrun ./bin/podman system info ;;
- varlink)
- if [[ "$RCLI" == "true" ]]
- then
- echo "(Trailing 100 lines of $VARLINK_LOG)"
- showrun tail -100 $VARLINK_LOG
- else
- die 0 "\$RCLI is not 'true': $RCLI"
- fi
+ server)
+ msg "(Trailing 100 lines of $PODMAN_SERVER_LOG)"
+ if [[ -r "$PODMAN_SERVER_LOG" ]]; then tail -100 $PODMAN_SERVER_LOG; fi
;;
packages)
# These names are common to Fedora and Ubuntu
@@ -78,7 +74,7 @@ case $1 in
echo "Kernel: " $(uname -r)
echo "Cgroups: " $(stat -f -c %T /sys/fs/cgroup)
# Any not-present packages will be listed as such
- $PKG_LST_CMD ${PKG_NAMES[@]} | sort -u
+ $PKG_LST_CMD "${PKG_NAMES[@]}" | sort -u
;;
- *) die 1 "Warning, $(basename $0) doesn't know how to handle the parameter '$1'"
+ *) die "Warning, $(basename $0) doesn't know how to handle the parameter '$1'"
esac
diff --git a/contrib/cirrus/networking.sh b/contrib/cirrus/networking.sh
deleted file mode 100755
index 824d03e44..000000000
--- a/contrib/cirrus/networking.sh
+++ /dev/null
@@ -1,17 +0,0 @@
-#!/usr/bin/env bash
-
-# This script attempts basic confirmation of functional networking
-# by connecting to a set of essential external servers and failing
-# if any cannot be reached.
-
-source $(dirname $0)/lib.sh
-
-while read host port
-do
- if [[ "$port" -eq "443" ]]
- then
- item_test "SSL/TLS to $host:$port" "$(echo -n '' | timeout 60 openssl s_client -quiet -no_ign_eof -connect $host:$port &> /dev/null; echo $?)" -eq "0"
- else
- item_test "Connect to $host:$port" "$(nc -zv -w 13 $host $port &> /dev/null; echo $?)" -eq 0
- fi
-done < ${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/required_host_ports.txt
diff --git a/contrib/cirrus/ooe.sh b/contrib/cirrus/ooe.sh
deleted file mode 100755
index 0966b5ce8..000000000
--- a/contrib/cirrus/ooe.sh
+++ /dev/null
@@ -1,39 +0,0 @@
-#!/usr/bin/env bash
-
-# This script executes a command while logging all output to a temporary
-# file. If the command exits non-zero, then all output is sent to the console,
-# before returning the exit code. If the script itself fails, the exit code 121
-# is returned.
-
-set -eo pipefail
-
-SCRIPT_BASEDIR="$(basename $0)"
-
-badusage() {
- echo "Incorrect usage: $SCRIPT_BASEDIR) <command> [options]" > /dev/stderr
- echo "ERROR: $1"
- exit 121
-}
-
-COMMAND="$@"
-[[ -n "$COMMAND" ]] || badusage "No command specified"
-
-OUTPUT_TMPFILE="$(mktemp -p '' ${SCRIPT_BASEDIR}_output_XXXX)"
-output_on_error() {
- RET=$?
- set +e
- if [[ "$RET" -ne "0" ]]
- then
- echo "---------------------------"
- cat "$OUTPUT_TMPFILE"
- echo "[$(date --iso-8601=second)] <exit $RET> $COMMAND"
- fi
- rm -f "$OUTPUT_TMPFILE"
-}
-trap "output_on_error" EXIT
-
-"$@" 2>&1 | while IFS='' read LINE # Preserve leading/trailing whitespace
-do
- # Every stdout and (copied) stderr line
- echo "[$(date --iso-8601=second)] $LINE"
-done >> "$OUTPUT_TMPFILE"
diff --git a/contrib/cirrus/rootless_test.sh b/contrib/cirrus/rootless_test.sh
deleted file mode 100755
index 31db18302..000000000
--- a/contrib/cirrus/rootless_test.sh
+++ /dev/null
@@ -1,32 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-echo "$(date --rfc-3339=seconds) $(basename $0) started with '$*'"
-
-source $(dirname $0)/lib.sh
-
-if [[ "$UID" == "0" ]]
-then
- echo "$(basename $0): Error: Expected to be running as a regular user"
- exit 1
-fi
-
-TESTSUITE=${1?Missing TESTSUITE argument (arg1)}
-LOCAL_OR_REMOTE=${2?Missing LOCAL_OR_REMOTE argument (arg2)}
-
-# Ensure environment setup correctly
-req_env_var GOSRC ROOTLESS_USER
-
-echo "."
-echo "Hello, my name is $USER and I live in $PWD can I be your friend?"
-echo "."
-
-show_env_vars
-
-set -x
-cd "$GOSRC"
-make
-make varlink_generate
-make test-binaries
-make ${LOCAL_OR_REMOTE}${TESTSUITE}
diff --git a/contrib/cirrus/runner.sh b/contrib/cirrus/runner.sh
new file mode 100755
index 000000000..82e512de9
--- /dev/null
+++ b/contrib/cirrus/runner.sh
@@ -0,0 +1,233 @@
+#!/bin/bash
+
+set -eo pipefail
+
+# This script is intended to be called by automation or humans,
+# from a specially configured environment. Depending on the contents
+# of various variable, entirely different operations will be performed.
+
+# shellcheck source=contrib/cirrus/lib.sh
+source $(dirname $0)/lib.sh
+
+exec_container() {
+ local var_val
+ local cmd
+ # Required to be defined by caller
+ # shellcheck disable=SC2154
+ msg "Re-executing runner inside container: $CTR_FQIN"
+ msg "************************************************************"
+
+ req_env_vars CTR_FQIN TEST_ENVIRON CONTAINER SECRET_ENV_RE
+
+ # Line-separated arguments which include shell-escaped special characters
+ declare -a envargs
+ while read -r var_val; do
+ envargs+=("-e $var_val")
+ done <<<"$(passthrough_envars)"
+
+ # VM Images and Container images are built using (nearly) identical operations.
+ set -x
+ # shellcheck disable=SC2154
+ exec podman run --rm --privileged --net=host --cgroupns=host \
+ -v /dev/fuse:/dev/fuse \
+ -v "$GOPATH:$GOPATH:Z" \
+ --workdir "$GOSRC" \
+ -e "CONTAINER=1" \
+ "${envargs[@]}" \
+ $CTR_FQIN bash -c "$SCRIPT_BASE/setup_environment.sh && $SCRIPT_BASE/runner.sh"
+}
+
+build_swagger() {
+ local download_url
+ # Building this is a PITA, just grab binary for use in automation
+ # Ref: https://goswagger.io/install.html#static-binary
+ download_url=$(\
+ curl -s https://api.github.com/repos/go-swagger/go-swagger/releases/latest | \
+ jq -r '.assets[] | select(.name | contains("linux_amd64")) | .browser_download_url')
+ curl -o /usr/local/bin/swagger -L'#' "$download_url"
+ chmod +x /usr/local/bin/swagger
+
+ cd $GOSRC
+ make swagger
+}
+
+altbuild() {
+ req_env_vars ALT_NAME
+ # Defined in .cirrus.yml
+ # shellcheck disable=SC2154
+ msg "Performing alternate build: $ALT_NAME"
+ msg "************************************************************"
+ cd $GOSRC
+ case "$ALT_NAME" in
+ *Each*)
+ git fetch origin
+ make build-all-new-commits GIT_BASE_BRANCH=origin/$DEST_BRANCH
+ ;;
+ *Windows*)
+ make podman-remote-windows-release
+ make podman.msi
+ ;;
+ *Without*)
+ make build-no-cgo
+ ;;
+ *varlink-API)
+ export SUGGESTION='remove API.md, then "make varlink_api_generate" and commit changes.'
+ make varlink_api_generate BUILDTAGS="varlink"
+ ./hack/tree_status.sh
+ ;;
+ *varlink-binaries)
+ make clean BUILDTAGS="varlink" binaries
+ ;;
+ *RPM*)
+ make -f ./.copr/Makefile
+ rpmbuild --rebuild ./podman-*.src.rpm
+ ;;
+ *Static*)
+ req_env_vars CTR_FQIN
+ [[ "$UID" -eq 0 ]] || \
+ die "Static build must execute nixos container as root on host"
+ mkdir -p /var/cache/nix
+ podman run -i --rm -v /var/cache/nix:/mnt/nix:Z \
+ $CTR_FQIN cp -rfT /nix /mnt/nix
+ podman run -i --rm -v /var/cache/nix:/nix:Z \
+ -v $PWD:$PWD:Z -w $PWD $CTR_FQIN \
+ nix --print-build-logs --option cores 4 --option max-jobs 4 \
+ build --file ./nix/
+ # result symlink is absolute from container perspective :(
+ cp /var/cache/$(readlink result)/bin/podman ./ # for cirrus-ci artifact
+ rm result # makes cirrus puke
+ ;;
+ *)
+ die "Unknown/Unsupported \$$ALT_NAME '$ALT_NAME'"
+ esac
+}
+
+integration_outputfilter() {
+ # Use similar format as human-friendly task name from .cirrus.yml
+ # shellcheck disable=SC2154
+ output_name="$TEST_FLAVOR-$PODBIN_NAME-$DISTRO_NV-$PRIV_NAME-$TEST_ENVIRON"
+ # Requires stdin and stderr combined!
+ cat - \
+ |& awk --file "${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/timestamp.awk" \
+ |& "${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/logformatter" "$output_name"
+}
+
+# Handle local|remote integration|system testing in a uniform way
+dotest() {
+ local testsuite="$1"
+ req_env_vars testsuite CONTAINER TEST_ENVIRON PRIV_NAME
+
+ # shellcheck disable=SC2154
+ if ((CONTAINER==0)) && [[ "$TEST_ENVIRON" == "container" ]]; then
+ exec_container # does not return
+ fi;
+
+ # shellcheck disable=SC2154
+ if [[ "$PRIV_NAME" == "rootless" ]] && [[ "$UID" -eq 0 ]]; then
+ req_env_vars ROOTLESS_USER
+ msg "Re-executing runner through ssh as user '$ROOTLESS_USER'"
+ msg "************************************************************"
+ set -x
+ exec ssh $ROOTLESS_USER@localhost \
+ -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \
+ -o CheckHostIP=no $GOSRC/$SCRIPT_BASE/runner.sh
+ # does not return
+ fi
+
+ output_filter="cat" # no filter
+ if [[ "$testsuite" == "integration" ]]; then
+ output_filter=integration_outputfilter
+ fi
+
+ # containers/automation sets this to 0 for it's dbg() function
+ # but the e2e integration tests are also sensitive to it.
+ unset DEBUG
+
+ # shellcheck disable=SC2154
+ case "$PODBIN_NAME" in
+ podman)
+ # ginkgo doesn't play nicely with C Go
+ make local${testsuite} \
+ |& "$output_filter"
+ ;;
+ remote)
+ make remote${testsuite} PODMAN_SERVER_LOG=$PODMAN_SERVER_LOG \
+ |& "$output_filter"
+ ;;
+ esac
+}
+
+msg "************************************************************"
+# Required to be defined by caller
+# shellcheck disable=SC2154
+msg "Runner executing $TEST_FLAVOR $PODBIN_NAME-tests as $PRIV_NAME on $DISTRO_NV($OS_REL_VER)"
+if ((CONTAINER)); then
+ # shellcheck disable=SC2154
+ msg "Current environment container image: $CTR_FQIN"
+else
+ # shellcheck disable=SC2154
+ msg "Current environment VM image: $VM_IMAGE_NAME"
+fi
+msg "************************************************************"
+
+((${SETUP_ENVIRONMENT:-0})) || \
+ die "Expecting setup_environment.sh to have completed successfully"
+
+cd "${GOSRC}/"
+
+case "$TEST_FLAVOR" in
+ ext_svc) $SCRIPT_BASE/ext_svc_check.sh ;;
+ smoke)
+ make gofmt
+ make .gitvalidation
+ ;;
+ automation)
+ $SCRIPT_BASE/cirrus_yaml_test.py
+ req_env_vars CI DEST_BRANCH IMAGE_SUFFIX TEST_FLAVOR TEST_ENVIRON \
+ PODBIN_NAME PRIV_NAME DISTRO_NV CONTAINER USER HOME \
+ UID GID AUTOMATION_LIB_PATH SCRIPT_BASE OS_RELEASE_ID \
+ OS_RELEASE_VER CG_FS_TYPE
+ bigto ooe.sh dnf install -y ShellCheck # small/quick addition
+ $SCRIPT_BASE/shellcheck.sh
+ ;;
+ altbuild) altbuild ;;
+ build)
+ make podman-release
+ make podman-remote-linux-release
+ ;;
+ validate)
+ # Confirm compiile via prior task + cache
+ bin/podman --version
+ bin/podman-remote --version
+ make validate # Some items require a build
+ ;;
+ bindings)
+ # shellcheck disable=SC2155
+ export PATH=$PATH:$GOSRC/hack
+ cd pkg/bindings/test && ginkgo -trace -noColor -debug -r
+ ;;
+ endpoint)
+ make test-binaries
+ make endpoint
+ ;;
+ swagger)
+ build_swagger
+ # Cirrus-CI Artifact instruction expects file here
+ cp -v $GOSRC/pkg/api/swagger.yaml $GOSRC/
+ ;;
+ vendor)
+ make vendor
+ ./hack/tree_status.sh
+ ;;
+ docker-py) msg "This is docker-py stub, it is only a stub" ;;
+ unit) make localunit ;;
+ int) dotest integration ;;
+ sys) dotest system ;;
+ release)
+ if bin/podman info |& grep -Eq -- '-dev'; then
+ die "Releases must never contain '-dev' in output of 'podman info'"
+ fi
+ ;;
+ *)
+ die "Unknown/Unsupported \$TEST_FLAVOR=$TEST_FLAVOR" ;;
+esac
diff --git a/contrib/cirrus/setup_container_environment.sh b/contrib/cirrus/setup_container_environment.sh
deleted file mode 100755
index 72542df17..000000000
--- a/contrib/cirrus/setup_container_environment.sh
+++ /dev/null
@@ -1,16 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-source $(dirname $0)/lib.sh
-
-req_env_var GOSRC OS_RELEASE_ID CONTAINER_RUNTIME
-
-# Since CRIU 3.11 has been pushed to Fedora 28 the checkpoint/restore
-# test cases are actually run. As CRIU uses iptables to lock and unlock
-# the network during checkpoint and restore it needs the following two
-# modules loaded.
-modprobe ip6table_nat || :
-modprobe iptable_nat || :
-
-# Pull the test image
-${CONTAINER_RUNTIME} pull ${IN_PODMAN_IMAGE}
diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh
index b406d7b5c..c064b6840 100755
--- a/contrib/cirrus/setup_environment.sh
+++ b/contrib/cirrus/setup_environment.sh
@@ -2,119 +2,187 @@
set -e
+# shellcheck source=./contrib/cirrus/lib.sh
source $(dirname $0)/lib.sh
-req_env_var USER HOME GOSRC SCRIPT_BASE SETUP_MARKER_FILEPATH
+die_unknown() {
+ local var_name="$1"
+ req_env_vars var_name
+ local var_value="${!var_name}"
+ die "Unknown/unsupported \$$var_name '$var_value'"
+}
-# Ensure this script only executes successfully once and always logs ending timestamp
-if [[ -e "$SETUP_MARKER_FILEPATH" ]]; then
- show_env_vars
- exit 0
-fi
+msg "************************************************************"
+msg "Setting up runtime environment"
+msg "************************************************************"
+show_env_vars
-exithandler() {
- RET=$?
- echo "."
- echo "$(basename $0) exit status: $RET"
- [[ "$RET" -eq "0" ]] && date +%s >> "$SETUP_MARKER_FILEPATH"
- show_env_vars
- [[ "$RET" -eq "0" ]] || warn "Non-zero exit caused by error ABOVE env. var. display."
-}
-trap exithandler EXIT
+req_env_vars USER HOME GOSRC SCRIPT_BASE TEST_FLAVOR TEST_ENVIRON \
+ PODBIN_NAME PRIV_NAME DISTRO_NV
# Verify basic dependencies
for depbin in go rsync unzip sha256sum curl make python3 git
do
if ! type -P "$depbin" &> /dev/null
then
- echo "***** WARNING: $depbin binary not found in $PATH *****"
+ warn "$depbin binary not found in $PATH"
fi
done
-# Sometimes environment setup needs to vary between distros
-# Note: This should only be used for environment variables, and temporary workarounds.
+# This is a possible manual maintenance gaff, check to be sure everything matches.
+# shellcheck disable=SC2154
+[[ "$DISTRO_NV" == "$OS_REL_VER" ]] || \
+ die "Automation spec. '$DISTRO_NV'; actual host '$OS_REL_VER'"
+
+# Only allow this script to execute once
+if ((${SETUP_ENVIRONMENT:-0})); then
+ # Comes from automation library
+ # shellcheck disable=SC2154
+ warn "Not executing $SCRIPT_FILENAME again"
+ exit 0
+fi
+
cd "${GOSRC}/"
-case "${OS_RELEASE_ID}" in
- ubuntu)
- ;;
- fedora)
- # All SELinux distros need this for systemd-in-a-container
- setsebool container_manage_cgroup true
- if [[ "$ADD_SECOND_PARTITION" == "true" ]]; then
- bash "$SCRIPT_BASE/add_second_partition.sh"
+# Defined by lib.sh: Does the host support cgroups v1 or v2
+case "$CG_FS_TYPE" in
+ tmpfs)
+ if ((CONTAINER==0)); then
+ warn "Forcing testing with runc instead of crun"
+ if [[ "$OS_RELEASE_ID" == "ubuntu" ]]; then
+ echo "export OCI_RUNTIME=/usr/lib/cri-o-runc/sbin/runc" >> /etc/environment
+ else
+ echo "export OCI_RUNTIME=/usr/bin/runc" >> /etc/environment
+ fi
fi
-
- warn "Forcing systemd cgroup manager"
- X=$(echo "export CGROUP_MANAGER=systemd" | \
- tee -a /etc/environment) && eval "$X" && echo "$X"
;;
- centos) # Current VM is an image-builder-image no local podman/testing
- echo "No further setup required for VM image building"
- exit 0
+ cgroup2fs)
+ if ((CONTAINER==0)); then
+ # This is necessary since we've built/installed from source,
+ # which uses runc as the default.
+ warn "Forcing testing with crun instead of runc"
+ echo "export OCI_RUNTIME=/usr/bin/crun" >> /etc/environment
+ fi
;;
- *) bad_os_id_ver ;;
+ *) die_unknown CG_FS_TYPE
esac
-# Reload to incorporate any changes from above
-source "$SCRIPT_BASE/lib.sh"
+# Required to be defined by caller: Which distribution are we testing on
+# shellcheck disable=SC2154
+case "$DISTRO_NV" in
+ ubuntu*) ;;
+ fedora*)
+ if ((CONTAINER==0)); then # Not yet running inside a container
+ msg "Configuring / Expanding host storage."
+ # VM is setup to allow flexibility in testing alternate storage.
+ # For general use, simply make use of all available space.
+ ooe.sh bash "$SCRIPT_BASE/add_second_partition.sh"
-case "$CG_FS_TYPE" in
- tmpfs)
- warn "Forcing testing with runc instead of crun"
- # On ubuntu, the default runc is usually not new enough.
- if [[ "$OS_RELEASE_ID" == "ubuntu" ]]; then
- X=$(echo "export OCI_RUNTIME=/usr/lib/cri-o-runc/sbin/runc" | \
- tee -a /etc/environment) && eval "$X" && echo "$X"
+ # All SELinux distros need this for systemd-in-a-container
+ msg "Enabling container_manage_cgroup"
+ setsebool container_manage_cgroup true
+ fi
+ ;;
+ *) die_unknown DISTRO_NV
+esac
+
+# Required to be defined by caller: The environment where primary testing happens
+# shellcheck disable=SC2154
+case "$TEST_ENVIRON" in
+ host)
+ if [[ "$OS_RELEASE_ID" == "fedora" ]]; then
+ # The e2e tests wrongly guess `--cgroup-manager cgroupfs`
+ msg "Forcing CGROUP_MANAGER=systemd"
+ _cgm="export CGROUP_MANAGER=systemd"
+ echo "$_cgm" >> /etc/environment
+ source /etc/environment
+ fi
+ ;;
+ container)
+ if ((CONTAINER==0)); then # not yet inside a container
+ msg "Force loading iptables modules"
+ # Since CRIU 3.11, uses iptables to lock and unlock
+ # the network during checkpoint and restore. Needs
+ # the following two modules loaded on the host.
+ modprobe ip6table_nat || :
+ modprobe iptable_nat || :
else
- X=$(echo "export OCI_RUNTIME=/usr/bin/runc" | \
- tee -a /etc/environment) && eval "$X" && echo "$X"
+ # The e2e tests wrongly guess `--cgroup-manager systemd`
+ msg "Forcing CGROUP_MANAGER=cgroupfs"
+ _cgm="export CGROUP_MANAGER=cgroupfs"
+ echo "$_cgm" >> /etc/environment
+ source /etc/environment
fi
;;
- cgroup2fs)
- # This is necessary since we've built/installed from source, which uses runc as the default.
- warn "Forcing testing with crun instead of runc"
- X=$(echo "export OCI_RUNTIME=/usr/bin/crun" | \
- tee -a /etc/environment) && eval "$X" && echo "$X"
+ *) die_unknown TEST_ENVIRON
+esac
+
+# Required to be defined by caller: Are we testing as root or a regular user
+# shellcheck disable=SC2154
+case "$PRIV_NAME" in
+ root)
+ if [[ "$TEST_ENVIRON" == "container" ]] && ((container)); then
+ # There's no practical way to detect userns w/in a container
+ # affected/related tests are sensitive to this variable.
+ _suns='export SKIP_USERNS=1'
+ echo "$_suns" >> /etc/environment
+ source /etc/environment
+ fi
;;
- *)
- die 110 "Unsure how to handle cgroup filesystem type '$CG_FS_TYPE'"
+ rootless)
+ _ru="export ROOTLESS_USER='${ROOTLESS_USER:-some${RANDOM}dude}'"
+ echo "$_ru" >> /etc/environment
+ source /etc/environment
+ setup_rootless
;;
+ *) die_unknown PRIV_NAME
esac
-# Must execute before possible setup_rootless()
-make install.tools
+# Required to be defined by caller: Are we testing podman or podman-remote client
+# shellcheck disable=SC2154
+case "$PODBIN_NAME" in
+ podman) ;;
+ remote) ;;
+ *) die_unknown PODBIN_NAME
+esac
-case "$SPECIALMODE" in
- none)
- [[ -n "$CROSS_PLATFORM" ]] || \
- remove_packaged_podman_files
+# Required to be defined by caller: The primary type of testing that will be performed
+# shellcheck disable=SC2154
+case "$TEST_FLAVOR" in
+ ext_svc) ;;
+ smoke) ;&
+ validate)
+ # For some reason, this is also needed for validation
+ make .install.pre-commit
;;
+ automation) ;;
+ altbuild)
+ # Defined in .cirrus.yml
+ # shellcheck disable=SC2154
+ if [[ "$ALT_NAME" =~ RPM ]]; then
+ bigto dnf install -y glibc-minimal-langpack rpm-build
+ fi
+ ;&
+ docker-py) ;&
+ build) make clean ;;
+ unit) ;;
+ int) ;&
+ sys) ;&
+ bindings) ;&
+ swagger) ;&
endpoint)
- remove_packaged_podman_files
- ;;
- bindings)
- remove_packaged_podman_files
- ;;
- rootless)
- # Only do this once, even if ROOTLESS_USER (somehow) changes
- if ! grep -q 'ROOTLESS_USER' /etc/environment
- then
- X=$(echo "export ROOTLESS_USER='${ROOTLESS_USER:-some${RANDOM}dude}'" | \
- tee -a /etc/environment) && eval "$X" && echo "$X"
- X=$(echo "export SPECIALMODE='${SPECIALMODE}'" | \
- tee -a /etc/environment) && eval "$X" && echo "$X"
- X=$(echo "export RCLI='${RCLI}'" | \
- tee -a /etc/environment) && eval "$X" && echo "$X"
- setup_rootless
+ # Use existing host bits when testing is to happen inside a container
+ # since this script will run again in that environment.
+ # shellcheck disable=SC2154
+ if ((CONTAINER==0)) && [[ "$TEST_ENVIRON" == "host" ]]; then
+ remove_packaged_podman_files
+ make install PREFIX=/usr ETCDIR=/etc
fi
- remove_packaged_podman_files
- ;;
- in_podman) # Assumed to be Fedora
- $SCRIPT_BASE/setup_container_environment.sh
;;
- *)
- die 111 "Unsupported \$SPECIALMODE: $SPECIALMODE"
+ vendor) make clean ;;
+ release) ;;
+ *) die_unknown TEST_FLAVOR
esac
-install_test_configs
+# Must be the very last command. Establishes successful setup.
+echo 'export SETUP_ENVIRONMENT=1' >> /etc/environment
diff --git a/contrib/cirrus/shellcheck.sh b/contrib/cirrus/shellcheck.sh
new file mode 100755
index 000000000..edf8248d3
--- /dev/null
+++ b/contrib/cirrus/shellcheck.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+
+set -eo pipefail
+
+# shellcheck source=./contrib/cirrus/lib.sh
+source $(dirname $0)/lib.sh
+
+cd $CIRRUS_WORKING_DIR
+shellcheck --color=always --format=tty \
+ --shell=bash --external-sources \
+ --enable add-default-case,avoid-nullary-conditions,check-unassigned-uppercase \
+ --exclude SC2046,SC2034,SC2090,SC2064 \
+ --wiki-link-count=0 --severity=warning \
+ $SCRIPT_BASE/*.sh
+
+echo "Shellcheck: PASS"
diff --git a/contrib/cirrus/system_test.sh b/contrib/cirrus/system_test.sh
deleted file mode 100755
index dbee078b6..000000000
--- a/contrib/cirrus/system_test.sh
+++ /dev/null
@@ -1,65 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-source $(dirname $0)/lib.sh
-
-req_env_var GOSRC SCRIPT_BASE OS_RELEASE_ID OS_RELEASE_VER CONTAINER_RUNTIME VARLINK_LOG
-
-LOCAL_OR_REMOTE=local
-if [[ "$RCLI" = "true" ]]; then
- LOCAL_OR_REMOTE=remote
-fi
-
-# Our name must be of the form xxxx_test or xxxx_test.sh, where xxxx is
-# the test suite to run; currently (2019-05) the only option is 'integration'
-# but pr2947 intends to add 'system'.
-TESTSUITE=$(expr $(basename $0) : '\(.*\)_test')
-if [[ -z $TESTSUITE ]]; then
- die 1 "Script name is not of the form xxxx_test.sh"
-fi
-
-cd "$GOSRC"
-
-case "$SPECIALMODE" in
- in_podman)
- ${CONTAINER_RUNTIME} run --rm --privileged --net=host --cgroupns=host \
- -v $GOSRC:$GOSRC:Z \
- --workdir $GOSRC \
- -e "CGROUP_MANAGER=cgroupfs" \
- -e "STORAGE_OPTIONS=--storage-driver=vfs" \
- -e "CRIO_ROOT=$GOSRC" \
- -e "PODMAN_BINARY=/usr/bin/podman" \
- -e "CONMON_BINARY=/usr/libexec/podman/conmon" \
- -e "DIST=$OS_RELEASE_ID" \
- -e "CONTAINER_RUNTIME=$CONTAINER_RUNTIME" \
- $IN_PODMAN_IMAGE bash $GOSRC/$SCRIPT_BASE/container_test.sh -b -i -t
- ;;
- rootless)
- req_env_var ROOTLESS_USER
- ssh $ROOTLESS_USER@localhost \
- -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \
- -o CheckHostIP=no $GOSRC/$SCRIPT_BASE/rootless_test.sh ${TESTSUITE} ${LOCAL_OR_REMOTE}
- ;;
- endpoint)
- make
- make install PREFIX=/usr ETCDIR=/etc
- make test-binaries
- make endpoint
- ;;
- bindings)
- make
- make install PREFIX=/usr ETCDIR=/etc
- export PATH=$PATH:`pwd`/hack
- cd pkg/bindings/test && ginkgo -trace -noColor -debug -r
- ;;
- none)
- make
- make install PREFIX=/usr ETCDIR=/etc
- make test-binaries
- make .install.bats
- make ${LOCAL_OR_REMOTE}${TESTSUITE} PODMAN_SERVER_LOG=$PODMAN_SERVER_LOG
- ;;
- *)
- die 110 "Unsupported \$SPECIALMODE: $SPECIALMODE"
-esac
diff --git a/contrib/cirrus/unit_test.sh b/contrib/cirrus/unit_test.sh
deleted file mode 100755
index 17a618a1c..000000000
--- a/contrib/cirrus/unit_test.sh
+++ /dev/null
@@ -1,22 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-source $(dirname $0)/lib.sh
-
-req_env_var GOSRC
-
-cd "$GOSRC"
-make install.tools
-make localunit
-
-case "$SPECIALMODE" in
- in_podman) ;&
- bindings) ;&
- rootless) ;&
- none)
- make
- ;;
- *)
- die 109 "Unsupported \$SPECIAL_MODE: $SPECIALMODE"
-esac
diff --git a/contrib/cirrus/update_meta.sh b/contrib/cirrus/update_meta.sh
deleted file mode 100755
index 6e4a473e9..000000000
--- a/contrib/cirrus/update_meta.sh
+++ /dev/null
@@ -1,22 +0,0 @@
-#!/usr/bin/env bash
-
-source $(dirname $0)/lib.sh
-
-# N/B: This script is expected to wrap $ENTRYPOINT when executing under the
-# 'meta' Cirrus task on the libpod repo.
-ENTRYPOINT=/usr/local/bin/entrypoint.sh
-
-req_env_var IMGNAMES BUILDID REPOREF GCPJSON GCPNAME GCPPROJECT CIRRUS_CI
-
-[[ -x "$ENTRYPOINT" ]] || \
- die 2 "Expecting to find an installed entrypoint script $ENTRYPOINT."
-
-# A better way of checking isn't compatible with old but functional images
-# in-use by other repos.
-grep -q 'compute images update' "$ENTRYPOINT" || \
- die 3 "Expecting to be running inside a specific imgts container image"
-
-canonicalize_image_names
-
-# Executing inside a container; proper hand-off for process control
-exec $ENTRYPOINT
diff --git a/contrib/cirrus/upload_release_archive.sh b/contrib/cirrus/upload_release_archive.sh
deleted file mode 100755
index 2e2f4ddde..000000000
--- a/contrib/cirrus/upload_release_archive.sh
+++ /dev/null
@@ -1,90 +0,0 @@
-#!/usr/bin/env bash
-
-set -eo pipefail
-
-source $(dirname $0)/lib.sh
-
-req_env_var CI UPLDREL_IMAGE CIRRUS_BUILD_ID GOSRC RELEASE_GCPJSON RELEASE_GCPNAME RELEASE_GCPROJECT
-
-[[ "$CI" == "true" ]] || \
- die 56 "$0 must be run under Cirrus-CI to function"
-
-SWAGGER_FILEPATH="pkg/api/swagger.yaml"
-
-# We store "releases" for each PR, mostly to validate the process is functional
-unset PR_OR_BRANCH BUCKET
-if [[ -n "$CIRRUS_PR" ]]
-then
- PR_OR_BRANCH="pr$CIRRUS_PR"
- BUCKET="libpod-pr-releases"
-elif [[ -n "$CIRRUS_BRANCH" ]]
-then
- # Only release binaries for docs
- if [[ $CIRRUS_TASK_NAME =~ "docs" ]]
- then
- PR_OR_BRANCH="$CIRRUS_BRANCH"
- BUCKET="libpod-$CIRRUS_BRANCH-releases"
- else
- warn "" "Skipping release processing for non-docs task."
- exit 0
- fi
-else
- die 1 "Expecting either \$CIRRUS_PR or \$CIRRUS_BRANCH to be non-empty."
-fi
-
-# Functional local podman required for uploading
-echo "Verifying a local, functional podman, building one if necessary."
-[[ -n "$(type -P podman)" ]] || \
- make install PREFIX=/usr || \
- die 57 "$0 requires working podman binary on path to function"
-
-TMPF=$(mktemp -p '' $(basename $0)_XXXX.json)
-trap "rm -f $TMPF" EXIT
-set +x
-echo "$RELEASE_GCPJSON" > "$TMPF"
-[[ "$OS_RELEASE_ID" == "ubuntu" ]] || \
- chcon -t container_file_t "$TMPF"
-unset RELEASE_GCPJSON
-
-cd $GOSRC
-for filename in $(ls -1 $SWAGGER_FILEPATH)
-do
- unset EXT
- EXT=$(echo "$filename" | sed -r -e 's/.+\.(.+$)/\1/g')
- if [[ -z "$EXT" ]] || [[ "$EXT" == "$filename" ]]
- then
- echo "Warning: Not processing $filename (invalid extension '$EXT')"
- continue
- fi
- if [[ "$EXT" =~ "gz" ]]
- then
- EXT="tar.gz"
- fi
-
- if [[ $filename == $SWAGGER_FILEPATH ]]
- then
- # Support other tools referencing branch and/or version-specific refs.
- TO_FILENAME="swagger-${RELEASE_VERSION}-${PR_OR_BRANCH}.yaml"
- # For doc. ref. this must always be a static filename, e.g. swagger-latest-master.yaml
- ALSO_FILENAME="swagger-latest-${PR_OR_BRANCH}.yaml"
- else
- die "Uploading non-docs files has been disabled"
- fi
-
- [[ "$OS_RELEASE_ID" == "ubuntu" ]] || \
- chcon -t container_file_t "$filename"
-
- echo "Running podman ... $UPLDREL_IMAGE for $filename -> $TO_FILENAME"
- podman run -i --rm \
- -e "GCPNAME=$RELEASE_GCPNAME" \
- -e "GCPPROJECT=$RELEASE_GCPROJECT" \
- -e "GCPJSON_FILEPATH=$TMPF" \
- -e "FROM_FILEPATH=/tmp/$filename" \
- -e "TO_FILENAME=$TO_FILENAME" \
- -e "ALSO_FILENAME=$ALSO_FILENAME" \
- -e "PR_OR_BRANCH=$PR_OR_BRANCH" \
- -e "BUCKET=$BUCKET" \
- -v "$TMPF:$TMPF:ro" \
- -v "$(realpath $GOSRC/$filename):/tmp/$filename:ro" \
- $UPLDREL_IMAGE
-done