From efd1422143c738ae268e4e86da05adb597c84083 Mon Sep 17 00:00:00 2001 From: Brent Baude Date: Wed, 27 May 2020 14:24:53 -0500 Subject: Enable Ubuntu tests in CI Add updates required for ubuntu and run integration tests Signed-off-by: Brent Baude --- contrib/cirrus/setup_environment.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'contrib') diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh index 25b7ff941..4066b813b 100755 --- a/contrib/cirrus/setup_environment.sh +++ b/contrib/cirrus/setup_environment.sh @@ -39,6 +39,17 @@ done cd "${GOSRC}/" case "${OS_RELEASE_ID}" in ubuntu) + apt-get update + apt-get install -y containers-common + sed -ie 's/^\(# \)\?apparmor_profile =.*/apparmor_profile = ""/' /etc/containers/containers.conf + if [[ "$OS_RELEASE_VER" == "19" ]]; then + apt-get purge -y --auto-remove golang* + apt-get install -y golang-1.13 + ln -s /usr/lib/go-1.13/bin/go /usr/bin/go + fi + if [[ "$OS_RELEASE_VER" == "20" ]]; then + apt-get install -y python-is-python3 + fi ;; fedora) # All SELinux distros need this for systemd-in-a-container -- cgit v1.2.3-54-g00ecf From b6148b657661db2b9d7858102e9aa3cf6f19f47f Mon Sep 17 00:00:00 2001 From: Valentin Rothberg Date: Mon, 8 Jun 2020 12:33:35 +0200 Subject: force bats version to v1.1.0 We experienced regression when using the latest `v1.2.0-dev` bats in Ubuntu 20.04 (see github.com/containers/libpod/pull/6418). Using bats v1.1.0 worked in the Ubuntu test VM. Signed-off-by: Valentin Rothberg --- Makefile | 6 +++++- contrib/cirrus/integration_test.sh | 1 + hack/install_bats.sh | 16 ++++++++++++++++ 3 files changed, 22 insertions(+), 1 deletion(-) create mode 100755 hack/install_bats.sh (limited to 'contrib') diff --git a/Makefile b/Makefile index 1d30d2534..67de05f57 100644 --- a/Makefile +++ b/Makefile @@ -612,7 +612,7 @@ uninstall: GIT_CHECK_EXCLUDE="./vendor:docs/make.bat" $(GOBIN)/git-validation -run DCO,short-subject,dangling-whitespace -range $(EPOCH_TEST_COMMIT)..$(HEAD) .PHONY: install.tools -install.tools: .install.gitvalidation .install.md2man .install.ginkgo .install.golangci-lint ## Install needed tools +install.tools: .install.gitvalidation .install.md2man .install.ginkgo .install.golangci-lint .install.bats ## Install needed tools define go-get env GO111MODULE=off \ @@ -635,6 +635,10 @@ endef .install.golangci-lint: .gopathok VERSION=1.18.0 GOBIN=$(GOBIN) sh ./hack/install_golangci.sh +.PHONY: .install.bats +.install.bats: .gopathok + VERSION=v1.1.0 ./hack/install_bats.sh + .PHONY: .install.pre-commit .install.pre-commit: if [ -z "$(PRE_COMMIT)" ]; then \ diff --git a/contrib/cirrus/integration_test.sh b/contrib/cirrus/integration_test.sh index 1aef678d4..c80367454 100755 --- a/contrib/cirrus/integration_test.sh +++ b/contrib/cirrus/integration_test.sh @@ -52,6 +52,7 @@ case "$SPECIALMODE" in make make install PREFIX=/usr ETCDIR=/etc make test-binaries + make .install.bats if [[ "$TEST_REMOTE_CLIENT" == "true" ]] then make remote${TESTSUITE} VARLINK_LOG=$VARLINK_LOG diff --git a/hack/install_bats.sh b/hack/install_bats.sh new file mode 100755 index 000000000..00ded07a9 --- /dev/null +++ b/hack/install_bats.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -e + +die() { echo "${1:-No error message given} (from $(basename $0))"; exit 1; } + +buildDir=$(mktemp -d) +git clone https://github.com/bats-core/bats-core $buildDir + +pushd $buildDir +pwd +git reset --hard ${VERSION} +./install.sh /usr/local +popd + +rm -rf $buildDir -- cgit v1.2.3-54-g00ecf From 4314336ce346efb3f9aa128a3dd9680603cffaf4 Mon Sep 17 00:00:00 2001 From: Chris Evich Date: Mon, 8 Jun 2020 13:45:26 -0400 Subject: Improve swagger+CORS metadata docs Signed-off-by: Chris Evich --- contrib/cirrus/README.md | 62 +++++++++++++++++++++++++-------- contrib/cirrus/swagger_stack_trace.png | Bin 0 -> 42799 bytes docs/Readme.md | 30 ++++++++++++---- 3 files changed, 70 insertions(+), 22 deletions(-) create mode 100644 contrib/cirrus/swagger_stack_trace.png (limited to 'contrib') diff --git a/contrib/cirrus/README.md b/contrib/cirrus/README.md index 541cf2f54..c8ec766e7 100644 --- a/contrib/cirrus/README.md +++ b/contrib/cirrus/README.md @@ -167,26 +167,50 @@ env: ### `docs` Task -Builds swagger API documentation YAML and uploads to google storage for both -PR's (for testing the process) and after a merge into any branch. For PR's +Builds swagger API documentation YAML and uploads to google storage (an online +service for storing unstructured data) for both +PR's (for testing the process) and the master branch. For PR's the YAML is uploaded into a [dedicated short-pruning cycle -bucket.](https://storage.googleapis.com/libpod-pr-releases/) For branches, -a [separate bucket is -used.](https://storage.googleapis.com/libpod-master-releases) -In both cases the filename includes the source -PR number or branch name. - -***Note***: [The online documentation](http://docs.podman.io/en/latest/_static/api.html) -is presented through javascript on the client-side. This requires CORS to be properly -configured on the bucket, for the `http://docs.podman.io` origin. Please see -[Configuring CORS on a bucket](https://cloud.google.com/storage/docs/configuring-cors#configure-cors-bucket) -for details. This may be performed by anybody with admin access to the google storage bucket, -using the following JSON: +bucket.](https://storage.googleapis.com/libpod-pr-releases/) for testing purposes +only. For the master branch, a [separate bucket is +used](https://storage.googleapis.com/libpod-master-releases) and provides the +content rendered on [the API Reference page](https://docs.podman.io/en/latest/_static/api.html) + +The online API reference is presented by javascript to the client. To prevent hijacking +of the client by malicious data, the [javascript utilises CORS](https://cloud.google.com/storage/docs/cross-origin). +This CORS metadata is served by `https://storage.googleapis.com` when configured correctly. +It will appear in [the request and response headers from the +client](https://cloud.google.com/storage/docs/configuring-cors#troubleshooting) when accessing +the API reference page. + +However, when the CORS metadata is missing or incorrectly configured, clients will receive an +error-message similar to: + +![Javascript Stack Trace Image](swagger_stack_trace.png) + +For documentation built by Read The Docs from the master branch, CORS metadata is +set on the `libpod-master-releases` storage bucket. Viewing or setting the CORS +metadata on the bucket requires having locally [installed and +configured the google-cloud SDK](https://cloud.google.com/sdk/docs). It also requires having +admin access to the google-storage bucket. Contact a project owner for help if you are +unsure of your permissions or need help resolving an error similar to the picture above. + +Assuming the SDK is installed, and you have the required admin access, the following command +will display the current CORS metadata: + +``` +gsutil cors get gs://libpod-master-releases +``` + +To function properly (allow client "trust" of content from `storage.googleapis.com`) the followiing +metadata JSON should be used. Following the JSON, is an example of the command used to set this +metadata on the libpod-master-releases bucket. For additional information about configuring CORS +please referr to [the google-storage documentation](https://cloud.google.com/storage/docs/configuring-cors). ```JSON [ { - "origin": ["http://docs.podman.io"], + "origin": ["http://docs.podman.io", "https://docs.podman.io"], "responseHeader": ["Content-Type"], "method": ["GET"], "maxAgeSeconds": 600 @@ -194,6 +218,14 @@ using the following JSON: ] ``` +``` +gsutil cors set /path/to/file.json gs://libpod-master-releases +``` + +***Note:*** The CORS metadata does _NOT_ change after the `docs` task uploads a new swagger YAML +file. Therefore, if it is not functioning or misconfigured, a person must have altered it or +changes were made to the referring site (e.g. `docs.podman.io`). + ## Base-images Base-images are VM disk-images specially prepared for executing as GCE VMs. diff --git a/contrib/cirrus/swagger_stack_trace.png b/contrib/cirrus/swagger_stack_trace.png new file mode 100644 index 000000000..6aa063bab Binary files /dev/null and b/contrib/cirrus/swagger_stack_trace.png differ diff --git a/docs/Readme.md b/docs/Readme.md index 987a5b8e4..9d3b9d06f 100644 --- a/docs/Readme.md +++ b/docs/Readme.md @@ -30,10 +30,26 @@ link on that page. ## API Reference The [latest online documentation](http://docs.podman.io/en/latest/_static/api.html) is -automatically generated from committed upstream sources. There is a short-duration -cache involved, in case old content or an error is returned, try clearing your browser -cache or returning to the site after 10-30 minutes. - -***Maintainers Note***: Please refer to [the Cirrus-CI tasks -documentation](../contrib/cirrus/README.md#docs-task) for -important operational details. +automatically generated by two cooperating automation systems based on committed upstream +source code. Firstly, [the Cirrus-CI docs task](../contrib/cirrus/README.md#docs-task) builds +`pkg/api/swagger.yaml` and uploads it to a public-facing location (Google Storage Bucket - +an online service for storing unstructured data). Second, [Read The Docs](readthedocs.com) +reacts to the github.com repository change, building the content for the [libpod documentation +site](https://podman.readthedocs.io/). This site includes for the API section, +some javascript which consumes the uploaded `swagger.yaml` file directly from the Google +Storage Bucket. + +Since there are multiple systems and local cache is involved, it's possible that updates to +documentation (especially the swagger/API docs) will lag by 10-or-so minutes. However, +because the client (i.e. your web browser) is fetching content from multiple locations that +do not share a common domain, accessing the API section may show a stack-trace similar to +the following: + +![Javascript Stack Trace Image](../contrib/cirrus/swagger_stack_trace.png) + +If reloading the page, or clearing your local cache does not fix the problem, it is +likely caused by broken metadata needed to protect clients from cross-site-scripting +style attacks. Please [notify a maintainer](https://github.com/containers/libpod#communications) +so they may investigate how/why the swagger.yaml file's CORS-metadata is incorrect. See +[the Cirrus-CI tasks documentation](../contrib/cirrus/README.md#docs-task) for +details regarding this situation. -- cgit v1.2.3-54-g00ecf From 15f273b9310399ba048f667dba40f553b2fb60cb Mon Sep 17 00:00:00 2001 From: Ed Santiago Date: Mon, 8 Jun 2020 14:57:37 -0600 Subject: WIP: Enable (and disable) remote testing podman-remote has not been tested. A principal part of the problem was #5387 - the YAML I wrote did not have the intended effect, it did not set TEST_REMOTE_CLIENT=true and because of my multiple iterations I did not catch this during testing. Part 1 of this PR is to fix .cirrus.yml to enable remote tests. Part 2 -- what I had first noticed and tried to fix -- is that rootless_test.sh was never running remote because, of course, envariables are not sent via ssh. I reworked integration_test.sh and rootless_test.sh to use a command-line decision instead. Part 3, sigh, is to disable one failing integration test and *all* system tests, because so many of the latter are failing. Addressing those failures needs to be done in subsequent PRs. Issues #6538, #6539, #6540 are filed for some of the problems I isolated. There will be more. Also, minor, fixed some stale references to varlink. Signed-off-by: Ed Santiago --- .cirrus.yml | 24 ++++++------------------ Makefile | 22 ++++++++++++++-------- contrib/cirrus/integration_test.sh | 14 +++++++------- contrib/cirrus/rootless_test.sh | 22 +++------------------- test/e2e/cp_test.go | 2 ++ 5 files changed, 32 insertions(+), 52 deletions(-) (limited to 'contrib') diff --git a/.cirrus.yml b/.cirrus.yml index 5b9dbdab8..919905c67 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -434,12 +434,8 @@ testing_task: env: ADD_SECOND_PARTITION: 'true' matrix: - - name: remote - env: - TEST_REMOTE_CLIENT: 'true' - - name: local - env: - TEST_REMOTE_CLIENT: 'false' + - TEST_REMOTE_CLIENT: 'true' + - TEST_REMOTE_CLIENT: 'false' networking_script: '${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/networking.sh' setup_environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}' @@ -486,12 +482,8 @@ special_testing_rootless_task: ADD_SECOND_PARTITION: 'true' SPECIALMODE: 'rootless' # See docs matrix: - - name: remote - env: - TEST_REMOTE_CLIENT: 'true' - - name: local - env: - TEST_REMOTE_CLIENT: 'false' + - TEST_REMOTE_CLIENT: 'true' + - TEST_REMOTE_CLIENT: 'false' timeout_in: 60m @@ -693,12 +685,8 @@ verify_test_built_images_task: env: ADD_SECOND_PARTITION: 'true' matrix: - - name: remote - env: - TEST_REMOTE_CLIENT: 'true' - - name: local - env: - TEST_REMOTE_CLIENT: 'false' + - TEST_REMOTE_CLIENT: 'true' + - TEST_REMOTE_CLIENT: 'false' matrix: # Required env. var. by check_image_script PACKER_BUILDER_NAME: "${FEDORA_NAME}" diff --git a/Makefile b/Makefile index 67de05f57..2ac6f426f 100644 --- a/Makefile +++ b/Makefile @@ -88,8 +88,8 @@ RELEASE_DIST_VER ?= $(shell hack/get_release_info.sh DIST_VER) RELEASE_ARCH ?= $(shell hack/get_release_info.sh ARCH) RELEASE_BASENAME := $(shell hack/get_release_info.sh BASENAME) -# If non-empty, logs all output from varlink during remote system testing -VARLINK_LOG ?= +# If non-empty, logs all output from server during remote system testing +PODMAN_SERVER_LOG ?= # If GOPATH not specified, use one in the local directory ifeq ($(GOPATH),) @@ -357,22 +357,28 @@ localsystem: remotesystem: # Wipe existing config, database, and cache: start with clean slate. $(RM) -rf ${HOME}/.local/share/containers ${HOME}/.config/containers - # Start varlink server using tmp socket; loop-wait for it; + # Start podman server using tmp socket; loop-wait for it; # test podman-remote; kill server, clean up tmp socket file. - # varlink server spews copious unhelpful output; ignore it. + # podman server spews copious unhelpful output; ignore it. + # FIXME FIXME FIXME: remove 'exit 0' after #6538 and #6539 are fixed + exit 0;\ rc=0;\ if timeout -v 1 true; then \ SOCK_FILE=$(shell mktemp --dry-run --tmpdir podman.XXXXXX);\ - export PODMAN_SOCKEY=unix:$$SOCK_FILE; \ - ./bin/podman system service --timeout=0 $$PODMAN_VARLINK_ADDRESS &> $(if $(VARLINK_LOG),$(VARLINK_LOG),/dev/null) & \ + export PODMAN_SOCKET=unix:$$SOCK_FILE; \ + ./bin/podman system service --timeout=0 $$PODMAN_SOCKET &> $(if $(PODMAN_SERVER_LOG),$(PODMAN_SERVER_LOG),/dev/null) & \ retry=5;\ while [[ $$retry -ge 0 ]]; do\ echo Waiting for server...;\ sleep 1;\ - ./bin/podman-remote --remote $(SOCK_FILE) info &>/dev/null && break;\ + ./bin/podman-remote --url $$PODMAN_SOCKET info &>/dev/null && break;\ retry=$$(expr $$retry - 1);\ done;\ - env PODMAN=./bin/podman-remote bats test/system/ ;\ + if [[ $$retry -lt 0 ]]; then\ + echo "Error: ./bin/podman system service did not come up on $$SOCK_FILE" >&2;\ + exit 1;\ + fi;\ + env PODMAN="./bin/podman-remote --url $$PODMAN_SOCKET" bats test/system/ ;\ rc=$$?;\ kill %1;\ rm -f $$SOCK_FILE;\ diff --git a/contrib/cirrus/integration_test.sh b/contrib/cirrus/integration_test.sh index c80367454..33e9fbc6b 100755 --- a/contrib/cirrus/integration_test.sh +++ b/contrib/cirrus/integration_test.sh @@ -6,6 +6,11 @@ source $(dirname $0)/lib.sh req_env_var GOSRC SCRIPT_BASE OS_RELEASE_ID OS_RELEASE_VER CONTAINER_RUNTIME VARLINK_LOG +LOCAL_OR_REMOTE=local +if [[ "$TEST_REMOTE_CLIENT" = "true" ]]; then + LOCAL_OR_REMOTE=remote +fi + # Our name must be of the form xxxx_test or xxxx_test.sh, where xxxx is # the test suite to run; currently (2019-05) the only option is 'integration' # but pr2947 intends to add 'system'. @@ -34,7 +39,7 @@ case "$SPECIALMODE" in req_env_var ROOTLESS_USER ssh $ROOTLESS_USER@localhost \ -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ - -o CheckHostIP=no $GOSRC/$SCRIPT_BASE/rootless_test.sh ${TESTSUITE} + -o CheckHostIP=no $GOSRC/$SCRIPT_BASE/rootless_test.sh ${TESTSUITE} ${LOCAL_OR_REMOTE} ;; endpoint) make @@ -53,12 +58,7 @@ case "$SPECIALMODE" in make install PREFIX=/usr ETCDIR=/etc make test-binaries make .install.bats - if [[ "$TEST_REMOTE_CLIENT" == "true" ]] - then - make remote${TESTSUITE} VARLINK_LOG=$VARLINK_LOG - else - make local${TESTSUITE} - fi + make ${LOCAL_OR_REMOTE}${TESTSUITE} PODMAN_SERVER_LOG=$PODMAN_SERVER_LOG ;; *) die 110 "Unsupported \$SPECIALMODE: $SPECIALMODE" diff --git a/contrib/cirrus/rootless_test.sh b/contrib/cirrus/rootless_test.sh index 3f45aac84..9e1b1d911 100755 --- a/contrib/cirrus/rootless_test.sh +++ b/contrib/cirrus/rootless_test.sh @@ -2,14 +2,6 @@ set -e -remote=0 - -# The TEST_REMOTE_CLIENT environment variable decides whether -# to test varlink -if [[ "$TEST_REMOTE_CLIENT" == "true" ]]; then - remote=1 -fi - source $(dirname $0)/lib.sh if [[ "$UID" == "0" ]] @@ -18,11 +10,8 @@ then exit 1 fi -# Which set of tests to run; possible alternative is "system" -TESTSUITE=integration -if [[ -n "$*" ]]; then - TESTSUITE="$1" -fi +TESTSUITE=${1?Missing TESTSUITE argument (arg1)} +LOCAL_OR_REMOTE=${2?Missing LOCAL_OR_REMOTE argument (arg2)} # Ensure environment setup correctly req_env_var GOSRC ROOTLESS_USER @@ -31,7 +20,6 @@ echo "." echo "Hello, my name is $USER and I live in $PWD can I be your friend?" echo "." -export PODMAN_VARLINK_ADDRESS=unix:/tmp/podman-$(id -u) show_env_vars set -x @@ -39,8 +27,4 @@ cd "$GOSRC" make make varlink_generate make test-binaries -if [ $remote -eq 0 ]; then - make local${TESTSUITE} -else - make remote${TESTSUITE} -fi +make ${LOCAL_OR_REMOTE}${TESTSUITE} diff --git a/test/e2e/cp_test.go b/test/e2e/cp_test.go index f95f8646c..6ae54ba34 100644 --- a/test/e2e/cp_test.go +++ b/test/e2e/cp_test.go @@ -141,6 +141,8 @@ var _ = Describe("Podman cp", func() { }) It("podman cp stdin/stdout", func() { + SkipIfRemote() + Skip("Looks like SkipIfRemote() is not working") session := podmanTest.Podman([]string{"create", ALPINE, "ls", "foo"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) -- cgit v1.2.3-54-g00ecf From 455a94d1f932cd32dc07d9bf62131423610f8e2b Mon Sep 17 00:00:00 2001 From: Chris Evich Date: Tue, 12 May 2020 12:23:35 -0400 Subject: Cirrus: Include packages for containers/conmon CI This allows the containers/conmon repository to share the same VM images produced by containers/libpod. Included are several packages which are downloaded only since they might otherwise interfere with testing for some repos. This allows stable versions to be at the ready at testing runtime, avoiding any version updates surprising developers. Also, re-enable running the VM-image check test which was not working due to a logic problem in Cirrus-CI configuration. Update the neglected tests so that they pass on all distros. Signed-off-by: Chris Evich --- .cirrus.yml | 21 ++++---- contrib/cirrus/check_image.sh | 42 +++++++++------ contrib/cirrus/lib.sh | 13 ++++- contrib/cirrus/packer/fedora_packaging.sh | 85 +++++++++++++++++++++---------- contrib/cirrus/packer/fedora_setup.sh | 8 +-- contrib/cirrus/packer/ubuntu_packaging.sh | 31 ++++++----- contrib/cirrus/setup_environment.sh | 8 --- 7 files changed, 129 insertions(+), 79 deletions(-) (limited to 'contrib') diff --git a/.cirrus.yml b/.cirrus.yml index 58c7cd871..0f3971dd2 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -39,7 +39,7 @@ env: UBUNTU_NAME: "ubuntu-20" PRIOR_UBUNTU_NAME: "ubuntu-19" - _BUILT_IMAGE_SUFFIX: "libpod-6268069335007232" # From the packer output of 'build_vm_images_script' + _BUILT_IMAGE_SUFFIX: "libpod-6508632441356288" FEDORA_CACHE_IMAGE_NAME: "${FEDORA_NAME}-${_BUILT_IMAGE_SUFFIX}" PRIOR_FEDORA_CACHE_IMAGE_NAME: "${PRIOR_FEDORA_NAME}-${_BUILT_IMAGE_SUFFIX}" UBUNTU_CACHE_IMAGE_NAME: "${UBUNTU_NAME}-${_BUILT_IMAGE_SUFFIX}" @@ -72,10 +72,6 @@ env: GCE_SSH_USERNAME: cirrus-ci # Name where this repositories cloud resources are located GCP_PROJECT_ID: ENCRYPTED[7c80e728e046b1c76147afd156a32c1c57d4a1ac1eab93b7e68e718c61ca8564fc61fef815952b8ae0a64e7034b8fe4f] - RELEASE_GCPJSON: ENCRYPTED[789d8f7e9a5972ce350fd8e60f1032ccbf4a35c3938b604774b711aad280e12c21faf10e25af1e0ba33597ffb9e39e46] - RELEASE_GCPNAME: ENCRYPTED[417d50488a4bd197bcc925ba6574de5823b97e68db1a17e3a5fde4bcf26576987345e75f8d9ea1c15a156b4612c072a1] - RELEASE_GCPROJECT: ENCRYPTED[7c80e728e046b1c76147afd156a32c1c57d4a1ac1eab93b7e68e718c61ca8564fc61fef815952b8ae0a64e7034b8fe4f] - # Default VM to use unless set or modified by task @@ -601,6 +597,7 @@ special_testing_bindings_task: env: SPECIALMODE: 'bindings' # See docs + ADD_SECOND_PARTITION: 'true' # More root fs space is required timeout_in: 40m @@ -701,20 +698,16 @@ verify_test_built_images_task: env: TEST_REMOTE_CLIENT: 'false' matrix: - # Required env. var. by check_image_script PACKER_BUILDER_NAME: "${FEDORA_NAME}" PACKER_BUILDER_NAME: "${PRIOR_FEDORA_NAME}" + PACKER_BUILDER_NAME: "${UBUNTU_NAME}" PACKER_BUILDER_NAME: "${PRIOR_UBUNTU_NAME}" - # Multiple test failures on ${UBUNTU_CACHE_IMAGE_NAME} - # PACKER_BUILDER_NAME: "${UBUNTU_NAME}" networking_script: '${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/networking.sh' installed_packages_script: '$SCRIPT_BASE/logcollector.sh packages' environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}' - # Verify expectations once per image - check_image_script: >- - [[ "$TEST_REMOTE_CLIENT" == "false" ]] || \ - $SCRIPT_BASE/check_image.sh |& ${TIMESTAMP} + # Verify expectations of built images + check_image_script: '$SCRIPT_BASE/check_image.sh |& ${TIMESTAMP}' # Note: A truncated form of normal testing. It only needs to confirm new images # "probably" work. A full round of testing will happen again after $*_CACHE_IMAGE_NAME # are updated in this or another PR (w/o '***CIRRUS: TEST IMAGES***'). @@ -732,6 +725,10 @@ docs_task: depends_on: - "gating" + env: + RELEASE_GCPJSON: ENCRYPTED[789d8f7e9a5972ce350fd8e60f1032ccbf4a35c3938b604774b711aad280e12c21faf10e25af1e0ba33597ffb9e39e46] + RELEASE_GCPNAME: ENCRYPTED[417d50488a4bd197bcc925ba6574de5823b97e68db1a17e3a5fde4bcf26576987345e75f8d9ea1c15a156b4612c072a1] + RELEASE_GCPROJECT: ENCRYPTED[7c80e728e046b1c76147afd156a32c1c57d4a1ac1eab93b7e68e718c61ca8564fc61fef815952b8ae0a64e7034b8fe4f] script: - "$SCRIPT_BASE/build_swagger.sh |& ${TIMESTAMP}" diff --git a/contrib/cirrus/check_image.sh b/contrib/cirrus/check_image.sh index 5423f67d6..0d33e55bf 100755 --- a/contrib/cirrus/check_image.sh +++ b/contrib/cirrus/check_image.sh @@ -6,7 +6,7 @@ source $(dirname $0)/lib.sh EVIL_UNITS="$($CIRRUS_WORKING_DIR/$PACKER_BASE/systemd_banish.sh --list)" -req_env_var PACKER_BUILDER_NAME TEST_REMOTE_CLIENT EVIL_UNITS OS_RELEASE_ID +req_env_var PACKER_BUILDER_NAME TEST_REMOTE_CLIENT EVIL_UNITS OS_RELEASE_ID CG_FS_TYPE NFAILS=0 echo "Validating VM image" @@ -22,7 +22,8 @@ item_test 'Minimum available memory' $MEM_FREE -ge $MIN_MEM_MB || let "NFAILS+=1 # We're testing a custom-built podman; make sure there isn't a distro-provided # binary anywhere; that could potentially taint our results. -item_test "remove_packaged_podman_files() did it's job" -z "$(type -P podman)" || let "NFAILS+=1" +remove_packaged_podman_files +item_test "remove_packaged_podman_files() does it's job" -z "$(type -P podman)" || let "NFAILS+=1" # Integration Tests require varlink in Fedora item_test "The varlink executable is present" -x "$(type -P varlink)" || let "NFAILS+=1" @@ -39,8 +40,10 @@ for REQ_UNIT in google-accounts-daemon.service \ google-shutdown-scripts.service \ google-startup-scripts.service do - item_test "required $REQ_UNIT enabled" \ - "$(systemctl list-unit-files --no-legend $REQ_UNIT)" = "$REQ_UNIT enabled" || let "NFAILS+=1" + # enabled/disabled appears at the end of the line, on some Ubuntu's it appears twice + service_status=$(systemctl list-unit-files --no-legend $REQ_UNIT | tac -s ' ' | head -1) + item_test "required $REQ_UNIT status is enabled" \ + "$service_status" = "enabled" || let "NFAILS+=1" done for evil_unit in $EVIL_UNITS @@ -50,19 +53,28 @@ do item_test "No $evil_unit unit is present or active:" "$unit_status" -ne "0" || let "NFAILS+=1" done -if [[ "$OS_RELEASE_ID" == "ubuntu" ]] && [[ -x "/usr/lib/cri-o-runc/sbin/runc" ]] -then - SAMESAME=$(diff --brief /usr/lib/cri-o-runc/sbin/runc /usr/bin/runc &> /dev/null; echo $?) - item_test "On ubuntu /usr/bin/runc is /usr/lib/cri-o-runc/sbin/runc" "$SAMESAME" -eq "0" || let "NFAILS+=1" -fi - -if [[ "$OS_RELEASE_ID" == "ubuntu" ]] -then - item_test "On ubuntu, no periodic apt crap is enabled" -z "$(egrep $PERIODIC_APT_RE /etc/apt/apt.conf.d/*)" -fi - echo "Checking items specific to ${PACKER_BUILDER_NAME}${BUILT_IMAGE_SUFFIX}" case "$PACKER_BUILDER_NAME" in + ubuntu*) + item_test "On ubuntu, no periodic apt crap is enabled" -z "$(egrep $PERIODIC_APT_RE /etc/apt/apt.conf.d/*)" + ;; + fedora*) + # Only runc -OR- crun should be installed, never both + case "$CG_FS_TYPE" in + tmpfs) + HAS=runc + HAS_NOT=crun + ;; + cgroup2fs) + HAS=crun + HAS_NOT=runc + ;; + esac + HAS_RC=$(rpm -qV $HAS &> /dev/null; echo $?) + HAS_NOT_RC=$(rpm -qV $HAS_NOT &> /dev/null; echo $?) + item_test "With a cgroups-fs type $CG_FS_TYPE, the $HAS package is installed" $HAS_RC -eq 0 + item_test "With a cgroups-fs type $CG_FS_TYPE, the $HAS_NOT package is not installed" $HAS_NOT_RC -ne 0 + ;; xfedora*) echo "Kernel Command-line: $(cat /proc/cmdline)" item_test \ diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh index cc5a3ffa7..66e8060cf 100644 --- a/contrib/cirrus/lib.sh +++ b/contrib/cirrus/lib.sh @@ -39,6 +39,8 @@ PACKER_BASE=${PACKER_BASE:-./contrib/cirrus/packer} # Important filepaths SETUP_MARKER_FILEPATH="${SETUP_MARKER_FILEPATH:-/var/tmp/.setup_environment_sh_complete}" AUTHOR_NICKS_FILEPATH="${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/git_authors_to_irc_nicks.csv" +# Downloaded, but not installed packages. +PACKAGE_DOWNLOAD_DIR=/var/cache/download # Log remote-client system test varlink output here export VARLINK_LOG=/var/tmp/varlink.log @@ -422,7 +424,7 @@ remove_packaged_podman_files() { then LISTING_CMD="$SUDO dpkg-query -L podman" else - LISTING_CMD='$SUDO rpm -ql podman' + LISTING_CMD="$SUDO rpm -ql podman" fi # yum/dnf/dpkg may list system directories, only remove files @@ -437,6 +439,14 @@ remove_packaged_podman_files() { sync && echo 3 > /proc/sys/vm/drop_caches } +# The version of CRI-O and Kubernetes must always match +get_kubernetes_version(){ + # TODO: Look up the kube RPM/DEB version installed, or in $PACKAGE_DOWNLOAD_DIR + # and retrieve the major-minor version directly. + local KUBERNETES_VERSION="1.15" + echo "$KUBERNETES_VERSION" +} + canonicalize_image_names() { req_env_var IMGNAMES echo "Adding all current base images to \$IMGNAMES for timestamp update" @@ -479,6 +489,7 @@ _finalize() { fi echo "Re-initializing so next boot does 'first-boot' setup again." cd / + $SUDO rm -rf $GOPATH/src # Actual source will be cloned at runtime $SUDO rm -rf /var/lib/cloud/instanc* $SUDO rm -rf /root/.ssh/* $SUDO rm -rf /etc/ssh/*key* diff --git a/contrib/cirrus/packer/fedora_packaging.sh b/contrib/cirrus/packer/fedora_packaging.sh index e80d48bc8..aecaaef93 100644 --- a/contrib/cirrus/packer/fedora_packaging.sh +++ b/contrib/cirrus/packer/fedora_packaging.sh @@ -11,6 +11,8 @@ echo "Updating/Installing repos and packages for $OS_REL_VER" source $GOSRC/$SCRIPT_BASE/lib.sh +req_env_var GOSRC SCRIPT_BASE BIGTO INSTALL_AUTOMATION_VERSION FEDORA_BASE_IMAGE PRIOR_FEDORA_BASE_IMAGE + # Pre-req. to install automation tooing $LILTO $SUDO dnf install -y git @@ -35,7 +37,7 @@ fi $BIGTO ooe.sh $SUDO dnf update -y -REMOVE_PACKAGES=() +REMOVE_PACKAGES=(runc) INSTALL_PACKAGES=(\ autoconf automake @@ -50,8 +52,11 @@ INSTALL_PACKAGES=(\ containernetworking-plugins containers-common criu + crun + curl device-mapper-devel dnsmasq + e2fsprogs-devel emacs-nox file findutils @@ -60,16 +65,26 @@ INSTALL_PACKAGES=(\ gcc git glib2-devel + glibc-devel glibc-static gnupg go-md2man golang + gpgme gpgme-devel + grubby + hostname iproute iptables jq + krb5-workstation + libassuan libassuan-devel + libblkid-devel libcap-devel + libffi-devel + libgpg-error-devel + libguestfs-tools libmsi1 libnet libnet-devel @@ -79,56 +94,60 @@ INSTALL_PACKAGES=(\ libselinux-devel libtool libvarlink-util + libxml2-devel + libxslt-devel lsof make + mlocate msitools + nfs-utils nmap-ncat + openssl + openssl-devel ostree-devel pandoc + pkgconfig podman + policycoreutils procps-ng protobuf protobuf-c protobuf-c-devel protobuf-devel - python + python2 + python3-PyYAML python3-dateutil python3-psutil python3-pytoml + python3-libsemanage + python3-libselinux + python3-libvirt + redhat-rpm-config + rpcbind rsync + sed selinux-policy-devel skopeo skopeo-containers slirp4netns + socat + tar unzip vim wget which xz zip + zlib-devel +) +DOWNLOAD_PACKAGES=(\ + "cri-o-$(get_kubernetes_version)*" + cri-tools + "kubernetes-$(get_kubernetes_version)*" + runc + oci-umount + parallel ) - -case "$OS_RELEASE_VER" in - 30) - INSTALL_PACKAGES+=(\ - atomic-registries - golang-github-cpuguy83-go-md2man - python2-future - runc - ) - REMOVE_PACKAGES+=(crun) - ;; - 31) - INSTALL_PACKAGES+=(crun) - REMOVE_PACKAGES+=(runc) - ;; - 32) - INSTALL_PACKAGES+=(crun) - REMOVE_PACKAGES+=(runc) - ;; - *) - bad_os_id_ver ;; -esac echo "Installing general build/test dependencies for Fedora '$OS_RELEASE_VER'" $BIGTO ooe.sh $SUDO dnf install -y ${INSTALL_PACKAGES[@]} @@ -136,6 +155,18 @@ $BIGTO ooe.sh $SUDO dnf install -y ${INSTALL_PACKAGES[@]} [[ ${#REMOVE_PACKAGES[@]} -eq 0 ]] || \ $LILTO ooe.sh $SUDO dnf erase -y ${REMOVE_PACKAGES[@]} -export GOPATH="$(mktemp -d)" -trap "$SUDO rm -rf $GOPATH" EXIT -ooe.sh $SUDO $GOSRC/hack/install_catatonit.sh +if [[ ${#DOWNLOAD_PACKAGES[@]} -gt 0 ]]; then + echo "Downloading packages for optional installation at runtime, as needed." + # Required for cri-o + ooe.sh $SUDO dnf -y module enable cri-o:$(get_kubernetes_version) + $SUDO mkdir -p "$PACKAGE_DOWNLOAD_DIR" + cd "$PACKAGE_DOWNLOAD_DIR" + $LILTO ooe.sh $SUDO dnf download -y --resolve ${DOWNLOAD_PACKAGES[@]} + ls -la "$PACKAGE_DOWNLOAD_DIR/" +fi + +echo "Installing runtime tooling" +# Save some runtime by having these already available +cd $GOSRC +$SUDO make install.tools +$SUDO $GOSRC/hack/install_catatonit.sh diff --git a/contrib/cirrus/packer/fedora_setup.sh b/contrib/cirrus/packer/fedora_setup.sh index 3830b3bc4..25b568e8a 100644 --- a/contrib/cirrus/packer/fedora_setup.sh +++ b/contrib/cirrus/packer/fedora_setup.sh @@ -12,11 +12,11 @@ req_env_var SCRIPT_BASE PACKER_BASE INSTALL_AUTOMATION_VERSION PACKER_BUILDER_NA workaround_bfq_bug -# Do not enable update-stesting on the previous Fedora release -if [[ "$FEDORA_BASE_IMAGE" =~ "${OS_RELEASE_ID}-cloud-base-${OS_RELEASE_VER}" ]]; then - DISABLE_UPDATES_TESTING=0 -else +# Do not enable updates-testing on the previous Fedora release +if [[ "$PRIOR_FEDORA_BASE_IMAGE" =~ "${OS_RELEASE_ID}-cloud-base-${OS_RELEASE_VER}" ]]; then DISABLE_UPDATES_TESTING=1 +else + DISABLE_UPDATES_TESTING=0 fi bash $PACKER_BASE/fedora_packaging.sh diff --git a/contrib/cirrus/packer/ubuntu_packaging.sh b/contrib/cirrus/packer/ubuntu_packaging.sh index fd0280230..09f9aab9f 100644 --- a/contrib/cirrus/packer/ubuntu_packaging.sh +++ b/contrib/cirrus/packer/ubuntu_packaging.sh @@ -11,6 +11,8 @@ echo "Updating/Installing repos and packages for $OS_REL_VER" source $GOSRC/$SCRIPT_BASE/lib.sh +req_env_var GOSRC SCRIPT_BASE BIGTO SUDOAPTGET INSTALL_AUTOMATION_VERSION + echo "Updating/configuring package repositories." $BIGTO $SUDOAPTGET update @@ -99,6 +101,7 @@ INSTALL_PACKAGES=(\ protobuf-c-compiler protobuf-compiler python-protobuf + python2 python3-dateutil python3-pip python3-psutil @@ -118,6 +121,11 @@ INSTALL_PACKAGES=(\ zip zlib1g-dev ) +DOWNLOAD_PACKAGES=(\ + cri-o-$(get_kubernetes_version) + cri-tools + parallel +) # These aren't resolvable on Ubuntu 20 if [[ "$OS_RELEASE_VER" -le 19 ]]; then @@ -137,16 +145,15 @@ echo "Installing general testing and system dependencies" $LILTO ooe.sh $SUDOAPTGET update $BIGTO ooe.sh $SUDOAPTGET install ${INSTALL_PACKAGES[@]} -export GOPATH="$(mktemp -d)" -trap "$SUDO rm -rf $GOPATH" EXIT -echo "Installing cataonit and libseccomp.sudo" -cd $GOSRC -ooe.sh $SUDO hack/install_catatonit.sh -ooe.sh $SUDO make install.libseccomp.sudo - -CRIO_RUNC_PATH="/usr/lib/cri-o-runc/sbin/runc" -if $SUDO dpkg -L cri-o-runc | grep -m 1 -q "$CRIO_RUNC_PATH" -then - echo "Linking $CRIO_RUNC_PATH to /usr/bin/runc for ease of testing." - $SUDO ln -f "$CRIO_RUNC_PATH" "/usr/bin/runc" +if [[ ${#DOWNLOAD_PACKAGES[@]} -gt 0 ]]; then + echo "Downloading packages for optional installation at runtime, as needed." + $SUDO ln -s /var/cache/apt/archives "$PACKAGE_DOWNLOAD_DIR" + $LILTO ooe.sh $SUDOAPTGET install --download-only ${DOWNLOAD_PACKAGES[@]} + ls -la "$PACKAGE_DOWNLOAD_DIR/" fi + +echo "Installing runtime tooling" +cd $GOSRC +$SUDO hack/install_catatonit.sh +$SUDO make install.libseccomp.sudo +$SUDO make install.tools diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh index 25b7ff941..31d63677f 100755 --- a/contrib/cirrus/setup_environment.sh +++ b/contrib/cirrus/setup_environment.sh @@ -78,14 +78,6 @@ case "$CG_FS_TYPE" in warn "Forcing testing with crun instead of runc" X=$(echo "export OCI_RUNTIME=/usr/bin/crun" | \ tee -a /etc/environment) && eval "$X" && echo "$X" - - if [[ "$OS_RELEASE_ID" == "fedora" ]]; then - warn "Upgrading to the latest crun" - # Normally not something to do for stable testing - # but crun is new, and late-breaking fixes may be required - # on short notice - dnf update -y crun containers-common - fi ;; *) die 110 "Unsure how to handle cgroup filesystem type '$CG_FS_TYPE'" -- cgit v1.2.3-54-g00ecf From 2ffb703d1b36346d43d6f11c870279934d7a1bb9 Mon Sep 17 00:00:00 2001 From: Daniel J Walsh Date: Thu, 11 Jun 2020 08:54:13 -0400 Subject: Fix builds for RDO Signed-off-by: Daniel J Walsh --- contrib/spec/podman.spec.in | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'contrib') diff --git a/contrib/spec/podman.spec.in b/contrib/spec/podman.spec.in index 8d3cba612..260de7b20 100644 --- a/contrib/spec/podman.spec.in +++ b/contrib/spec/podman.spec.in @@ -77,8 +77,9 @@ BuildRequires: systemd-devel Requires: skopeo-containers Requires: containernetworking-plugins >= 0.6.0-3 Requires: iptables -%if 0%{?rhel} <= 7 +%if 0%{?rhel} < 8 || 0%{?centos} < 8 Requires: container-selinux +Requires: runc %else %if 0%{?rhel} || 0%{?centos} Requires: runc -- cgit v1.2.3-54-g00ecf