From 84c6f7c55da4a1f4d6968c5f8bd8a8553ab5d55e Mon Sep 17 00:00:00 2001 From: Chris Evich Date: Fri, 1 Feb 2019 10:26:43 -0500 Subject: Cirrus: Remove "too new" runc hack Hack was to workaround not having fix for: https://bodhi.fedoraproject.org/updates/FEDORA-2019-b4356521ba Update `RUNC_COMMIT` value to match commit id to working package. Signed-off-by: Chris Evich --- contrib/cirrus/packer/fedora_setup.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'contrib/cirrus/packer') diff --git a/contrib/cirrus/packer/fedora_setup.sh b/contrib/cirrus/packer/fedora_setup.sh index 36a65eb71..18490fdac 100644 --- a/contrib/cirrus/packer/fedora_setup.sh +++ b/contrib/cirrus/packer/fedora_setup.sh @@ -22,6 +22,7 @@ ooe.sh sudo dnf install -y \ bats \ btrfs-progs-devel \ bzip2 \ + criu \ device-mapper-devel \ emacs-nox \ findutils \ @@ -75,8 +76,6 @@ install_buildah install_conmon -install_criu - install_packer_copied_files rh_finalize # N/B: Halts system! -- cgit v1.2.3-54-g00ecf From 191a08ae43fcb2b846e6118073f852e4eb875d5d Mon Sep 17 00:00:00 2001 From: Chris Evich Date: Wed, 6 Mar 2019 11:47:24 -0500 Subject: Cirrus: Support testing of VM cache-image changes Previously, it was quite difficult to affect changes to VM cache images without lots of manual work. This commit adds a new optional testing task which mirrors the official-image build task which only runs on master. In contrast, the new task may be run at any time in a PR, but including a magic phrase in the PR description. Update documentation to describe the new task and inform on it's usage. Signed-off-by: Chris Evich --- .cirrus.yml | 175 +++++++++++++++++++++------- contrib/cirrus/README.md | 73 +++++++----- contrib/cirrus/build_vm_images.sh | 5 +- contrib/cirrus/lib.sh | 5 +- contrib/cirrus/packer/centos_setup.sh | 3 + contrib/cirrus/packer/fedora_setup.sh | 3 + contrib/cirrus/packer/libpod_images.yml | 6 +- contrib/cirrus/packer/rhel_setup.sh | 3 + contrib/cirrus/packer/ubuntu_setup.sh | 3 + contrib/cirrus/rootless_test.sh | 6 +- contrib/cirrus/setup_and_run_rootless.sh | 26 +++++ contrib/cirrus/test/test_dot_cirrus_yaml.py | 78 ------------- 12 files changed, 228 insertions(+), 158 deletions(-) create mode 100755 contrib/cirrus/setup_and_run_rootless.sh delete mode 100755 contrib/cirrus/test/test_dot_cirrus_yaml.py (limited to 'contrib/cirrus/packer') diff --git a/.cirrus.yml b/.cirrus.yml index fccecee7b..6035284c8 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -36,12 +36,12 @@ env: ubuntu-18-libpod-548c1c05 rhel-7-libpod-548c1c05 image-builder-image-1541772081 - FEDORA_CACHE_IMAGE_NAME: "fedora-29-libpod-548c1c05" - PRIOR_FEDORA_CACHE_IMAGE_NAME: "fedora-28-libpod-548c1c05" - UBUNTU_CACHE_IMAGE_NAME: "ubuntu-18-libpod-548c1c05" - PRIOR_RHEL_CACHE_IMAGE_NAME: "rhel-7-libpod-548c1c05" - # RHEL_CACHE_IMAGE_NAME: "rhel-8-notready" - # CENTOS_CACHE_IMAGE_NAME: "centos-7-notready" + FEDORA_CACHE_IMAGE_NAME: "fedora-29-libpod-5699523102900224" + PRIOR_FEDORA_CACHE_IMAGE_NAME: "fedora-28-libpod-5699523102900224" + UBUNTU_CACHE_IMAGE_NAME: "ubuntu-18-libpod-5699523102900224" + PRIOR_RHEL_CACHE_IMAGE_NAME: "rhel-7-libpod-5699523102900224" + RHEL_CACHE_IMAGE_NAME: "rhel-8-notready" + CENTOS_CACHE_IMAGE_NAME: "centos-7-notready" #### #### Variables for composing new cache-images (used in PR testing) from @@ -106,6 +106,17 @@ env: XDG_DATA_DIRS XDG_RUNTIME_DIR XDG_SESSION_ID ROOTLESS_USER +# Default VM to use unless set or modified by task +gce_instance: + image_project: "libpod-218412" + zone: "us-central1-a" # Required by Cirrus for the time being + cpu: 2 + memory: "4Gb" + disk: 200 + # A matrix could be used here, for now just one VM + image_name: "${FEDORA_CACHE_IMAGE_NAME}" + + # Every *_task runs in parallel in separate VMsd. The name prefix only for reference # in WebUI, and will be followed by matrix details. This task gates all others with # quick format, lint, and unit tests on the standard platform. @@ -131,7 +142,6 @@ gating_task: - '/usr/local/bin/entrypoint.sh install.tools |& ${TIMESTAMP}' - '/usr/local/bin/entrypoint.sh validate |& ${TIMESTAMP}' - '/usr/local/bin/entrypoint.sh lint |& ${TIMESTAMP}' - - '${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/test/test_dot_cirrus_yaml.py |& ${TIMESTAMP}' # This task builds Podman with different buildtags to ensure the build does # not break. It also verifies all sub-commands have man pages. @@ -157,6 +167,8 @@ gating_task: # in sync at all times. vendor_task: + only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\*\*\*\s*CIRRUS:\s*TEST\s*IMAGES\s*\*\*\*.*' + depends_on: - "gating" @@ -184,11 +196,15 @@ vendor_task: # whether the git tree is clean. varlink_api_task: + only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\*\*\*\s*CIRRUS:\s*TEST\s*IMAGES\s*\*\*\*.*' + depends_on: - "gating" env: CIRRUS_WORKING_DIR: "/usr/src/libpod" + GOPATH: "/go" + GOSRC: "/go/src/github.com/containers/libpod" # Used by tree_status.sh SUGGESTION: 'remove API.md, then "make varlink_api_generate" and commit changes.' @@ -200,9 +216,9 @@ varlink_api_task: timeout_in: 10m - vendor_script: - - '/usr/local/bin/entrypoint.sh varlink_api_generate' - - 'cd /go/src/github.com/containers/libpod && ./hack/tree_status.sh' + api_md_script: + - '/usr/local/bin/entrypoint.sh varlink_api_generate |& ${TIMESTAMP}' + - 'cd ${GOSRC} && ./hack/tree_status.sh |& ${TIMESTAMP}' on_failure: failed_master_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_master_failure.sh' @@ -216,7 +232,8 @@ build_each_commit_task: - "varlink_api" # $CIRRUS_BASE_BRANCH is only set when testing a PR - only_if: $CIRRUS_BRANCH != 'master' + only_if: $CIRRUS_BRANCH != 'master' && + $CIRRUS_CHANGE_MESSAGE !=~ '.*\*\*\*\s*CIRRUS:\s*TEST\s*IMAGES\s*\*\*\*.*' gce_instance: image_project: "libpod-218412" @@ -250,7 +267,15 @@ meta_task: env: # Space-separated list of images used by this repository state - IMGNAMES: "${ACTIVE_CACHE_IMAGE_NAMES}" + IMGNAMES: >- + ${FEDORA_CACHE_IMAGE_NAME} + ${PRIOR_FEDORA_CACHE_IMAGE_NAME} + ${UBUNTU_CACHE_IMAGE_NAME} + ${PRIOR_RHEL_CACHE_IMAGE_NAME} + ${RHEL_CACHE_IMAGE_NAME} + ${PRIOR_CENTOS_CACHE_IMAGE_NAME} + ${CENTOS_CACHE_IMAGE_NAME} + ${IMAGE_BUILDER_CACHE_IMAGE_NAME} BUILDID: "${CIRRUS_BUILD_ID}" REPOREF: "${CIRRUS_CHANGE_IN_REPO}" GCPJSON: ENCRYPTED[950d9c64ad78f7b1f0c7e499b42dc058d2b23aa67e38b315e68f557f2aba0bf83068d4734f7b1e1bdd22deabe99629df] @@ -258,6 +283,8 @@ meta_task: GCPPROJECT: ENCRYPTED[7c80e728e046b1c76147afd156a32c1c57d4a1ac1eab93b7e68e718c61ca8564fc61fef815952b8ae0a64e7034b8fe4f] CIRRUS_CLONE_DEPTH: 1 # source not used + timeout_in: 10m + script: '/usr/local/bin/entrypoint.sh |& ${TIMESTAMP}' @@ -270,19 +297,10 @@ testing_task: - "vendor" - "build_each_commit" - env: - matrix: - TEST_REMOTE_CLIENT: true - TEST_REMOTE_CLIENT: false + # Only test build cache-images, if that's what's requested + only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\*\*\*\s*CIRRUS:\s*TEST\s*IMAGES\s*\*\*\*.*' gce_instance: - image_project: "libpod-218412" - zone: "us-central1-a" # Required by Cirrus for the time being - cpu: 2 - memory: "4Gb" - disk: 200 # see https://developers.google.com/compute/docs/disks#performance - # Generate multiple parallel tasks, covering all possible - # 'matrix' combinations. matrix: # Images are generated separately, from build_images_task (below) image_name: "${FEDORA_CACHE_IMAGE_NAME}" @@ -291,8 +309,11 @@ testing_task: timeout_in: 120m - # Every *_script runs in sequence, for each task. The name prefix is for - # WebUI reference. The values may be strings... + env: + matrix: + TEST_REMOTE_CLIENT: true + TEST_REMOTE_CLIENT: false + setup_environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}' unit_test_script: '$SCRIPT_BASE/unit_test.sh |& ${TIMESTAMP}' integration_test_script: '$SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP}' @@ -317,21 +338,14 @@ special_testing_task: - "vendor" - "build_each_commit" - gce_instance: - image_project: "libpod-218412" - zone: "us-central1-a" # Required by Cirrus for the time being - cpu: 2 - memory: "4Gb" - disk: 200 - # A matrix could be used here, for now just one VM - image_name: "${FEDORA_CACHE_IMAGE_NAME}" + only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\*\*\*\s*CIRRUS:\s*TEST\s*IMAGES\s*\*\*\*.*' env: matrix: SPECIALMODE: 'rootless' # See docs SPECIALMODE: 'in_podman' # See docs - timeout_in: 120m + timeout_in: 60m setup_environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}' integration_test_script: '$SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP}' @@ -354,16 +368,18 @@ optional_testing_task: # later from OS distribution's build systems. only_if: >- $CIRRUS_BRANCH != 'master' && + $CIRRUS_CHANGE_MESSAGE !=~ '.*\*\*\*\s*CIRRUS:\s*TEST\s*IMAGES\s*\*\*\*.*' && $CIRRUS_CHANGE_MESSAGE =~ '.*\*\*\*\s*CIRRUS:\s*SYSTEM\s*TEST\s*\*\*\*.*' gce_instance: - image_project: "libpod-218412" matrix: image_name: "${FEDORA_CACHE_IMAGE_NAME}" image_name: "${PRIOR_FEDORA_CACHE_IMAGE_NAME}" image_name: "${UBUNTU_CACHE_IMAGE_NAME}" image_name: "${PRIOR_RHEL_CACHE_IMAGE_NAME}" - image_name: "${CENTOS_CACHE_IMAGE_NAME}" + # image_name: "${RHEL_CACHE_IMAGE_NAME}" + image_name: "${PRIOR_CENTOS_CACHE_IMAGE_NAME}" + # image_name: "${CENTOS_CACHE_IMAGE_NAME}" timeout_in: 60m @@ -371,13 +387,88 @@ optional_testing_task: system_test_script: '$SCRIPT_BASE/system_test.sh |& ${TIMESTAMP}' +# Test building of new cache-images for future PR testing, in this PR. +test_build_cache_images_task: + + only_if: >- + $CIRRUS_BRANCH != 'master' && + $CIRRUS_CHANGE_MESSAGE =~ '.*\*\*\*\s*CIRRUS:\s*TEST\s*IMAGES\s*\*\*\*.*' && + $CIRRUS_CHANGE_MESSAGE !=~ '.*\*\*\*\s*CIRRUS:\s*SYSTEM\s*TEST\s*\*\*\*.*' + + depends_on: + - "gating" + + # VMs created by packer are not cleaned up by cirrus, must allow task to complete + auto_cancellation: $CI != "true" + + env: + # Unique to this specific run of _all_ tasks + BUILT_IMAGE_SUFFIX: "-${CIRRUS_REPO_NAME}-${CIRRUS_BUILD_ID}" + + gce_instance: + image_project: "libpod-218412" + zone: "us-central1-a" + cpu: 4 + memory: "4Gb" + disk: 200 + image_name: "${IMAGE_BUILDER_CACHE_IMAGE_NAME}" + scopes: # required for image building + - compute + - devstorage.full_control + + environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}' + build_vm_images_script: '$SCRIPT_BASE/build_vm_images.sh |& ${TIMESTAMP}' + + +# Test building of new cache-images for future PR testing, in this PR. +test_built_images_task: + + only_if: >- + $CIRRUS_BRANCH != 'master' && + $CIRRUS_CHANGE_MESSAGE =~ '.*\*\*\*\s*CIRRUS:\s*TEST\s*IMAGES\s*\*\*\*.*' && + $CIRRUS_CHANGE_MESSAGE !=~ '.*\*\*\*\s*CIRRUS:\s*SYSTEM\s*TEST\s*\*\*\*.*' + + + depends_on: + - "gating" + - "test_build_cache_images" + + # VMs created by packer are not cleaned up by cirrus, must allow task to complete + auto_cancellation: $CI != "true" + + env: + # Unique to this specific run of _all_ tasks + BUILT_IMAGE_SUFFIX: "-${CIRRUS_REPO_NAME}-${CIRRUS_BUILD_ID}" + + gce_instance: + matrix: + # Images are generated separately, from build_images_task (below) + image_name: "fedora-28${BUILT_IMAGE_SUFFIX}" + image_name: "fedora-29${BUILT_IMAGE_SUFFIX}" + image_name: "ubuntu-18${BUILT_IMAGE_SUFFIX}" + + env: + matrix: + TEST_REMOTE_CLIENT: true + TEST_REMOTE_CLIENT: false + + environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}' + unit_test_script: '$SCRIPT_BASE/unit_test.sh |& ${TIMESTAMP}' + integration_test_script: '$SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP}' + rootless_testing_script: + - export ROOTLESS_USER=yarrpilferingpirate + - export ROOTLESS_UID=234567 + - export ROOTLESS_GID=234567 + - cd $GOSRC && $SCRIPT_BASE/setup_and_run_rootless.sh |& ${TIMESTAMP} + + # Build new cache-images for future PR testing, but only after a PR merge. # The cache-images save install/setup time needed test every PR. The 'active' images # are selected by the 'image_name' items tasks above. Currently this requires # manually updating the names, but this could be automated (see comment below). -cache_images_task: +build_cache_images_task: # Only produce new cache-images after a PR merge, and if a magic string - # is present in the most recent commit-message. + # is present in the most recent ___commit-message___. only_if: >- $CIRRUS_BRANCH == 'master' && $CIRRUS_CHANGE_MESSAGE =~ '.*\*\*\*\s*CIRRUS:\s*REBUILD\s*IMAGES\s*\*\*\*.*' @@ -426,12 +517,14 @@ success_task: depends_on: # ignores any dependent task conditions - "gating" - - "varlink_api" + - "build_each_commit" - "vendor" - - "build_each_commit_task" + - "varlink_api" - "testing" - - "rootless_testing_task" + - "special_testing" - "optional_testing" + - "test_build_cache_images" + - "build_cache_images" env: CIRRUS_WORKING_DIR: "/usr/src/libpod" diff --git a/contrib/cirrus/README.md b/contrib/cirrus/README.md index ea358d2d7..ce3ea7777 100644 --- a/contrib/cirrus/README.md +++ b/contrib/cirrus/README.md @@ -99,43 +99,64 @@ contents of the ``$SPECIALMODE`` environment variable. then execute `make localsystem` from the repository root. -### ``cache_images`` Task - -Modifying the contents of cache-images is done by making changes to -one or more of the ``./contrib/cirrus/packer/*_setup.sh`` files. Testing -those changes currently requires adding a temporary commit to a PR that -updates ``.cirrus.yml``: +### ``test_build_cache_images_task`` Task -* Remove all task sections except ``cache_images_task``. -* Remove the ``only_if`` condition and ``depends_on`` dependencies - -The new image names will be displayed at the end of output, assuming the build -is successful, at that point the temporary commit may be removed. Finally, -the new names may be used as ``image_name`` values in ``.cirrus.yml``. +Modifying the contents of cache-images is tested by making changes to +one or more of the ``./contrib/cirrus/packer/*_setup.sh`` files. Then +in the PR description, add the magic string: ``***CIRRUS: TEST IMAGES***`` ***N/B: Steps below are performed by automation*** -1. When a PR is merged (``$CIRRUS_BRANCH`` == ``master``), run another - round of the ``gating`` and ``testing`` tasks (above). - -2. Assuming tests pass, if the commit message contains the magic string - ``***CIRRUS: REBUILD IMAGES***``, then this task continues. Otherwise - simply mark the master branch as 'passed'. +1. ``setup_environment.sh``: Same as for other tasks. -3. ``setup_environment.sh``: Same as for other tasks. - -4. ``build_vm_images.sh``: Utilize [the packer tool](http://packer.io/docs/) +2. ``build_vm_images.sh``: Utilize [the packer tool](http://packer.io/docs/) to produce new VM images. Create a new VM from each base-image, connect to them with ``ssh``, and perform the steps as defined by the - ``$PACKER_BASE/libpod_images.json`` file: + ``$PACKER_BASE/libpod_images.yml`` file: 1. On a base-image VM, as root, copy the current state of the repository into ``/tmp/libpod``. 2. Execute distribution-specific scripts to prepare the image for - use by the ``integration_testing`` task (above). For example, - ``fedora_setup.sh``. - 3. If successful, shut down each VM and create a new GCE Image - named with the base image, and the commit sha of the merge. + use. For example, ``fedora_setup.sh``. + 3. If successful, shut down each VM and record the names, and dates + into a json manifest file. + 4. Move the manifest file, into a google storage bucket object. + This is a retained as a secondary method for tracking/auditing + creation of VM images, should it ever be needed. + +***Manual Steps:*** In order to utilize built images, their names must be upated +in ``.cirrus.yml``. For example, if the image ``blah-1234`` was produced above: + +```yaml +env: + #### + #### Cache-image names to test with + ### + BLAH_CACHE_IMAGE_NAME: "blah-1234" +``` + +A new pull-request with that change, will run tasks utilizing that image. + + +### ``test_built_images`` Task + +Only runs following successful ``test_build_cache_images_task`` task. Uses +images following the standard naming format, with execution of +the 'gate', 'testing' and 'rootless_testing' scripts. Validating the images +suitability for wide-spread use. + + +### ``cache_images`` Task + +Exactly the same as ``test_build_cache_images_task`` task, but only runs on +the master branch. Requires a magic string to be in the `HEAD` +commit message: ``***CIRRUS: BUILD IMAGES***`` + +When successful, the manifest file along with all VM disks, are moved +into a dedicated google storage bucket, separate from the one used by +`test_build_cache_images_task`. These may be used to create new cache-images for +PR testing by manually importing them as described above. + ### Base-images diff --git a/contrib/cirrus/build_vm_images.sh b/contrib/cirrus/build_vm_images.sh index 43eb3c057..cfdfd5d7d 100755 --- a/contrib/cirrus/build_vm_images.sh +++ b/contrib/cirrus/build_vm_images.sh @@ -46,7 +46,4 @@ make libpod_images \ URI="gs://packer-import${POST_MERGE_BUCKET_SUFFIX}/manifest${BUILT_IMAGE_SUFFIX}.json" gsutil cp packer-manifest.json "$URI" -echo "Finished." -echo "Any tarball URI's referenced above at at $URI" -echo "may be used to create VM images suitable for use in" -echo ".cirrus.yml as values for the 'image_name' keys." +echo "Finished. A JSON manifest of produced images is available at $URI" diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh index a285b133b..565646c12 100644 --- a/contrib/cirrus/lib.sh +++ b/contrib/cirrus/lib.sh @@ -19,6 +19,9 @@ CIRRUS_BASE_SHA=${CIRRUS_BASE_SHA:-HEAD} CIRRUS_CHANGE_IN_REPO=${CIRRUS_CHANGE_IN_REPO:-FETCH_HEAD} SPECIALMODE="${SPECIALMODE:-none}" export CONTAINER_RUNTIME=${CONTAINER_RUNTIME:-podman} +ROOTLESS_USER="${ROOTLESS_USER:-}" +ROOTLESS_UID="${ROOTLESS_UID:-}" +ROOTLESS_GID="${ROOTLESS_GID:-}" if ! [[ "$PATH" =~ "/usr/local/bin" ]] then @@ -200,7 +203,7 @@ setup_rootless() { su --login --command 'go env' $ROOTLESS_USER | \ while read envline do - X=$(echo "export $envline" | tee -a "/home/$ROOTLESS_USER/$ENVLIB") && echo "$X" + X=$(echo "export $envline" | tee -a "/home/$ROOTLESS_USER/.bash_profile") && echo "$X" done } diff --git a/contrib/cirrus/packer/centos_setup.sh b/contrib/cirrus/packer/centos_setup.sh index 91b1963c2..b86ba1a1c 100644 --- a/contrib/cirrus/packer/centos_setup.sh +++ b/contrib/cirrus/packer/centos_setup.sh @@ -36,6 +36,7 @@ ooe.sh sudo yum -y install \ golang-github-cpuguy83-go-md2man \ gpgme-devel \ iptables \ + jq \ libassuan-devel \ libcap-devel \ libnet \ @@ -77,6 +78,8 @@ install_criu install_packer_copied_files +sudo /tmp/libpod/hack/install_catatonit.sh + rh_finalize echo "SUCCESS!" diff --git a/contrib/cirrus/packer/fedora_setup.sh b/contrib/cirrus/packer/fedora_setup.sh index 18490fdac..fef74179d 100644 --- a/contrib/cirrus/packer/fedora_setup.sh +++ b/contrib/cirrus/packer/fedora_setup.sh @@ -36,6 +36,7 @@ ooe.sh sudo dnf install -y \ gpgme-devel \ iptables \ iproute \ + jq \ libassuan-devel \ libcap-devel \ libnet \ @@ -78,6 +79,8 @@ install_conmon install_packer_copied_files +sudo /tmp/libpod/hack/install_catatonit.sh + rh_finalize # N/B: Halts system! echo "SUCCESS!" diff --git a/contrib/cirrus/packer/libpod_images.yml b/contrib/cirrus/packer/libpod_images.yml index 30ad0723a..5c42fdae1 100644 --- a/contrib/cirrus/packer/libpod_images.yml +++ b/contrib/cirrus/packer/libpod_images.yml @@ -98,8 +98,4 @@ provisioners: - 'RHSM_COMMAND={{user `RHSM_COMMAND`}}' post-processors: - # Store VM disk in GCP storage, where it will expire based on a defined - # lifecycle. This prevents GCE from filling with disused images. - - - type: 'googlecompute-export' - paths: ['gs://packer-import{{user `POST_MERGE_BUCKET_SUFFIX`}}/{{build_name}}{{user `BUILT_IMAGE_SUFFIX`}}.tar.gz'] - - type: 'manifest' # writes packer-manifest.json + - - type: 'manifest' # writes packer-manifest.json diff --git a/contrib/cirrus/packer/rhel_setup.sh b/contrib/cirrus/packer/rhel_setup.sh index 45f5c3e9b..b6240be80 100644 --- a/contrib/cirrus/packer/rhel_setup.sh +++ b/contrib/cirrus/packer/rhel_setup.sh @@ -41,6 +41,7 @@ ooe.sh sudo yum -y install \ golang-github-cpuguy83-go-md2man \ gpgme-devel \ iptables \ + jq \ libassuan-devel \ libcap-devel \ libnet \ @@ -84,6 +85,8 @@ install_criu install_packer_copied_files +sudo /tmp/libpod/hack/install_catatonit.sh + rhel_exit_handler # release subscription! rh_finalize diff --git a/contrib/cirrus/packer/ubuntu_setup.sh b/contrib/cirrus/packer/ubuntu_setup.sh index d3ac8bddb..93dbd0fa0 100644 --- a/contrib/cirrus/packer/ubuntu_setup.sh +++ b/contrib/cirrus/packer/ubuntu_setup.sh @@ -45,6 +45,7 @@ ooe.sh sudo -E apt-get -qq install \ golang \ iproute2 \ iptables \ + jq \ libaio-dev \ libapparmor-dev \ libcap-dev \ @@ -101,6 +102,8 @@ install_buildah install_packer_copied_files +sudo /tmp/libpod/hack/install_catatonit.sh + install_varlink sudo curl https://raw.githubusercontent.com/projectatomic/registries/master/registries.fedora\ diff --git a/contrib/cirrus/rootless_test.sh b/contrib/cirrus/rootless_test.sh index eab06bac0..13386bb09 100755 --- a/contrib/cirrus/rootless_test.sh +++ b/contrib/cirrus/rootless_test.sh @@ -1,10 +1,10 @@ #!/bin/bash -set -e -source $HOME/.bash_profile +set -ex +source "$HOME/.bash_profile" +source $GOSRC/$SCRIPT_BASE/lib.sh cd $GOSRC -source $(dirname $0)/lib.sh req_env_var GOSRC OS_RELEASE_ID OS_RELEASE_VER diff --git a/contrib/cirrus/setup_and_run_rootless.sh b/contrib/cirrus/setup_and_run_rootless.sh new file mode 100755 index 000000000..7ca34a6ac --- /dev/null +++ b/contrib/cirrus/setup_and_run_rootless.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +set -ex + +source $(dirname $0)/lib.sh + +req_env_var " +CIRRUS_WORKING_DIR $CIRRUS_WORKING_DIR +GOSRC $GOSRC +SCRIPT_BASE $SCRIPT_BASE +ROOTLESS_USER $ROOTLESS_USER +ROOTLESS_UID $ROOTLESS_UID +ROOTLESS_GID $ROOTLESS_GID +" + +if run_rootless +then + die 86 "Error: Expected rootless env. vars not set or empty" +fi + +cd $GOSRC +setup_rootless + +ssh $ROOTLESS_USER@localhost \ + -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o CheckHostIP=no \ + $CIRRUS_WORKING_DIR/$SCRIPT_BASE/rootless_test.sh diff --git a/contrib/cirrus/test/test_dot_cirrus_yaml.py b/contrib/cirrus/test/test_dot_cirrus_yaml.py deleted file mode 100755 index 2894bc45e..000000000 --- a/contrib/cirrus/test/test_dot_cirrus_yaml.py +++ /dev/null @@ -1,78 +0,0 @@ -#!/bin/env python3 - -import sys -import os -import os.path -import unittest -import warnings -import yaml - -class TestCaseBase(unittest.TestCase): - - SCRIPT_PATH = os.path.realpath((os.path.dirname(sys.argv[0]))) - CIRRUS_WORKING_DIR = os.environ.get('CIRRUS_WORKING_DIR', - '{0}/../../../'.format(SCRIPT_PATH)) - - def setUp(self): - os.chdir(self.CIRRUS_WORKING_DIR) - - -class TestCirrusYAML(TestCaseBase): - - IMAGE_NAME_SUFFIX = '_CACHE_IMAGE_NAME' - ACTIVE_IMAGES_NAME = 'ACTIVE_CACHE_IMAGE_NAMES' - - def setUp(self): - TestCirrusYAML._cirrus = None - super().setUp() - - @property - def cirrus(self): - if TestCirrusYAML._cirrus is None: - with warnings.catch_warnings(): - warnings.filterwarnings("ignore",category=DeprecationWarning) - with open('.cirrus.yml', "r") as dot_cirrus_dot_yaml: - TestCirrusYAML._cirrus = yaml.load(dot_cirrus_dot_yaml) - return TestCirrusYAML._cirrus - - def _assert_get_cache_image_names(self, env): - inames = set([key for key in env.keys() - if key.endswith(self.IMAGE_NAME_SUFFIX)]) - self.assertNotEqual(inames, set()) - - ivalues = set([value for key, value in env.items() - if key in inames]) - self.assertNotEqual(ivalues, set()) - return ivalues - - def _assert_get_subdct(self, key, dct): - self.assertIn(key, dct) - return dct[key] - - def test_parse_yaml(self): - self.assertIsInstance(self.cirrus, dict) - - def test_active_cache_image_names(self): - env = self._assert_get_subdct('env', self.cirrus) - acin = self._assert_get_subdct(self.ACTIVE_IMAGES_NAME, env) - - for ivalue in self._assert_get_cache_image_names(env): - self.assertIn(ivalue, acin, - "The '{}' sub-key of 'env' should contain this among" - " its space-separated values." - "".format(self.ACTIVE_IMAGES_NAME)) - - - def test_cache_image_names_active(self): - env = self._assert_get_subdct('env', self.cirrus) - ivalues = self._assert_get_cache_image_names(env) - - for avalue in set(self._assert_get_subdct(self.ACTIVE_IMAGES_NAME, env).split()): - self.assertIn(avalue, ivalues, - "All space-separated values in the '{}' sub-key" - " of 'env' must also be used in a key with a '{}' suffix." - "".format(self.ACTIVE_IMAGES_NAME, self.IMAGE_NAME_SUFFIX)) - - -if __name__ == '__main__': - unittest.main(failfast=True, catchbreak=True, verbosity=0) -- cgit v1.2.3-54-g00ecf From 51a95e6ef1b4616f308d0a7a5507c2ece74a9f0f Mon Sep 17 00:00:00 2001 From: Chris Evich Date: Wed, 13 Mar 2019 15:13:17 -0400 Subject: Cirrus: Run tests on test-built cache-images Also, add jq and catatonit installs to images. Signed-off-by: Chris Evich --- .cirrus.yml | 37 +++------------------ contrib/cirrus/README.md | 51 +++++++++++++++++++++-------- contrib/cirrus/lib.sh | 52 ++++++++++++++++++++++++++---- contrib/cirrus/packer/centos_setup.sh | 11 ++++--- contrib/cirrus/packer/fedora_setup.sh | 7 ++-- contrib/cirrus/packer/rhel_setup.sh | 14 +++++--- contrib/cirrus/packer/ubuntu_setup.sh | 4 +-- contrib/cirrus/resetup_and_run_rootless.sh | 32 ++++++++++++++++++ contrib/cirrus/rootless_test.sh | 12 ++++--- contrib/cirrus/setup_and_run_rootless.sh | 26 --------------- contrib/cirrus/setup_environment.sh | 1 - 11 files changed, 148 insertions(+), 99 deletions(-) create mode 100755 contrib/cirrus/resetup_and_run_rootless.sh delete mode 100755 contrib/cirrus/setup_and_run_rootless.sh (limited to 'contrib/cirrus/packer') diff --git a/.cirrus.yml b/.cirrus.yml index 6035284c8..08875a354 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -26,6 +26,7 @@ env: # Command to prefix every output line with a timestamp # (can't do inline awk script, Cirrus-CI or YAML mangles quoting) TIMESTAMP: "awk --file ${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/timestamp.awk" + BUILT_IMAGE_SUFFIX: "-${CIRRUS_REPO_NAME}-${CIRRUS_BUILD_ID}" #### #### Cache-image names to test with @@ -89,22 +90,6 @@ env: # Name where this repositories cloud resources are located GCP_PROJECT_ID: ENCRYPTED[7c80e728e046b1c76147afd156a32c1c57d4a1ac1eab93b7e68e718c61ca8564fc61fef815952b8ae0a64e7034b8fe4f] - # Space separated list of environment variables to unset before testing - UNSET_ENV_VARS: >- - GCP_PROJECT_ID GCE_SSH_USERNAME SERVICE_ACCOUNT RHSM_COMMAND BUILT_IMAGE_SUFFIX - IRCID RHEL_BASE_IMAGE FAH_BASE_IMAGE FEDORA_BASE_IMAGE CENTOS_BASE_IMAGE - UBUNTU_BASE_IMAGE PACKER_VER PACKER_BUILDS RUNC_COMMIT CRIU_COMMIT - CRIO_COMMIT CNI_COMMIT FEDORA_CNI_COMMIT PACKER_BASE SCRIPT_BASE - CIRRUS_SHELL CIRRUS_WORKING_DIR ENVLIB BUILT_IMAGE_SUFFIX CIRRUS_CI - CI_NODE_INDEX CI_NODE_TOTAL CIRRUS_BASE_BRANCH CIRRUS_BASE_SHA - CIRRUS_BRANCH CIRRUS_BUILD_ID CIRRUS_CHANGE_IN_REPO CIRRUS_CLONE_DEPTH - CIRRUS_COMMIT_MESSAGE CIRRUS_CHANGE_MESSAGE CIRRUS_REPO_CLONE_HOST - CIRRUS_DEFAULT_BRANCH CIRRUS_PR CIRRUS_TAG CIRRUS_OS CIRRUS_TASK_NAME - CIRRUS_TASK_ID CIRRUS_REPO_NAME CIRRUS_REPO_OWNER CIRRUS_REPO_FULL_NAME - CIRRUS_REPO_CLONE_URL CIRRUS_SHELL CIRRUS_USER_COLLABORATOR CIRRUS_USER_PERMISSION - CIRRUS_WORKING_DIR CIRRUS_HTTP_CACHE_HOST PACKER_BUILDS BUILT_IMAGE_SUFFIX - XDG_DATA_DIRS XDG_RUNTIME_DIR XDG_SESSION_ID ROOTLESS_USER - # Default VM to use unless set or modified by task gce_instance: @@ -401,10 +386,6 @@ test_build_cache_images_task: # VMs created by packer are not cleaned up by cirrus, must allow task to complete auto_cancellation: $CI != "true" - env: - # Unique to this specific run of _all_ tasks - BUILT_IMAGE_SUFFIX: "-${CIRRUS_REPO_NAME}-${CIRRUS_BUILD_ID}" - gce_instance: image_project: "libpod-218412" zone: "us-central1-a" @@ -421,7 +402,7 @@ test_build_cache_images_task: # Test building of new cache-images for future PR testing, in this PR. -test_built_images_task: +verify_test_built_images_task: only_if: >- $CIRRUS_BRANCH != 'master' && @@ -433,13 +414,6 @@ test_built_images_task: - "gating" - "test_build_cache_images" - # VMs created by packer are not cleaned up by cirrus, must allow task to complete - auto_cancellation: $CI != "true" - - env: - # Unique to this specific run of _all_ tasks - BUILT_IMAGE_SUFFIX: "-${CIRRUS_REPO_NAME}-${CIRRUS_BUILD_ID}" - gce_instance: matrix: # Images are generated separately, from build_images_task (below) @@ -455,11 +429,7 @@ test_built_images_task: environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}' unit_test_script: '$SCRIPT_BASE/unit_test.sh |& ${TIMESTAMP}' integration_test_script: '$SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP}' - rootless_testing_script: - - export ROOTLESS_USER=yarrpilferingpirate - - export ROOTLESS_UID=234567 - - export ROOTLESS_GID=234567 - - cd $GOSRC && $SCRIPT_BASE/setup_and_run_rootless.sh |& ${TIMESTAMP} + rootless_testing_script: '$SCRIPT_BASE/resetup_and_run_rootless.sh |& ${TIMESTAMP}' # Build new cache-images for future PR testing, but only after a PR merge. @@ -524,6 +494,7 @@ success_task: - "special_testing" - "optional_testing" - "test_build_cache_images" + - "verify_test_built_images" - "build_cache_images" env: diff --git a/contrib/cirrus/README.md b/contrib/cirrus/README.md index ce3ea7777..1d62f71ce 100644 --- a/contrib/cirrus/README.md +++ b/contrib/cirrus/README.md @@ -124,29 +124,54 @@ in the PR description, add the magic string: ``***CIRRUS: TEST IMAGES***`` This is a retained as a secondary method for tracking/auditing creation of VM images, should it ever be needed. -***Manual Steps:*** In order to utilize built images, their names must be upated -in ``.cirrus.yml``. For example, if the image ``blah-1234`` was produced above: +### ``verify_test_built_images`` Task + +Only runs following successful ``test_build_cache_images_task`` task. Uses +images following the standard naming format; ***however, only runs a limited +sub-set of automated tests***. Validating newly built images fully, requires +updating ``.cirrus.yml``. + +***Manual Steps:*** Assuming `verify_test_built_images` passes, then +you'll find the new image names displayed at the end of the +`test_build_cache_images_task` in the `build_vm_images` output. +For example: + +``` +...cut... +==> Builds finished. The artifacts of successful builds are: +--> rhel-7: A disk image was created: rhel-7-libpod-5699523102900224 +--> rhel-7: +--> ubuntu-18: A disk image was created: ubuntu-18-libpod-5699523102900224 +--> ubuntu-18: +--> fedora-29: A disk image was created: fedora-29-libpod-5699523102900224 +--> fedora-29: +--> fedora-28: A disk image was created: fedora-28-libpod-5699523102900224 +``` + +An updated (or new) pull-request with this change, will utilize +the new cache-images: + ```yaml env: + ...cut... #### #### Cache-image names to test with ### - BLAH_CACHE_IMAGE_NAME: "blah-1234" + FEDORA_CACHE_IMAGE_NAME: "fedora-29-libpod-5699523102900224" + PRIOR_FEDORA_CACHE_IMAGE_NAME: "fedora-28-libpod-5699523102900224" + UBUNTU_CACHE_IMAGE_NAME: "ubuntu-18-libpod-5699523102900224" + PRIOR_RHEL_CACHE_IMAGE_NAME: "rhel-7-libpod-5699523102900224" + ...cut... ``` -A new pull-request with that change, will run tasks utilizing that image. - - -### ``test_built_images`` Task - -Only runs following successful ``test_build_cache_images_task`` task. Uses -images following the standard naming format, with execution of -the 'gate', 'testing' and 'rootless_testing' scripts. Validating the images -suitability for wide-spread use. +Take care to also update the PR description if any 'magic' phrases were used +(they affect which tests run). In other words, you'll likely want to +remove the ``***CIRRUS: TEST IMAGES***`` string - otherwise Cirrus-CI will +simply build new and test again. -### ``cache_images`` Task +### ``build_cache_images`` Task *(Deprecated)* Exactly the same as ``test_build_cache_images_task`` task, but only runs on the master branch. Requires a magic string to be in the `HEAD` diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh index 565646c12..fb1685104 100644 --- a/contrib/cirrus/lib.sh +++ b/contrib/cirrus/lib.sh @@ -19,9 +19,13 @@ CIRRUS_BASE_SHA=${CIRRUS_BASE_SHA:-HEAD} CIRRUS_CHANGE_IN_REPO=${CIRRUS_CHANGE_IN_REPO:-FETCH_HEAD} SPECIALMODE="${SPECIALMODE:-none}" export CONTAINER_RUNTIME=${CONTAINER_RUNTIME:-podman} -ROOTLESS_USER="${ROOTLESS_USER:-}" -ROOTLESS_UID="${ROOTLESS_UID:-}" -ROOTLESS_GID="${ROOTLESS_GID:-}" + +if [[ "$USER" == "root" ]] +then + ROOTLESS_USER="${ROOTLESS_USER:-}" +else + ROOTLESS_USER="${ROOTLESS_USER:-$USER}" +fi if ! [[ "$PATH" =~ "/usr/local/bin" ]] then @@ -35,6 +39,23 @@ then source "$HOME/$ENVLIB" fi +# Space separated list of environment variables to unset before testing +UNSET_ENV_VARS=' + GCP_PROJECT_ID GCE_SSH_USERNAME SERVICE_ACCOUNT RHSM_COMMAND BUILT_IMAGE_SUFFIX + IRCID RHEL_BASE_IMAGE FAH_BASE_IMAGE FEDORA_BASE_IMAGE CENTOS_BASE_IMAGE + UBUNTU_BASE_IMAGE PACKER_VER PACKER_BUILDS RUNC_COMMIT CRIU_COMMIT + CRIO_COMMIT CNI_COMMIT FEDORA_CNI_COMMIT PACKER_BASE SCRIPT_BASE + CIRRUS_SHELL CIRRUS_WORKING_DIR ENVLIB CIRRUS_CI + CI_NODE_INDEX CI_NODE_TOTAL CIRRUS_BASE_BRANCH CIRRUS_BASE_SHA + CIRRUS_BRANCH CIRRUS_BUILD_ID CIRRUS_CHANGE_IN_REPO CIRRUS_CLONE_DEPTH + CIRRUS_COMMIT_MESSAGE CIRRUS_CHANGE_MESSAGE CIRRUS_REPO_CLONE_HOST + CIRRUS_DEFAULT_BRANCH CIRRUS_PR CIRRUS_TAG CIRRUS_OS CIRRUS_TASK_NAME + CIRRUS_TASK_ID CIRRUS_REPO_NAME CIRRUS_REPO_OWNER CIRRUS_REPO_FULL_NAME + CIRRUS_REPO_CLONE_URL CIRRUS_SHELL CIRRUS_USER_COLLABORATOR CIRRUS_USER_PERMISSION + CIRRUS_WORKING_DIR CIRRUS_HTTP_CACHE_HOST PACKER_BUILDS + XDG_DATA_DIRS XDG_RUNTIME_DIR XDG_SESSION_ID ROOTLESS_USER +' + # Pass in a list of one or more envariable names; exit non-zero with # helpful error message if any value is empty req_env_var() { @@ -126,12 +147,14 @@ os_release_id() { eval "$(egrep -m 1 '^ID=' /etc/os-release | tr -d \' | tr -d \")" echo "$ID" } +export OS_RELEASE_ID="$(os_release_id)" # Return a GCE image-name compatible string representation of distribution major version os_release_ver() { eval "$(egrep -m 1 '^VERSION_ID=' /etc/os-release | tr -d \' | tr -d \")" echo "$VERSION_ID" | cut -d '.' -f 1 } +export OS_RELEASE_VER="$(os_release_VER)" bad_os_id_ver() { echo "Unknown/Unsupported distro. $OS_RELEASE_ID and/or version $OS_RELEASE_VER for $ARGS" @@ -158,6 +181,10 @@ ircmsg() { setup_rootless() { req_env_var ROOTLESS_USER GOSRC ENVLIB + make install.catatonit + go get github.com/onsi/ginkgo/ginkgo + go get github.com/onsi/gomega/... + if passwd --status $ROOTLESS_USER then echo "Updating $ROOTLESS_USER user permissions on possibly changed libpod code" @@ -180,7 +207,8 @@ setup_rootless() { chown -R $ROOTLESS_USER:$ROOTLESS_USER "$GOSRC" echo "creating ssh keypair for $USER" - ssh-keygen -P "" -f $HOME/.ssh/id_rsa + [[ -r "$HOME/.ssh/id_rsa" ]] || \ + ssh-keygen -P "" -f "$HOME/.ssh/id_rsa" echo "Allowing ssh key for $ROOTLESS_USER" (umask 077 && mkdir "/home/$ROOTLESS_USER/.ssh") @@ -199,11 +227,14 @@ setup_rootless() { install -o $ROOTLESS_USER -g $ROOTLESS_USER -m 0700 \ "$HOME/$ENVLIB" "/home/$ROOTLESS_USER/$ENVLIB" + # Allow the tests to run + echo "export ROOTLESS_USER=$ROOTLESS_USER" >> "/home/$ROOTLESS_USER/$ENVLIB" + echo "Configuring user's go environment variables" su --login --command 'go env' $ROOTLESS_USER | \ while read envline do - X=$(echo "export $envline" | tee -a "/home/$ROOTLESS_USER/.bash_profile") && echo "$X" + X=$(echo "export $envline" >> "/home/$ROOTLESS_USER/$ENVLIB") done } @@ -242,6 +273,10 @@ install_cni_plugins() { } install_runc_from_git(){ + req_env_var " + GOPATH $GOPATH + OS_RELEASE_ID $OS_RELEASE_ID + " wd=$(pwd) DEST="$GOPATH/src/github.com/opencontainers/runc" rm -rf "$DEST" @@ -249,7 +284,12 @@ install_runc_from_git(){ cd "$DEST" ooe.sh git fetch origin --tags ooe.sh git checkout -q "$RUNC_COMMIT" - ooe.sh make static BUILDTAGS="seccomp apparmor selinux" + if [[ "${OS_RELEASE_ID}" == "ubuntu" ]] + then + ooe.sh make static BUILDTAGS="seccomp apparmor" + else + ooe.sh make BUILDTAGS="seccomp selinux" + fi sudo install -m 755 runc /usr/bin/runc cd $wd } diff --git a/contrib/cirrus/packer/centos_setup.sh b/contrib/cirrus/packer/centos_setup.sh index b86ba1a1c..f9728f2f7 100644 --- a/contrib/cirrus/packer/centos_setup.sh +++ b/contrib/cirrus/packer/centos_setup.sh @@ -59,7 +59,6 @@ ooe.sh sudo yum -y install \ python3-dateutil \ python3-psutil \ python3-pytoml \ - runc \ skopeo-containers \ unzip \ vim \ @@ -68,14 +67,16 @@ ooe.sh sudo yum -y install \ install_scl_git -install_cni_plugins - -install_buildah - install_conmon +install_runc + install_criu +install_cni_plugins + +install_buildah + install_packer_copied_files sudo /tmp/libpod/hack/install_catatonit.sh diff --git a/contrib/cirrus/packer/fedora_setup.sh b/contrib/cirrus/packer/fedora_setup.sh index fef74179d..d9229c668 100644 --- a/contrib/cirrus/packer/fedora_setup.sh +++ b/contrib/cirrus/packer/fedora_setup.sh @@ -60,7 +60,6 @@ ooe.sh sudo dnf install -y \ python3-dateutil \ python3-psutil \ python3-pytoml \ - runc \ skopeo-containers \ slirp4netns \ unzip \ @@ -70,13 +69,15 @@ ooe.sh sudo dnf install -y \ install_varlink +install_conmon + +install_runc + CNI_COMMIT=$FEDORA_CNI_COMMIT install_cni_plugins install_buildah -install_conmon - install_packer_copied_files sudo /tmp/libpod/hack/install_catatonit.sh diff --git a/contrib/cirrus/packer/rhel_setup.sh b/contrib/cirrus/packer/rhel_setup.sh index b6240be80..dee35a102 100644 --- a/contrib/cirrus/packer/rhel_setup.sh +++ b/contrib/cirrus/packer/rhel_setup.sh @@ -22,6 +22,9 @@ ooe.sh sudo subscription-manager repos \ --enable=rhel-7-server-extras-rpms \ --enable=rhel-server-rhscl-7-rpms +sudo sed -r -i -e 's/^notify_only=.+/notify_only=0/g' \ + /etc/yum/pluginconf.d/search-disabled-repos.conf + ooe.sh sudo yum -y update ooe.sh sudo yum -y install \ @@ -66,7 +69,6 @@ ooe.sh sudo yum -y install \ python34-psutil \ python34-pytoml \ python34-PyYAML \ - runc \ skopeo-containers \ unzip \ vim \ @@ -75,14 +77,16 @@ ooe.sh sudo yum -y install \ install_scl_git -install_cni_plugins - -install_buildah - install_conmon +install_runc + install_criu +install_cni_plugins + +install_buildah + install_packer_copied_files sudo /tmp/libpod/hack/install_catatonit.sh diff --git a/contrib/cirrus/packer/ubuntu_setup.sh b/contrib/cirrus/packer/ubuntu_setup.sh index 93dbd0fa0..d02b71e12 100644 --- a/contrib/cirrus/packer/ubuntu_setup.sh +++ b/contrib/cirrus/packer/ubuntu_setup.sh @@ -90,10 +90,10 @@ ooe.sh sudo sed -re "$SEDCMD" -i /etc/default/grub.d/* ooe.sh sudo sed -re "$SEDCMD" -i /etc/default/grub ooe.sh sudo update-grub -install_runc - install_conmon +install_runc + install_criu install_cni_plugins diff --git a/contrib/cirrus/resetup_and_run_rootless.sh b/contrib/cirrus/resetup_and_run_rootless.sh new file mode 100755 index 000000000..1c387fdf5 --- /dev/null +++ b/contrib/cirrus/resetup_and_run_rootless.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +set -e + +# N/B: This script is only intended to be used for the special-case of +# setting up and executing the rootless tests AFTER normal tests complete +# while testing a freshly built image. + +source $(dirname $0)/lib.sh + +# must be after source lib.sh b/c it loads $ENVLIB +export ROOTLESS_USER="pilferingpirate$RANDOM" + +req_env_var " +CIRRUS_WORKING_DIR $CIRRUS_WORKING_DIR +GOSRC $GOSRC +SCRIPT_BASE $SCRIPT_BASE +ROOTLESS_USER $ROOTLESS_USER +" + +if ! run_rootless +then + die 86 "Error: Expected rootless env. var not set or empty" +fi + +cd $GOSRC +make clean +setup_rootless + +ssh $ROOTLESS_USER@localhost \ + -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o CheckHostIP=no \ + $CIRRUS_WORKING_DIR/$SCRIPT_BASE/rootless_test.sh diff --git a/contrib/cirrus/rootless_test.sh b/contrib/cirrus/rootless_test.sh index 13386bb09..d4cc6965e 100755 --- a/contrib/cirrus/rootless_test.sh +++ b/contrib/cirrus/rootless_test.sh @@ -1,12 +1,10 @@ #!/bin/bash -set -ex +set -e -source "$HOME/.bash_profile" -source $GOSRC/$SCRIPT_BASE/lib.sh -cd $GOSRC +source $(dirname $0)/lib.sh -req_env_var GOSRC OS_RELEASE_ID OS_RELEASE_VER +req_env_var GOSRC ROOTLESS_USER if [[ "$UID" == "0" ]] then @@ -17,6 +15,10 @@ fi echo "." echo "Hello, my name is $USER and I live in $PWD can I be your friend?" +show_env_vars + +export PODMAN_VARLINK_ADDRESS=unix:/tmp/podman-$(id -u) + cd "$GOSRC" make make varlink_generate diff --git a/contrib/cirrus/setup_and_run_rootless.sh b/contrib/cirrus/setup_and_run_rootless.sh deleted file mode 100755 index 7ca34a6ac..000000000 --- a/contrib/cirrus/setup_and_run_rootless.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -set -ex - -source $(dirname $0)/lib.sh - -req_env_var " -CIRRUS_WORKING_DIR $CIRRUS_WORKING_DIR -GOSRC $GOSRC -SCRIPT_BASE $SCRIPT_BASE -ROOTLESS_USER $ROOTLESS_USER -ROOTLESS_UID $ROOTLESS_UID -ROOTLESS_GID $ROOTLESS_GID -" - -if run_rootless -then - die 86 "Error: Expected rootless env. vars not set or empty" -fi - -cd $GOSRC -setup_rootless - -ssh $ROOTLESS_USER@localhost \ - -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o CheckHostIP=no \ - $CIRRUS_WORKING_DIR/$SCRIPT_BASE/rootless_test.sh diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh index 1a89a5b71..b3319a3e1 100755 --- a/contrib/cirrus/setup_environment.sh +++ b/contrib/cirrus/setup_environment.sh @@ -37,7 +37,6 @@ then "export OS_RELEASE_VER=\"$(os_release_ver)\"" \ "export OS_REL_VER=\"$(os_release_id)-$(os_release_ver)\"" \ "export TEST_REMOTE_CLIENT=\"$TEST_REMOTE_CLIENT\"" \ - "export BUILT_IMAGE_SUFFIX=\"-$CIRRUS_REPO_NAME-${CIRRUS_CHANGE_IN_REPO:0:8}\"" \ "export GOPATH=\"/var/tmp/go\"" \ 'export PATH="$HOME/bin:$GOPATH/bin:/usr/local/bin:$PATH"' \ 'export LD_LIBRARY_PATH="/usr/local/lib${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}"' -- cgit v1.2.3-54-g00ecf From c4d9b03f7800e37df0a8821b450932700aac3c5d Mon Sep 17 00:00:00 2001 From: Chris Evich Date: Fri, 22 Mar 2019 09:44:03 -0400 Subject: Cirrus: Overhaul/Simplify env. var setup Signed-off-by: Chris Evich --- .cirrus.yml | 46 +++--- contrib/cirrus/build_vm_images.sh | 5 +- contrib/cirrus/integration_test.sh | 15 -- contrib/cirrus/lib.sh | 238 ++++++++++++----------------- contrib/cirrus/notice_master_failure.sh | 2 +- contrib/cirrus/packer/centos_setup.sh | 4 +- contrib/cirrus/packer/fedora_setup.sh | 5 +- contrib/cirrus/packer/rhel_setup.sh | 4 +- contrib/cirrus/packer/ubuntu_setup.sh | 13 +- contrib/cirrus/resetup_and_run_rootless.sh | 32 ---- contrib/cirrus/rootless_test.sh | 3 +- contrib/cirrus/setup_environment.sh | 112 +++++++------- contrib/cirrus/system_test.sh | 2 - contrib/cirrus/unit_test.sh | 4 +- test/registries.conf | 2 +- 15 files changed, 187 insertions(+), 300 deletions(-) delete mode 100755 contrib/cirrus/resetup_and_run_rootless.sh (limited to 'contrib/cirrus/packer') diff --git a/.cirrus.yml b/.cirrus.yml index 08875a354..b3c6ff5a9 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -13,30 +13,22 @@ env: #### #### Global variables used for all tasks #### - # File to update in home-dir with task-specific env. var values - ENVLIB: ".bash_profile" # Overrides default location (/tmp/cirrus) for repo clone + GOPATH: "/var/tmp/go" + GOSRC: "/var/tmp/go/src/github.com/containers/libpod" CIRRUS_WORKING_DIR: "/var/tmp/go/src/github.com/containers/libpod" - # Required so $ENVLIB gets loaded + # The default is 'sh' if unspecified CIRRUS_SHELL: "/bin/bash" # Save a little typing (path relative to $CIRRUS_WORKING_DIR) SCRIPT_BASE: "./contrib/cirrus" - PACKER_BASE: "./contrib/cirrus/packer" - CIRRUS_CLONE_DEPTH: 200 # Command to prefix every output line with a timestamp # (can't do inline awk script, Cirrus-CI or YAML mangles quoting) TIMESTAMP: "awk --file ${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/timestamp.awk" - BUILT_IMAGE_SUFFIX: "-${CIRRUS_REPO_NAME}-${CIRRUS_BUILD_ID}" + CIRRUS_CLONE_DEPTH: 50 #### #### Cache-image names to test with ### - ACTIVE_CACHE_IMAGE_NAMES: >- - fedora-29-libpod-548c1c05 - fedora-28-libpod-548c1c05 - ubuntu-18-libpod-548c1c05 - rhel-7-libpod-548c1c05 - image-builder-image-1541772081 FEDORA_CACHE_IMAGE_NAME: "fedora-29-libpod-5699523102900224" PRIOR_FEDORA_CACHE_IMAGE_NAME: "fedora-28-libpod-5699523102900224" UBUNTU_CACHE_IMAGE_NAME: "ubuntu-18-libpod-5699523102900224" @@ -48,11 +40,13 @@ env: #### Variables for composing new cache-images (used in PR testing) from #### base-images (pre-existing in GCE) #### + BUILT_IMAGE_SUFFIX: "-${CIRRUS_REPO_NAME}-${CIRRUS_BUILD_ID}" # Git commits to use while building dependencies into cache-images FEDORA_CNI_COMMIT: "412b6d31280682bb4fab4446f113c22ff1886554" CNI_COMMIT: "7480240de9749f9a0a5c8614b17f1f03e0c06ab9" CRIO_COMMIT: "7a283c391abb7bd25086a8ff91dbb36ebdd24466" CRIU_COMMIT: "c74b83cd49c00589c0c0468ba5fe685b67fdbd0a" + # Only RHEL/Centos VM cache-image building needs RUNC_COMMIT RUNC_COMMIT: "029124da7af7360afa781a0234d1b083550f797c" # CSV of cache-image names to build (see $PACKER_BASE/libpod_images.json) PACKER_BUILDS: "ubuntu-18,fedora-29,fedora-28,rhel-7" # TODO: rhel-8,centos-7 @@ -60,15 +54,6 @@ env: PACKER_VER: "1.3.2" # Special image w/ nested-libvirt + tools for creating new cache and base images IMAGE_BUILDER_CACHE_IMAGE_NAME: "image-builder-image-1541772081" - # Google-maintained base-image names - UBUNTU_BASE_IMAGE: "ubuntu-1804-bionic-v20181203a" - CENTOS_BASE_IMAGE: "centos-7-v20181113" - # Manually produced base-image names (see $SCRIPT_BASE/README.md) - FEDORA_BASE_IMAGE: "fedora-cloud-base-29-1-2-1541789245" - PRIOR_FEDORA_BASE_IMAGE: "fedora-cloud-base-28-1-1-1544474897" - FAH_BASE_IMAGE: "fedora-atomichost-29-20181025-1-1541787861" - # RHEL image must be imported, google bills extra for their native image. - RHEL_BASE_IMAGE: "rhel-guest-image-7-6-210-x86-64-qcow2-1548099756" #### #### Default to NOT operating in any special-case testing mode @@ -109,6 +94,7 @@ gating_task: env: CIRRUS_WORKING_DIR: "/usr/src/libpod" + GOPATH: "/go" GOSRC: "/go/src/github.com/containers/libpod" # Runs within Cirrus's "community cluster" @@ -159,6 +145,8 @@ vendor_task: env: CIRRUS_WORKING_DIR: "/usr/src/libpod" + GOPATH: "/go" + GOSRC: "/go/src/github.com/containers/libpod" # Runs within Cirrus's "community cluster" container: @@ -171,7 +159,7 @@ vendor_task: vendor_script: - '/usr/local/bin/entrypoint.sh .install.vndr |& ${TIMESTAMP}' - '/usr/local/bin/entrypoint.sh vendor |& ${TIMESTAMP}' - - 'cd /go/src/github.com/containers/libpod && ./hack/tree_status.sh |& ${TIMESTAMP}' + - 'cd ${GOSRC} && ./hack/tree_status.sh |& ${TIMESTAMP}' on_failure: failed_master_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_master_failure.sh |& ${TIMESTAMP}' @@ -348,6 +336,9 @@ special_testing_task: # necessary to execute them within a PR to validate changes. optional_testing_task: + depends_on: + - "gating" + # Only run system tests in PRs (not on merge) if magic string is present # in the PR description. Post-merge system testing is assumed to happen # later from OS distribution's build systems. @@ -426,10 +417,11 @@ verify_test_built_images_task: TEST_REMOTE_CLIENT: true TEST_REMOTE_CLIENT: false + # Note: A truncated form of normal testing. It only needs to confirm new images + # "probably" work. A full round of testing will happen again after $*_CACHE_IMAGE_NAME + # are updated in this or another PR (w/o '***CIRRUS: TEST IMAGES***'). environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}' - unit_test_script: '$SCRIPT_BASE/unit_test.sh |& ${TIMESTAMP}' integration_test_script: '$SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP}' - rootless_testing_script: '$SCRIPT_BASE/resetup_and_run_rootless.sh |& ${TIMESTAMP}' # Build new cache-images for future PR testing, but only after a PR merge. @@ -447,6 +439,7 @@ build_cache_images_task: depends_on: - "gating" - "testing" + - "rootless_testing" # VMs created by packer are not cleaned up by cirrus auto_cancellation: $CI != "true" @@ -462,6 +455,7 @@ build_cache_images_task: scopes: - compute - devstorage.full_control + environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}' build_vm_images_script: '$SCRIPT_BASE/build_vm_images.sh |& ${TIMESTAMP}' @@ -499,10 +493,12 @@ success_task: env: CIRRUS_WORKING_DIR: "/usr/src/libpod" + GOPATH: "/go" + GOSRC: "/go/src/github.com/containers/libpod" container: image: "quay.io/libpod/gate:latest" cpu: 1 memory: 1 - success_script: '$SCRIPT_BASE/success.sh |& ${TIMESTAMP}' + success_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/success.sh |& ${TIMESTAMP}' diff --git a/contrib/cirrus/build_vm_images.sh b/contrib/cirrus/build_vm_images.sh index cfdfd5d7d..773f8e486 100755 --- a/contrib/cirrus/build_vm_images.sh +++ b/contrib/cirrus/build_vm_images.sh @@ -3,7 +3,10 @@ set -e source $(dirname $0)/lib.sh -req_env_var CNI_COMMIT CRIO_COMMIT RUNC_COMMIT PACKER_BUILDS BUILT_IMAGE_SUFFIX CENTOS_BASE_IMAGE UBUNTU_BASE_IMAGE FEDORA_BASE_IMAGE FAH_BASE_IMAGE RHEL_BASE_IMAGE RHSM_COMMAND SERVICE_ACCOUNT GCE_SSH_USERNAME GCP_PROJECT_ID PACKER_VER SCRIPT_BASE PACKER_BASE +ENV_VARS='CNI_COMMIT CRIO_COMMIT RUNC_COMMIT PACKER_BUILDS BUILT_IMAGE_SUFFIX CENTOS_BASE_IMAGE UBUNTU_BASE_IMAGE FEDORA_BASE_IMAGE PRIOR_FEDORA_BASE_IMAGE FAH_BASE_IMAGE RHEL_BASE_IMAGE RHSM_COMMAND SERVICE_ACCOUNT GCE_SSH_USERNAME GCP_PROJECT_ID PACKER_VER SCRIPT_BASE PACKER_BASE' +req_env_var $ENV_VARS +# Must also be made available through make, into packer process +export $ENV_VARS show_env_vars diff --git a/contrib/cirrus/integration_test.sh b/contrib/cirrus/integration_test.sh index c7d381318..e65b3b8ec 100755 --- a/contrib/cirrus/integration_test.sh +++ b/contrib/cirrus/integration_test.sh @@ -9,7 +9,6 @@ cd "$GOSRC" if [[ "$SPECIALMODE" == "in_podman" ]] then - set -x ${CONTAINER_RUNTIME} run --rm --privileged --net=host \ -v $GOSRC:$GOSRC:Z \ --workdir $GOSRC \ @@ -32,24 +31,10 @@ then $GOSRC/$SCRIPT_BASE/rootless_test.sh exit $? else - set -x make make install PREFIX=/usr ETCDIR=/etc make test-binaries make install.tools - clean_env - - case "${OS_RELEASE_ID}-${OS_RELEASE_VER}" in - ubuntu-18) ;; - fedora-29) ;& # Continue to the next item - fedora-28) ;& - centos-7) ;& - rhel-7) - make podman-remote - install bin/podman-remote /usr/bin - ;; - *) bad_os_id_ver ;; - esac if [[ "$TEST_REMOTE_CLIENT" == "true" ]] then make remoteintegration diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh index fb1685104..7886d230b 100644 --- a/contrib/cirrus/lib.sh +++ b/contrib/cirrus/lib.sh @@ -4,22 +4,73 @@ # to be sourced by other scripts, not called directly. # Under some contexts these values are not set, make sure they are. -export USER="$(whoami)" -export HOME="$(getent passwd $USER | cut -d : -f 6)" - -# These are normally set by cirrus, but can't be for VMs setup by hack/get_ci_vm.sh -# Pick some reasonable defaults -ENVLIB=${ENVLIB:-.bash_profile} -CIRRUS_WORKING_DIR="${CIRRUS_WORKING_DIR:-/var/tmp/go/src/github.com/containers/libpod}" -GOSRC="${GOSRC:-$CIRRUS_WORKING_DIR}" +USER="$(whoami)" +HOME="$(getent passwd $USER | cut -d : -f 6)" +[[ -n "$UID" ]] || UID=$(getent passwd $USER | cut -d : -f 3) +GID=$(getent passwd $USER | cut -d : -f 4) + +# Essential default paths, many are overriden when executing under Cirrus-CI +export GOPATH="${GOPATH:-/var/tmp/go}" +if type -P go &> /dev/null +then + # required for go 1.12+ + export GOCACHE="${GOCACHE:-$HOME/.cache/go-build}" + eval "$(go env)" + # required by make and other tools + export $(go env | cut -d '=' -f 1) +fi +CIRRUS_WORKING_DIR="${CIRRUS_WORKING_DIR:-$GOPATH/src/github.com/containers/libpod}" +export GOSRC="${GOSRC:-$CIRRUS_WORKING_DIR}" +export PATH="$HOME/bin:$GOPATH/bin:/usr/local/bin:$PATH" +export LD_LIBRARY_PATH="/usr/local/lib${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}" +TIMESTAMPS_FILEPATH="${TIMESTAMPS_FILEPATH:-/var/tmp/timestamps}" +SETUP_MARKER_FILEPATH="${SETUP_MARKER_FILEPATH:-/var/tmp/.setup_environment_sh_complete}" +# Saves typing / in case location ever moves SCRIPT_BASE=${SCRIPT_BASE:-./contrib/cirrus} PACKER_BASE=${PACKER_BASE:-./contrib/cirrus/packer} -CIRRUS_BUILD_ID=${CIRRUS_BUILD_ID:-DEADBEEF} # a human -CIRRUS_BASE_SHA=${CIRRUS_BASE_SHA:-HEAD} -CIRRUS_CHANGE_IN_REPO=${CIRRUS_CHANGE_IN_REPO:-FETCH_HEAD} + +cd $GOSRC +if type -P git &> /dev/null +then + CIRRUS_CHANGE_IN_REPO=${CIRRUS_CHANGE_IN_REPO:-$(git show-ref --hash=8 HEAD || date +%s)} +else # pick something unique and obviously not from Cirrus + CIRRUS_CHANGE_IN_REPO=${CIRRUS_CHANGE_IN_REPO:-no_git_$(date +%s)} +fi + +# Defaults when not running under CI +export CI="${CI:-false}" +CIRRUS_CI="${CIRRUS_CI:-false}" +CONTINUOUS_INTEGRATION="${CONTINUOUS_INTEGRATION:-false}" +CIRRUS_REPO_NAME=${CIRRUS_REPO_NAME:-libpod} +CIRRUS_BASE_SHA=${CIRRUS_BASE_SHA:-unknown$(date +%s)} # difficult to reliably discover +CIRRUS_BUILD_ID=${CIRRUS_BUILD_ID:-$RANDOM$(date +%s)} # must be short and unique +# Vars. for image-building +PACKER_VER="1.3.5" +# CSV of cache-image names to build (see $PACKER_BASE/libpod_images.json) +PACKER_BUILDS="${PACKER_BUILDS:-ubuntu-18,fedora-29,fedora-28,rhel-7,centos-7}" + +# Base-images rarely change, define them here so they're out of the way. +# Google-maintained base-image names +UBUNTU_BASE_IMAGE="ubuntu-1804-bionic-v20181203a" +CENTOS_BASE_IMAGE="centos-7-v20181113" +# Manually produced base-image names (see $SCRIPT_BASE/README.md) +FEDORA_BASE_IMAGE="fedora-cloud-base-29-1-2-1541789245" +PRIOR_FEDORA_BASE_IMAGE="fedora-cloud-base-28-1-1-1544474897" +FAH_BASE_IMAGE="fedora-atomichost-29-20181025-1-1541787861" +# RHEL image must be imported, native image bills for subscription. +RHEL_BASE_IMAGE="rhel-guest-image-7-6-210-x86-64-qcow2-1548099756" +BUILT_IMAGE_SUFFIX="${BUILT_IMAGE_SUFFIX:--$CIRRUS_REPO_NAME-${CIRRUS_BUILD_ID}}" +RHSM_COMMAND="${RHSM_COMMAND:-/bin/true}" + +# Safe env. vars. to transfer from root -> $ROOTLESS_USER (go env handled separetly) +ROOTLESS_ENV_RE='(CIRRUS_.+)|(ROOTLESS_.+)|(.+_IMAGE.*)|(.+_BASE)|(.*DIRPATH)|(.*FILEPATH)|(SOURCE.*)|(DEPEND.*)|(.+_DEPS_.+)|(OS_REL.*)|(.+_ENV_RE)|(TRAVIS)|(CI.+)' +# Unsafe env. vars for display +SECRET_ENV_RE='(IRCID)|(RHSM)|(ACCOUNT)|(^GC[EP]..+)|(SSH)' + SPECIALMODE="${SPECIALMODE:-none}" +TEST_REMOTE_CLIENT="${TEST_REMOTE_CLIENT:-false}" export CONTAINER_RUNTIME=${CONTAINER_RUNTIME:-podman} - +# When running as root, this may be empty or not, as a user, it MUST be set. if [[ "$USER" == "root" ]] then ROOTLESS_USER="${ROOTLESS_USER:-}" @@ -27,34 +78,12 @@ else ROOTLESS_USER="${ROOTLESS_USER:-$USER}" fi -if ! [[ "$PATH" =~ "/usr/local/bin" ]] -then - export PATH="$PATH:/usr/local/bin" -fi - -# In ci/testing environment, ensure variables are always loaded -if [[ -r "$HOME/$ENVLIB" ]] && [[ -n "$CI" ]] -then - # Make sure this is always loaded - source "$HOME/$ENVLIB" -fi - -# Space separated list of environment variables to unset before testing -UNSET_ENV_VARS=' - GCP_PROJECT_ID GCE_SSH_USERNAME SERVICE_ACCOUNT RHSM_COMMAND BUILT_IMAGE_SUFFIX - IRCID RHEL_BASE_IMAGE FAH_BASE_IMAGE FEDORA_BASE_IMAGE CENTOS_BASE_IMAGE - UBUNTU_BASE_IMAGE PACKER_VER PACKER_BUILDS RUNC_COMMIT CRIU_COMMIT - CRIO_COMMIT CNI_COMMIT FEDORA_CNI_COMMIT PACKER_BASE SCRIPT_BASE - CIRRUS_SHELL CIRRUS_WORKING_DIR ENVLIB CIRRUS_CI - CI_NODE_INDEX CI_NODE_TOTAL CIRRUS_BASE_BRANCH CIRRUS_BASE_SHA - CIRRUS_BRANCH CIRRUS_BUILD_ID CIRRUS_CHANGE_IN_REPO CIRRUS_CLONE_DEPTH - CIRRUS_COMMIT_MESSAGE CIRRUS_CHANGE_MESSAGE CIRRUS_REPO_CLONE_HOST - CIRRUS_DEFAULT_BRANCH CIRRUS_PR CIRRUS_TAG CIRRUS_OS CIRRUS_TASK_NAME - CIRRUS_TASK_ID CIRRUS_REPO_NAME CIRRUS_REPO_OWNER CIRRUS_REPO_FULL_NAME - CIRRUS_REPO_CLONE_URL CIRRUS_SHELL CIRRUS_USER_COLLABORATOR CIRRUS_USER_PERMISSION - CIRRUS_WORKING_DIR CIRRUS_HTTP_CACHE_HOST PACKER_BUILDS - XDG_DATA_DIRS XDG_RUNTIME_DIR XDG_SESSION_ID ROOTLESS_USER -' +# GCE image-name compatible string representation of distribution name +OS_RELEASE_ID="$(egrep -m 1 '^ID=' /etc/os-release | cut -d = -f 2 | tr -d \' | tr -d \")" +# GCE image-name compatible string representation of distribution major version +OS_RELEASE_VER="$(egrep -m 1 '^VERSION_ID=' /etc/os-release | cut -d = -f 2 | tr -d \' | tr -d \" | cut -d '.' -f 1)" +# Combined to ease soe usage +OS_REL_VER="${OS_RELEASE_ID}-${OS_RELEASE_VER}" # Pass in a list of one or more envariable names; exit non-zero with # helpful error message if any value is empty @@ -81,81 +110,27 @@ req_env_var() { done } -# Some env. vars may contain secrets. Display values for known "safe" -# and useful variables. -# ref: https://cirrus-ci.org/guide/writing-tasks/#environment-variables show_env_vars() { - # This is almost always multi-line, print it separately - echo "export CIRRUS_CHANGE_MESSAGE=$CIRRUS_CHANGE_MESSAGE" - echo " -BUILDTAGS $BUILDTAGS -BUILT_IMAGE_SUFFIX $BUILT_IMAGE_SUFFIX -ROOTLESS_USER $ROOTLESS_USER -CI $CI -CIRRUS_CI $CIRRUS_CI -CI_NODE_INDEX $CI_NODE_INDEX -CI_NODE_TOTAL $CI_NODE_TOTAL -CONTINUOUS_INTEGRATION $CONTINUOUS_INTEGRATION -CIRRUS_BASE_BRANCH $CIRRUS_BASE_BRANCH -CIRRUS_BASE_SHA $CIRRUS_BASE_SHA -CIRRUS_BRANCH $CIRRUS_BRANCH -CIRRUS_BUILD_ID $CIRRUS_BUILD_ID -CIRRUS_CHANGE_IN_REPO $CIRRUS_CHANGE_IN_REPO -CIRRUS_CLONE_DEPTH $CIRRUS_CLONE_DEPTH -CIRRUS_DEFAULT_BRANCH $CIRRUS_DEFAULT_BRANCH -CIRRUS_PR $CIRRUS_PR -CIRRUS_TAG $CIRRUS_TAG -CIRRUS_OS $CIRRUS_OS -OS $OS -CIRRUS_TASK_NAME $CIRRUS_TASK_NAME -CIRRUS_TASK_ID $CIRRUS_TASK_ID -CIRRUS_REPO_NAME $CIRRUS_REPO_NAME -CIRRUS_REPO_OWNER $CIRRUS_REPO_OWNER -CIRRUS_REPO_FULL_NAME $CIRRUS_REPO_FULL_NAME -CIRRUS_REPO_CLONE_URL $CIRRUS_REPO_CLONE_URL -CIRRUS_SHELL $CIRRUS_SHELL -CIRRUS_USER_COLLABORATOR $CIRRUS_USER_COLLABORATOR -CIRRUS_USER_PERMISSION $CIRRUS_USER_PERMISSION -CIRRUS_WORKING_DIR $CIRRUS_WORKING_DIR -CIRRUS_HTTP_CACHE_HOST $CIRRUS_HTTP_CACHE_HOST -SPECIALMODE $SPECIALMODE -$(go env) -PACKER_BUILDS $PACKER_BUILDS - " | while read NAME VALUE + echo "Showing selection of environment variable definitions:" + _ENV_VAR_NAMES=$(awk 'BEGIN{for(v in ENVIRON) print v}' | \ + egrep -v "(^PATH$)|(^BASH_FUNC)|(^[[:punct:][:space:]]+)|$SECRET_ENV_RE" | \ + sort -u) + for _env_var_name in $_ENV_VAR_NAMES do - [[ -z "$NAME" ]] || echo "export $NAME=\"$VALUE\"" + # Supports older BASH versions + _value="$(printenv $_env_var_name)" + printf " ${_env_var_name}=%q\n" "${_value}" done echo "" echo "##### $(go version) #####" echo "" } -# Unset environment variables not needed for testing purposes -clean_env() { - req_env_var UNSET_ENV_VARS - echo "Unsetting $(echo $UNSET_ENV_VARS | wc -w) environment variables" - unset -v UNSET_ENV_VARS $UNSET_ENV_VARS || true # don't fail on read-only -} - die() { echo "${2:-FATAL ERROR (but no message given!) in ${FUNCNAME[1]}()}" exit ${1:-1} } -# Return a GCE image-name compatible string representation of distribution name -os_release_id() { - eval "$(egrep -m 1 '^ID=' /etc/os-release | tr -d \' | tr -d \")" - echo "$ID" -} -export OS_RELEASE_ID="$(os_release_id)" - -# Return a GCE image-name compatible string representation of distribution major version -os_release_ver() { - eval "$(egrep -m 1 '^VERSION_ID=' /etc/os-release | tr -d \' | tr -d \")" - echo "$VERSION_ID" | cut -d '.' -f 1 -} -export OS_RELEASE_VER="$(os_release_VER)" - bad_os_id_ver() { echo "Unknown/Unsupported distro. $OS_RELEASE_ID and/or version $OS_RELEASE_VER for $ARGS" exit 42 @@ -166,8 +141,8 @@ stub() { } ircmsg() { - req_env_var CIRRUS_TASK_ID - [[ -n "$*" ]] || die 9 "ircmsg() invoked without args" + req_env_var CIRRUS_TASK_ID MSG + [[ -n "$*" ]] || die 9 "ircmsg() invoked without message text argument" # Sometimes setup_environment.sh didn't run SCRIPT="$(dirname $0)/podbot.py" NICK="podbot_$CIRRUS_TASK_ID" @@ -179,12 +154,9 @@ ircmsg() { } setup_rootless() { - req_env_var ROOTLESS_USER GOSRC ENVLIB - - make install.catatonit - go get github.com/onsi/ginkgo/ginkgo - go get github.com/onsi/gomega/... + req_env_var ROOTLESS_USER GOSRC + # Only do this once if passwd --status $ROOTLESS_USER then echo "Updating $ROOTLESS_USER user permissions on possibly changed libpod code" @@ -192,12 +164,7 @@ setup_rootless() { return 0 fi - # Only do this once cd $GOSRC - make install.catatonit - go get github.com/onsi/ginkgo/ginkgo - go get github.com/onsi/gomega/... - # Guarantee independence from specific values ROOTLESS_UID=$[RANDOM+1000] ROOTLESS_GID=$[RANDOM+1000] @@ -223,19 +190,18 @@ setup_rootless() { echo "${ROOTLESS_USER}:$[ROOTLESS_UID * 100]:65536" | \ tee -a /etc/subuid >> /etc/subgid - echo "Copying $HOME/$ENVLIB" - install -o $ROOTLESS_USER -g $ROOTLESS_USER -m 0700 \ - "$HOME/$ENVLIB" "/home/$ROOTLESS_USER/$ENVLIB" - - # Allow the tests to run - echo "export ROOTLESS_USER=$ROOTLESS_USER" >> "/home/$ROOTLESS_USER/$ENVLIB" - - echo "Configuring user's go environment variables" - su --login --command 'go env' $ROOTLESS_USER | \ - while read envline - do - X=$(echo "export $envline" >> "/home/$ROOTLESS_USER/$ENVLIB") - done + # Env. vars set by Cirrus and setup_environment.sh must be explicitly + # transfered to the test-user. + echo "Configuring rootless user's environment variables:" + _ENV_VAR_NAMES=$(awk 'BEGIN{for(v in ENVIRON) print v}' | \ + egrep -v "(^PATH$)|(^BASH_FUNC)|(^[[:punct:][:space:]]+)|$SECRET_ENV_RE" | \ + egrep "$ROOTLESS_ENV_RE" | \ + sort -u) + for _env_var_name in $_ENV_VAR_NAMES + do + _value="$(printenv $_env_var_name)" + printf "${_env_var_name}=%q" "${_value}" | tee -a "/home/$ROOTLESS_USER/.bashrc" + done } # Helper/wrapper script to only show stderr/stdout on non-zero exit @@ -273,10 +239,7 @@ install_cni_plugins() { } install_runc_from_git(){ - req_env_var " - GOPATH $GOPATH - OS_RELEASE_ID $OS_RELEASE_ID - " + req_env_var GOPATH OS_RELEASE_ID RUNC_COMMIT wd=$(pwd) DEST="$GOPATH/src/github.com/opencontainers/runc" rm -rf "$DEST" @@ -295,7 +258,6 @@ install_runc_from_git(){ } install_runc(){ - OS_RELEASE_ID=$(os_release_id) echo "Installing RunC from commit $RUNC_COMMIT" echo "Platform is $OS_RELEASE_ID" req_env_var GOPATH RUNC_COMMIT OS_RELEASE_ID @@ -341,8 +303,6 @@ install_conmon(){ } install_criu(){ - OS_RELEASE_ID=$(os_release_id) - OS_RELEASE_VER=$(os_release_ver) echo "Installing CRIU" echo "Installing CRIU from commit $CRIU_COMMIT" echo "Platform is $OS_RELEASE_ID" @@ -381,16 +341,6 @@ EOF fi } -install_packer_copied_files(){ - # Install cni config, policy and registry config - sudo install -D -m 755 /tmp/libpod/cni/87-podman-bridge.conflist \ - /etc/cni/net.d/87-podman-bridge.conflist - sudo install -D -m 755 /tmp/libpod/test/policy.json \ - /etc/containers/policy.json - sudo install -D -m 755 /tmp/libpod/test/redhat_sigstore.yaml \ - /etc/containers/registries.d/registry.access.redhat.com.yaml -} - install_varlink() { echo "Installing varlink from the cheese-factory" ooe.sh sudo -H pip3 install varlink diff --git a/contrib/cirrus/notice_master_failure.sh b/contrib/cirrus/notice_master_failure.sh index 4b09331d3..1fc15cdf9 100755 --- a/contrib/cirrus/notice_master_failure.sh +++ b/contrib/cirrus/notice_master_failure.sh @@ -12,7 +12,7 @@ NOR="$(echo -n -e '\x0f')" if [[ "$CIRRUS_BRANCH" =~ "master" ]] then BURL="https://cirrus-ci.com/build/$CIRRUS_BUILD_ID" - ircmsg "${RED}[Action Recommended]: ${NOR}Post-merge testing ${RED}$CIRRUS_BRANCH failed${NOR} in $CIRRUS_TASK_NAME on $(os_release_id)-$(os_release_ver): $BURL. Please investigate, and re-run if appropriate." + ircmsg "${RED}[Action Recommended]: ${NOR}Post-merge testing ${RED}$CIRRUS_BRANCH failed${NOR} in $CIRRUS_TASK_NAME on $(OS_RELEASE_ID)-$(OS_RELEASE_VER): $BURL. Please investigate, and re-run if appropriate." fi # This script assumed to be executed on failure diff --git a/contrib/cirrus/packer/centos_setup.sh b/contrib/cirrus/packer/centos_setup.sh index f9728f2f7..f63625789 100644 --- a/contrib/cirrus/packer/centos_setup.sh +++ b/contrib/cirrus/packer/centos_setup.sh @@ -8,7 +8,7 @@ set -e # Load in library (copied by packer, before this script was run) source /tmp/libpod/$SCRIPT_BASE/lib.sh -req_env_var SCRIPT_BASE CNI_COMMIT CRIO_COMMIT CRIU_COMMIT +req_env_var SCRIPT_BASE CNI_COMMIT CRIO_COMMIT CRIU_COMMIT RUNC_COMMIT install_ooe @@ -77,8 +77,6 @@ install_cni_plugins install_buildah -install_packer_copied_files - sudo /tmp/libpod/hack/install_catatonit.sh rh_finalize diff --git a/contrib/cirrus/packer/fedora_setup.sh b/contrib/cirrus/packer/fedora_setup.sh index d9229c668..41d1785f9 100644 --- a/contrib/cirrus/packer/fedora_setup.sh +++ b/contrib/cirrus/packer/fedora_setup.sh @@ -60,6 +60,7 @@ ooe.sh sudo dnf install -y \ python3-dateutil \ python3-psutil \ python3-pytoml \ + runc \ skopeo-containers \ slirp4netns \ unzip \ @@ -71,15 +72,11 @@ install_varlink install_conmon -install_runc - CNI_COMMIT=$FEDORA_CNI_COMMIT install_cni_plugins install_buildah -install_packer_copied_files - sudo /tmp/libpod/hack/install_catatonit.sh rh_finalize # N/B: Halts system! diff --git a/contrib/cirrus/packer/rhel_setup.sh b/contrib/cirrus/packer/rhel_setup.sh index dee35a102..577c8721a 100644 --- a/contrib/cirrus/packer/rhel_setup.sh +++ b/contrib/cirrus/packer/rhel_setup.sh @@ -8,7 +8,7 @@ set -e # Load in library (copied by packer, before this script was run) source /tmp/libpod/$SCRIPT_BASE/lib.sh -req_env_var SCRIPT_BASE CNI_COMMIT CRIO_COMMIT CRIU_COMMIT RHSM_COMMAND +req_env_var SCRIPT_BASE CNI_COMMIT CRIO_COMMIT CRIU_COMMIT RHSM_COMMAND RUNC_COMMIT install_ooe @@ -87,8 +87,6 @@ install_cni_plugins install_buildah -install_packer_copied_files - sudo /tmp/libpod/hack/install_catatonit.sh rhel_exit_handler # release subscription! diff --git a/contrib/cirrus/packer/ubuntu_setup.sh b/contrib/cirrus/packer/ubuntu_setup.sh index d02b71e12..45a987df0 100644 --- a/contrib/cirrus/packer/ubuntu_setup.sh +++ b/contrib/cirrus/packer/ubuntu_setup.sh @@ -8,7 +8,7 @@ set -e # Load in library (copied by packer, before this script was run) source /tmp/libpod/$SCRIPT_BASE/lib.sh -req_env_var SCRIPT_BASE CNI_COMMIT CRIO_COMMIT CRIU_COMMIT RUNC_COMMIT +req_env_var SCRIPT_BASE CNI_COMMIT CRIO_COMMIT CRIU_COMMIT install_ooe @@ -26,6 +26,8 @@ ooe.sh sudo -E apt-get -qq install software-properties-common # Required to have Go 1.11 on Ubuntu 18.0.4 ooe.sh sudo -E add-apt-repository --yes ppa:longsleep/golang-backports +ooe.sh sudo -E add-apt-repository --yes ppa:projectatomic/ppa +ooe.sh sudo -E add-apt-repository --yes ppa:criu/ppa ooe.sh sudo -E apt-get -qq update || sudo -E apt-get -qq update ooe.sh sudo -E apt-get -qq install \ @@ -36,6 +38,8 @@ ooe.sh sudo -E apt-get -qq install \ bison \ btrfs-tools \ build-essential \ + cri-o-runc \ + criu \ curl \ e2fslibs-dev \ emacs-nox \ @@ -92,20 +96,15 @@ ooe.sh sudo update-grub install_conmon -install_runc - -install_criu - install_cni_plugins install_buildah -install_packer_copied_files - sudo /tmp/libpod/hack/install_catatonit.sh install_varlink +sudo mkdir -p /etc/containers sudo curl https://raw.githubusercontent.com/projectatomic/registries/master/registries.fedora\ -o /etc/containers/registries.conf diff --git a/contrib/cirrus/resetup_and_run_rootless.sh b/contrib/cirrus/resetup_and_run_rootless.sh deleted file mode 100755 index 1c387fdf5..000000000 --- a/contrib/cirrus/resetup_and_run_rootless.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash - -set -e - -# N/B: This script is only intended to be used for the special-case of -# setting up and executing the rootless tests AFTER normal tests complete -# while testing a freshly built image. - -source $(dirname $0)/lib.sh - -# must be after source lib.sh b/c it loads $ENVLIB -export ROOTLESS_USER="pilferingpirate$RANDOM" - -req_env_var " -CIRRUS_WORKING_DIR $CIRRUS_WORKING_DIR -GOSRC $GOSRC -SCRIPT_BASE $SCRIPT_BASE -ROOTLESS_USER $ROOTLESS_USER -" - -if ! run_rootless -then - die 86 "Error: Expected rootless env. var not set or empty" -fi - -cd $GOSRC -make clean -setup_rootless - -ssh $ROOTLESS_USER@localhost \ - -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o CheckHostIP=no \ - $CIRRUS_WORKING_DIR/$SCRIPT_BASE/rootless_test.sh diff --git a/contrib/cirrus/rootless_test.sh b/contrib/cirrus/rootless_test.sh index d4cc6965e..5a509c101 100755 --- a/contrib/cirrus/rootless_test.sh +++ b/contrib/cirrus/rootless_test.sh @@ -15,9 +15,8 @@ fi echo "." echo "Hello, my name is $USER and I live in $PWD can I be your friend?" -show_env_vars - export PODMAN_VARLINK_ADDRESS=unix:/tmp/podman-$(id -u) +show_env_vars cd "$GOSRC" make diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh index b3319a3e1..5fc90d61d 100755 --- a/contrib/cirrus/setup_environment.sh +++ b/contrib/cirrus/setup_environment.sh @@ -4,11 +4,22 @@ set -e source $(dirname $0)/lib.sh -req_env_var USER HOME ENVLIB SCRIPT_BASE CIRRUS_BUILD_ID +req_env_var USER HOME GOSRC SCRIPT_BASE SETUP_MARKER_FILEPATH -[[ "$SHELL" =~ "bash" ]] || chsh -s /bin/bash +# Ensure this script only executes successfully once and always logs ending timestamp +[[ ! -e "$SETUP_MARKER_FILEPATH" ]] || exit 0 +exithandler() { + RET=$? + set +e + show_env_vars + record_timestamp "env. setup end" + echo "$(basename $0) exit status: $RET" + [[ "$RET" -eq "0" ]] && date +%s >> "SETUP_MARKER_FILEPATH" +} +trap exithandler EXIT -cd "$CIRRUS_WORKING_DIR" # for clarity of initial conditions +# Must be bash, always bash +[[ "$SHELL" =~ "bash" ]] || chsh -s /bin/bash # Verify basic dependencies for depbin in go rsync unzip sha256sum curl make python3 git @@ -19,61 +30,48 @@ do fi done -# Setup env. vars common to all tasks/scripts/platforms and -# ensure they return for every following script execution. -MARK="# Added by $0, manual changes will be lost." -touch "$HOME/$ENVLIB" -if ! grep -q "$MARK" "$HOME/$ENVLIB" -then - cp "$HOME/$ENVLIB" "$HOME/${ENVLIB}_original" - # N/B: Single-quote items evaluated every time, double-quotes only once (right now). - for envstr in \ - "$MARK" \ - "export EPOCH_TEST_COMMIT=\"$CIRRUS_BASE_SHA\"" \ - "export HEAD=\"$CIRRUS_CHANGE_IN_REPO\"" \ - "export TRAVIS=\"1\"" \ - "export GOSRC=\"$CIRRUS_WORKING_DIR\"" \ - "export OS_RELEASE_ID=\"$(os_release_id)\"" \ - "export OS_RELEASE_VER=\"$(os_release_ver)\"" \ - "export OS_REL_VER=\"$(os_release_id)-$(os_release_ver)\"" \ - "export TEST_REMOTE_CLIENT=\"$TEST_REMOTE_CLIENT\"" \ - "export GOPATH=\"/var/tmp/go\"" \ - 'export PATH="$HOME/bin:$GOPATH/bin:/usr/local/bin:$PATH"' \ - 'export LD_LIBRARY_PATH="/usr/local/lib${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}"' - do - # Make permanent in later shells, and set in current shell - X=$(echo "$envstr" | tee -a "$HOME/$ENVLIB") && eval "$X" && echo "$X" - done +# Sometimes environment setup needs to vary between distros +# Note: This should only be used for environment variables, and temporary workarounds. +# Anything externally dependent, should be made fixed-in-time by adding to +# contrib/cirrus/packer/*_setup.sh to be incorporated into VM cache-images +# (see docs). +case "${OS_REL_VER}" in + ubuntu-18) ;; + fedora-29) ;; + fedora-28) ;; + centos-7) ;; + rhel-7) ;; + *) bad_os_id_ver ;; +esac - # Some environment setup needs to vary between distros - # Note: This should only be used for environment variables, and minor details. - # Anything that could vary from one run to the next, should go into - # contrib/cirrus/packer/*_setup.sh and be incorporated into VM cache-images - # (see docs) - case "${OS_RELEASE_ID}-${OS_RELEASE_VER}" in - ubuntu-18) ;; - fedora-29) ;; - fedora-28) ;; - centos-7) ;; - rhel-7) ;; - *) bad_os_id_ver ;; - esac +cd "${GOSRC}/" +# Reload to incorporate any changes from above +source "$SCRIPT_BASE/lib.sh" - cd "${GOSRC}/" - # Reload to incorporate any changes from above - source "$SCRIPT_BASE/lib.sh" +echo "Installing cni config, policy and registry config" +req_env_var GOSRC +sudo install -D -m 755 $GOSRC/cni/87-podman-bridge.conflist \ + /etc/cni/net.d/87-podman-bridge.conflist +sudo install -D -m 755 $GOSRC/test/policy.json \ + /etc/containers/policy.json +sudo install -D -m 755 $GOSRC/test/registries.conf \ + /etc/containers/registries.conf +# cri-o if installed will mess with testing in non-obvious ways +rm -f /etc/cni/net.d/*cri* - case "$SPECIALMODE" in - rootless) - X=$(echo "export ROOTLESS_USER='some${RANDOM}dude'" | \ - tee -a "$HOME/$ENVLIB") && eval "$X" && echo "$X" - setup_rootless - ;; - in_podman) # Assumed to be Fedora - dnf install -y podman buildah - $SCRIPT_BASE/setup_container_environment.sh - ;; - esac -fi +go get github.com/onsi/ginkgo/ginkgo +go get github.com/onsi/gomega/... -show_env_vars +case "$SPECIALMODE" in + rootless) + X=$(echo "export ROOTLESS_USER='some${RANDOM}dude'" | \ + tee -a "$HOME/$ENVLIB") && eval "$X" && echo "$X" + X=$(echo "export SPECIALMODE='$SPECIALMODE'"| \ + tee -a "$HOME/$ENVLIB") && eval "$X" && echo "$X" + setup_rootless + ;; + in_podman) # Assumed to be Fedora + dnf install -y podman buildah + $SCRIPT_BASE/setup_container_environment.sh + ;; +esac diff --git a/contrib/cirrus/system_test.sh b/contrib/cirrus/system_test.sh index dd5ef511d..0da170d6c 100755 --- a/contrib/cirrus/system_test.sh +++ b/contrib/cirrus/system_test.sh @@ -5,8 +5,6 @@ source $(dirname $0)/lib.sh req_env_var GOSRC OS_RELEASE_ID OS_RELEASE_VER -clean_env - set -x cd "$GOSRC" diff --git a/contrib/cirrus/unit_test.sh b/contrib/cirrus/unit_test.sh index 0e8c9e2e2..202663fb7 100755 --- a/contrib/cirrus/unit_test.sh +++ b/contrib/cirrus/unit_test.sh @@ -3,9 +3,7 @@ set -e source $(dirname $0)/lib.sh -req_env_var GOSRC OS_RELEASE_ID OS_RELEASE_VER - -clean_env +req_env_var GOSRC set -x cd "$GOSRC" diff --git a/test/registries.conf b/test/registries.conf index 6c9d39bbc..bb7072d45 100644 --- a/test/registries.conf +++ b/test/registries.conf @@ -1,5 +1,5 @@ [registries.search] -registries = ['docker.io', 'quay.io'] +registries = ['docker.io', 'quay.io', 'registry.fedoraproject.org'] [registries.insecure] registries = [] -- cgit v1.2.3-54-g00ecf From 95141f88d4e6f994c2972a63be19c82ef759ea5b Mon Sep 17 00:00:00 2001 From: Chris Evich Date: Tue, 30 Apr 2019 13:09:32 -0400 Subject: Cirrus: fixups based on review feedback Also remove disused distros (RHEL/CentOS/FAH) and fix get_ci_vm script Signed-off-by: Chris Evich --- .cirrus.yml | 25 +----- contrib/cirrus/README.md | 29 ++---- contrib/cirrus/build_vm_images.sh | 2 +- contrib/cirrus/integration_test.sh | 11 ++- contrib/cirrus/lib.sh | 77 +++++----------- contrib/cirrus/lib.sh.t | 17 ++-- contrib/cirrus/packer/Makefile | 15 +--- contrib/cirrus/packer/centos_setup.sh | 84 ------------------ contrib/cirrus/packer/fah_base-setup.sh | 45 ---------- contrib/cirrus/packer/fah_setup.sh | 21 ----- contrib/cirrus/packer/fedora_setup.sh | 2 +- contrib/cirrus/packer/libpod_base_images.yml | 53 +---------- contrib/cirrus/packer/libpod_images.yml | 18 ---- contrib/cirrus/packer/rhel_base-setup.sh | 78 ---------------- contrib/cirrus/packer/rhel_setup.sh | 96 -------------------- contrib/cirrus/rootless_test.sh | 7 +- contrib/cirrus/setup_environment.sh | 32 ++++--- contrib/cirrus/system_test.sh | 9 +- hack/get_ci_vm.sh | 127 ++++++++++++++++++--------- 19 files changed, 171 insertions(+), 577 deletions(-) delete mode 100644 contrib/cirrus/packer/centos_setup.sh delete mode 100644 contrib/cirrus/packer/fah_base-setup.sh delete mode 100644 contrib/cirrus/packer/fah_setup.sh delete mode 100644 contrib/cirrus/packer/rhel_base-setup.sh delete mode 100644 contrib/cirrus/packer/rhel_setup.sh (limited to 'contrib/cirrus/packer') diff --git a/.cirrus.yml b/.cirrus.yml index b3c6ff5a9..263593018 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -29,12 +29,9 @@ env: #### #### Cache-image names to test with ### - FEDORA_CACHE_IMAGE_NAME: "fedora-29-libpod-5699523102900224" - PRIOR_FEDORA_CACHE_IMAGE_NAME: "fedora-28-libpod-5699523102900224" - UBUNTU_CACHE_IMAGE_NAME: "ubuntu-18-libpod-5699523102900224" - PRIOR_RHEL_CACHE_IMAGE_NAME: "rhel-7-libpod-5699523102900224" - RHEL_CACHE_IMAGE_NAME: "rhel-8-notready" - CENTOS_CACHE_IMAGE_NAME: "centos-7-notready" + FEDORA_CACHE_IMAGE_NAME: "fedora-29-libpod-5170730531028992" + PRIOR_FEDORA_CACHE_IMAGE_NAME: "fedora-28-libpod-5170730531028992" + UBUNTU_CACHE_IMAGE_NAME: "ubuntu-18-libpod-5170730531028992" #### #### Variables for composing new cache-images (used in PR testing) from @@ -46,12 +43,6 @@ env: CNI_COMMIT: "7480240de9749f9a0a5c8614b17f1f03e0c06ab9" CRIO_COMMIT: "7a283c391abb7bd25086a8ff91dbb36ebdd24466" CRIU_COMMIT: "c74b83cd49c00589c0c0468ba5fe685b67fdbd0a" - # Only RHEL/Centos VM cache-image building needs RUNC_COMMIT - RUNC_COMMIT: "029124da7af7360afa781a0234d1b083550f797c" - # CSV of cache-image names to build (see $PACKER_BASE/libpod_images.json) - PACKER_BUILDS: "ubuntu-18,fedora-29,fedora-28,rhel-7" # TODO: rhel-8,centos-7 - # Version of packer to use - PACKER_VER: "1.3.2" # Special image w/ nested-libvirt + tools for creating new cache and base images IMAGE_BUILDER_CACHE_IMAGE_NAME: "image-builder-image-1541772081" @@ -66,8 +57,6 @@ env: #### # Freenode IRC credentials for posting status messages IRCID: ENCRYPTED[e87bba62a8e924dc70bdb2b66b16f6ab4a60d2870e6e5534ae9e2b0076f483c71c84091c655ca239101e6816c5ec0883] - # Command to register a RHEL VM to install/update packages - RHSM_COMMAND: ENCRYPTED[5caa5ff8c5370c3d25c7a1a28168501ab0fa2e5e3b627926f6eaba02b3fed965a7638a6151657809661f8c905c7dc187] # Needed to build GCE images, within a GCE VM SERVICE_ACCOUNT: ENCRYPTED[99e9a0b1c23f8dd29e83dfdf164f064cfd17afd9b895ca3b5e4c41170bd4290a8366fe2ad8e7a210b9f751711d1d002a] # User ID for cirrus to ssh into VMs @@ -244,10 +233,6 @@ meta_task: ${FEDORA_CACHE_IMAGE_NAME} ${PRIOR_FEDORA_CACHE_IMAGE_NAME} ${UBUNTU_CACHE_IMAGE_NAME} - ${PRIOR_RHEL_CACHE_IMAGE_NAME} - ${RHEL_CACHE_IMAGE_NAME} - ${PRIOR_CENTOS_CACHE_IMAGE_NAME} - ${CENTOS_CACHE_IMAGE_NAME} ${IMAGE_BUILDER_CACHE_IMAGE_NAME} BUILDID: "${CIRRUS_BUILD_ID}" REPOREF: "${CIRRUS_CHANGE_IN_REPO}" @@ -352,10 +337,6 @@ optional_testing_task: image_name: "${FEDORA_CACHE_IMAGE_NAME}" image_name: "${PRIOR_FEDORA_CACHE_IMAGE_NAME}" image_name: "${UBUNTU_CACHE_IMAGE_NAME}" - image_name: "${PRIOR_RHEL_CACHE_IMAGE_NAME}" - # image_name: "${RHEL_CACHE_IMAGE_NAME}" - image_name: "${PRIOR_CENTOS_CACHE_IMAGE_NAME}" - # image_name: "${CENTOS_CACHE_IMAGE_NAME}" timeout_in: 60m diff --git a/contrib/cirrus/README.md b/contrib/cirrus/README.md index 1d62f71ce..69d8653fe 100644 --- a/contrib/cirrus/README.md +++ b/contrib/cirrus/README.md @@ -136,11 +136,10 @@ you'll find the new image names displayed at the end of the `test_build_cache_images_task` in the `build_vm_images` output. For example: + ``` ...cut... ==> Builds finished. The artifacts of successful builds are: ---> rhel-7: A disk image was created: rhel-7-libpod-5699523102900224 ---> rhel-7: --> ubuntu-18: A disk image was created: ubuntu-18-libpod-5699523102900224 --> ubuntu-18: --> fedora-29: A disk image was created: fedora-29-libpod-5699523102900224 @@ -148,8 +147,8 @@ For example: --> fedora-28: A disk image was created: fedora-28-libpod-5699523102900224 ``` -An updated (or new) pull-request with this change, will utilize -the new cache-images: +Now edit `.cirrus.yml`, updating the `*_IMAGE_NAME` lines to reflect the +images from above: ```yaml @@ -161,14 +160,14 @@ env: FEDORA_CACHE_IMAGE_NAME: "fedora-29-libpod-5699523102900224" PRIOR_FEDORA_CACHE_IMAGE_NAME: "fedora-28-libpod-5699523102900224" UBUNTU_CACHE_IMAGE_NAME: "ubuntu-18-libpod-5699523102900224" - PRIOR_RHEL_CACHE_IMAGE_NAME: "rhel-7-libpod-5699523102900224" ...cut... ``` -Take care to also update the PR description if any 'magic' phrases were used -(they affect which tests run). In other words, you'll likely want to -remove the ``***CIRRUS: TEST IMAGES***`` string - otherwise Cirrus-CI will -simply build new and test again. +***NOTE:*** If re-using the same PR with new images in `.cirrus.yml`, +take care to also *update the PR description* to remove +the magic ``***CIRRUS: TEST IMAGES***`` string. Keeping it and +`--force` pushing would needlessly cause Cirrus-CI to build +and test images again. ### ``build_cache_images`` Task *(Deprecated)* @@ -216,15 +215,6 @@ the ``cache_images`` Task) some input parameters are required: or [end-user credentials](https://cloud.google.com/docs/authentication/end-user#creating_your_client_credentials) -* ``RHEL_IMAGE_FILE`` and ``RHEL_CSUM_FILE`` complete paths - to a `rhel-server-ec2-*.raw.xz` and it's cooresponding - checksum file. These must be supplied manually because - they're not available directly via URL like other images. - -* ``RHSM_COMMAND`` contains the complete string needed to register - the VM for installing package dependencies. The VM will be de-registered - upon completion. - * Optionally, CSV's may be specified to ``PACKER_BUILDS`` to limit the base-images produced. For example, ``PACKER_BUILDS=fedora,image-builder-image``. @@ -270,9 +260,6 @@ When ready, change to the ``packer`` sub-directory, and build the images: $ cd libpod/contrib/cirrus/packer $ make libpod_base_images GCP_PROJECT_ID= \ GOOGLE_APPLICATION_CREDENTIALS= \ - RHEL_IMAGE_FILE= \ - RHEL_CSUM_FILE= \ - RHSM_COMMAND= \ PACKER_BUILDS= ``` diff --git a/contrib/cirrus/build_vm_images.sh b/contrib/cirrus/build_vm_images.sh index 773f8e486..f4567ce74 100755 --- a/contrib/cirrus/build_vm_images.sh +++ b/contrib/cirrus/build_vm_images.sh @@ -3,7 +3,7 @@ set -e source $(dirname $0)/lib.sh -ENV_VARS='CNI_COMMIT CRIO_COMMIT RUNC_COMMIT PACKER_BUILDS BUILT_IMAGE_SUFFIX CENTOS_BASE_IMAGE UBUNTU_BASE_IMAGE FEDORA_BASE_IMAGE PRIOR_FEDORA_BASE_IMAGE FAH_BASE_IMAGE RHEL_BASE_IMAGE RHSM_COMMAND SERVICE_ACCOUNT GCE_SSH_USERNAME GCP_PROJECT_ID PACKER_VER SCRIPT_BASE PACKER_BASE' +ENV_VARS='CNI_COMMIT CRIO_COMMIT PACKER_BUILDS BUILT_IMAGE_SUFFIX UBUNTU_BASE_IMAGE FEDORA_BASE_IMAGE PRIOR_FEDORA_BASE_IMAGE SERVICE_ACCOUNT GCE_SSH_USERNAME GCP_PROJECT_ID PACKER_VER SCRIPT_BASE PACKER_BASE' req_env_var $ENV_VARS # Must also be made available through make, into packer process export $ENV_VARS diff --git a/contrib/cirrus/integration_test.sh b/contrib/cirrus/integration_test.sh index e65b3b8ec..e7f582b42 100755 --- a/contrib/cirrus/integration_test.sh +++ b/contrib/cirrus/integration_test.sh @@ -25,16 +25,19 @@ then elif [[ "$SPECIALMODE" == "rootless" ]] then req_env_var ROOTLESS_USER - set -x - ssh $ROOTLESS_USER@localhost \ + + if [[ "$USER" == "$ROOTLESS_USER" ]] + then + $GOSRC/$SCRIPT_BASE/rootless_test.sh + else + ssh $ROOTLESS_USER@localhost \ -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o CheckHostIP=no \ $GOSRC/$SCRIPT_BASE/rootless_test.sh - exit $? + fi else make make install PREFIX=/usr ETCDIR=/etc make test-binaries - make install.tools if [[ "$TEST_REMOTE_CLIENT" == "true" ]] then make remoteintegration diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh index 7886d230b..d0a2e1579 100644 --- a/contrib/cirrus/lib.sh +++ b/contrib/cirrus/lib.sh @@ -3,6 +3,9 @@ # Library of common, shared utility functions. This file is intended # to be sourced by other scripts, not called directly. +# Global details persist here +source /etc/environment # not always loaded under all circumstances + # Under some contexts these values are not set, make sure they are. USER="$(whoami)" HOME="$(getent passwd $USER | cut -d : -f 6)" @@ -18,6 +21,9 @@ then eval "$(go env)" # required by make and other tools export $(go env | cut -d '=' -f 1) + + # Ensure compiled tooling is reachable + export PATH="$PATH:$GOPATH/bin" fi CIRRUS_WORKING_DIR="${CIRRUS_WORKING_DIR:-$GOPATH/src/github.com/containers/libpod}" export GOSRC="${GOSRC:-$CIRRUS_WORKING_DIR}" @@ -47,29 +53,25 @@ CIRRUS_BUILD_ID=${CIRRUS_BUILD_ID:-$RANDOM$(date +%s)} # must be short and uniq # Vars. for image-building PACKER_VER="1.3.5" # CSV of cache-image names to build (see $PACKER_BASE/libpod_images.json) -PACKER_BUILDS="${PACKER_BUILDS:-ubuntu-18,fedora-29,fedora-28,rhel-7,centos-7}" # Base-images rarely change, define them here so they're out of the way. +PACKER_BUILDS="${PACKER_BUILDS:-ubuntu-18,fedora-29,fedora-28}" # Google-maintained base-image names UBUNTU_BASE_IMAGE="ubuntu-1804-bionic-v20181203a" -CENTOS_BASE_IMAGE="centos-7-v20181113" # Manually produced base-image names (see $SCRIPT_BASE/README.md) FEDORA_BASE_IMAGE="fedora-cloud-base-29-1-2-1541789245" PRIOR_FEDORA_BASE_IMAGE="fedora-cloud-base-28-1-1-1544474897" -FAH_BASE_IMAGE="fedora-atomichost-29-20181025-1-1541787861" -# RHEL image must be imported, native image bills for subscription. -RHEL_BASE_IMAGE="rhel-guest-image-7-6-210-x86-64-qcow2-1548099756" BUILT_IMAGE_SUFFIX="${BUILT_IMAGE_SUFFIX:--$CIRRUS_REPO_NAME-${CIRRUS_BUILD_ID}}" -RHSM_COMMAND="${RHSM_COMMAND:-/bin/true}" # Safe env. vars. to transfer from root -> $ROOTLESS_USER (go env handled separetly) ROOTLESS_ENV_RE='(CIRRUS_.+)|(ROOTLESS_.+)|(.+_IMAGE.*)|(.+_BASE)|(.*DIRPATH)|(.*FILEPATH)|(SOURCE.*)|(DEPEND.*)|(.+_DEPS_.+)|(OS_REL.*)|(.+_ENV_RE)|(TRAVIS)|(CI.+)' # Unsafe env. vars for display -SECRET_ENV_RE='(IRCID)|(RHSM)|(ACCOUNT)|(^GC[EP]..+)|(SSH)' +SECRET_ENV_RE='(IRCID)|(ACCOUNT)|(^GC[EP]..+)|(SSH)' SPECIALMODE="${SPECIALMODE:-none}" TEST_REMOTE_CLIENT="${TEST_REMOTE_CLIENT:-false}" export CONTAINER_RUNTIME=${CONTAINER_RUNTIME:-podman} + # When running as root, this may be empty or not, as a user, it MUST be set. if [[ "$USER" == "root" ]] then @@ -79,9 +81,9 @@ else fi # GCE image-name compatible string representation of distribution name -OS_RELEASE_ID="$(egrep -m 1 '^ID=' /etc/os-release | cut -d = -f 2 | tr -d \' | tr -d \")" -# GCE image-name compatible string representation of distribution major version -OS_RELEASE_VER="$(egrep -m 1 '^VERSION_ID=' /etc/os-release | cut -d = -f 2 | tr -d \' | tr -d \" | cut -d '.' -f 1)" +OS_RELEASE_ID="$(source /etc/os-release; echo $ID)" +# GCE image-name compatible string representation of distribution _major_ version +OS_RELEASE_VER="$(source /etc/os-release; echo $VERSION_ID | cut -d '.' -f 1)" # Combined to ease soe usage OS_REL_VER="${OS_RELEASE_ID}-${OS_RELEASE_VER}" @@ -118,8 +120,7 @@ show_env_vars() { for _env_var_name in $_ENV_VAR_NAMES do # Supports older BASH versions - _value="$(printenv $_env_var_name)" - printf " ${_env_var_name}=%q\n" "${_value}" + printf " ${_env_var_name}=%q\n" "$(printenv $_env_var_name)" done echo "" echo "##### $(go version) #####" @@ -127,12 +128,14 @@ show_env_vars() { } die() { - echo "${2:-FATAL ERROR (but no message given!) in ${FUNCNAME[1]}()}" + echo "************************************************" + echo ">>>>> ${2:-FATAL ERROR (but no message given!) in ${FUNCNAME[1]}()}" + echo "************************************************" exit ${1:-1} } bad_os_id_ver() { - echo "Unknown/Unsupported distro. $OS_RELEASE_ID and/or version $OS_RELEASE_VER for $ARGS" + echo "Unknown/Unsupported distro. $OS_RELEASE_ID and/or version $OS_RELEASE_VER for $(basename $0)" exit 42 } @@ -141,7 +144,7 @@ stub() { } ircmsg() { - req_env_var CIRRUS_TASK_ID MSG + req_env_var CIRRUS_TASK_ID IRCID [[ -n "$*" ]] || die 9 "ircmsg() invoked without message text argument" # Sometimes setup_environment.sh didn't run SCRIPT="$(dirname $0)/podbot.py" @@ -193,14 +196,15 @@ setup_rootless() { # Env. vars set by Cirrus and setup_environment.sh must be explicitly # transfered to the test-user. echo "Configuring rootless user's environment variables:" + echo "# Added by $GOSRC/$SCRIPT_PATH/lib.sh setup_rootless()" _ENV_VAR_NAMES=$(awk 'BEGIN{for(v in ENVIRON) print v}' | \ egrep -v "(^PATH$)|(^BASH_FUNC)|(^[[:punct:][:space:]]+)|$SECRET_ENV_RE" | \ egrep "$ROOTLESS_ENV_RE" | \ sort -u) for _env_var_name in $_ENV_VAR_NAMES do - _value="$(printenv $_env_var_name)" - printf "${_env_var_name}=%q" "${_value}" | tee -a "/home/$ROOTLESS_USER/.bashrc" + # Works with older versions of bash + printf "${_env_var_name}=%q\n" "$(printenv $_env_var_name)" >> "/home/$ROOTLESS_USER/.bashrc" done } @@ -312,21 +316,6 @@ install_criu(){ ooe.sh sudo -E add-apt-repository -y ppa:criu/ppa ooe.sh sudo -E apt-get -qq -y update ooe.sh sudo -E apt-get -qq -y install criu - elif [[ ( "$OS_RELEASE_ID" =~ "centos" || "$OS_RELEASE_ID" =~ "rhel" ) && "$OS_RELEASE_VER" =~ "7"* ]]; then - echo "Configuring Repositories for latest CRIU" - ooe.sh sudo tee /etc/yum.repos.d/adrian-criu-el7.repo < "$RHSMCMD" - ooe.sh sudo bash "$RHSMCMD" - sudo rm -rf "$RHSMCMD" -} diff --git a/contrib/cirrus/lib.sh.t b/contrib/cirrus/lib.sh.t index ce51f8ad2..1f05b3bb5 100755 --- a/contrib/cirrus/lib.sh.t +++ b/contrib/cirrus/lib.sh.t @@ -10,12 +10,13 @@ rc=0 function check_result { testnum=$(expr $testnum + 1) - if [ "$1" = "$2" ]; then - echo "ok $testnum $3 = $1" + MSG=$(echo "$1" | tr -d '*>\012'|sed -e 's/^ \+//') + if [ "$MSG" = "$2" ]; then + echo "ok $testnum $3 = $MSG" else echo "not ok $testnum $3" echo "# expected: $2" - echo "# actual: $1" + echo "# actual: $MSG" rc=1 fi } @@ -64,11 +65,17 @@ test_rev '' 1 'FATAL: req_env_var: invoked without arguments' unset FOO BAR test_rev FOO 9 'FATAL: test_rev() requires $FOO to be non-empty' test_rev BAR 9 'FATAL: test_rev() requires $BAR to be non-empty' - -# OK if desired envariable is unset +# OK if desired envariable was unset FOO=1 test_rev FOO 0 '' +# OK if multiple vars are non-empty +FOO="stuff" +BAR="things" +ENV_VARS="FOO BAR" +test_rev "$ENV_VARS" 0 '' +unset BAR + # ...but error if any single desired one is unset test_rev "FOO BAR" 9 'FATAL: test_rev() requires $BAR to be non-empty' diff --git a/contrib/cirrus/packer/Makefile b/contrib/cirrus/packer/Makefile index 91a1dfeef..0ae0deca6 100644 --- a/contrib/cirrus/packer/Makefile +++ b/contrib/cirrus/packer/Makefile @@ -49,6 +49,7 @@ ifndef PACKER_BUILDS $(error PACKER_BUILDS is undefined, expected builder-names CSV) endif ./packer build -only=${PACKER_BUILDS} \ + -force \ -var GOSRC=$(GOSRC) \ -var PACKER_BASE=$(PACKER_BASE) \ -var SCRIPT_BASE=$(SCRIPT_BASE) \ @@ -77,17 +78,9 @@ ifndef GCP_PROJECT_ID endif ifndef GOOGLE_APPLICATION_CREDENTIALS $(error GOOGLE_APPLICATION_CREDENTIALS is undefined, expected absolute path to JSON file, like $HOME/.config/gcloud/legacy_credentials/*/adc.json) -endif -ifndef RHEL_IMAGE_FILE - $(error RHEL_IMAGE_FILE is undefined, expected full path to a rhel-server-ec2-*.raw.xz file) -endif -ifndef RHEL_CSUM_FILE - $(error RHEL_CSUM_FILE is undefined, expected full path to a rhel-server-ec2-*.raw.xz.SHA256SUM file) -endif -ifndef RHSM_COMMAND - $(error RHSM_COMMAND is undefined, expected string required for temporarily registering VM) endif PACKER_CACHE_DIR=/tmp ./packer build \ + -force \ -var TIMESTAMP=$(TIMESTAMP) \ -var TTYDEV=$(TTYDEV) \ -var GCP_PROJECT_ID=$(GCP_PROJECT_ID) \ @@ -95,10 +88,6 @@ endif -var GOSRC=$(GOSRC) \ -var PACKER_BASE=$(PACKER_BASE) \ -var SCRIPT_BASE=$(SCRIPT_BASE) \ - -var RHEL_BASE_IMAGE_NAME=$(shell basename $(RHEL_IMAGE_FILE) | tr -d '[[:space:]]' | sed -r -e 's/\.${ARCH}\.raw\.xz//' | tr '[[:upper:]]' '[[:lower:]]' | tr '[[:punct:]]' '-') \ - -var RHEL_IMAGE_FILE=$(RHEL_IMAGE_FILE) \ - -var RHEL_CSUM_FILE=$(RHEL_CSUM_FILE) \ - -var 'RHSM_COMMAND=$(RHSM_COMMAND)' \ -var POST_MERGE_BUCKET_SUFFIX=$(POST_MERGE_BUCKET_SUFFIX) \ -only $(PACKER_BUILDS) \ libpod_base_images.json diff --git a/contrib/cirrus/packer/centos_setup.sh b/contrib/cirrus/packer/centos_setup.sh deleted file mode 100644 index f63625789..000000000 --- a/contrib/cirrus/packer/centos_setup.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/bin/bash - -# This script is called by packer on the subject CentOS VM, to setup the podman -# build/test environment. It's not intended to be used outside of this context. - -set -e - -# Load in library (copied by packer, before this script was run) -source /tmp/libpod/$SCRIPT_BASE/lib.sh - -req_env_var SCRIPT_BASE CNI_COMMIT CRIO_COMMIT CRIU_COMMIT RUNC_COMMIT - -install_ooe - -export GOPATH="$(mktemp -d)" -trap "sudo rm -rf $GOPATH" EXIT - -ooe.sh sudo yum -y update - -ooe.sh sudo yum -y install centos-release-scl epel-release - -ooe.sh sudo yum -y install \ - PyYAML \ - atomic-registries \ - bats \ - btrfs-progs-devel \ - bzip2 \ - device-mapper-devel \ - emacs-nox \ - findutils \ - glib2-devel \ - glibc-static \ - gnupg \ - golang \ - golang-github-cpuguy83-go-md2man \ - golang-github-cpuguy83-go-md2man \ - gpgme-devel \ - iptables \ - jq \ - libassuan-devel \ - libcap-devel \ - libnet \ - libnet-devel \ - libnl3-devel \ - libseccomp-devel \ - libselinux-devel \ - lsof \ - make \ - nmap-ncat \ - ostree-devel \ - protobuf \ - protobuf-c \ - protobuf-c-devel \ - protobuf-compiler \ - protobuf-devel \ - protobuf-python \ - python \ - python2-future \ - python3-dateutil \ - python3-psutil \ - python3-pytoml \ - skopeo-containers \ - unzip \ - vim \ - which \ - xz - -install_scl_git - -install_conmon - -install_runc - -install_criu - -install_cni_plugins - -install_buildah - -sudo /tmp/libpod/hack/install_catatonit.sh - -rh_finalize - -echo "SUCCESS!" diff --git a/contrib/cirrus/packer/fah_base-setup.sh b/contrib/cirrus/packer/fah_base-setup.sh deleted file mode 100644 index 606c4f336..000000000 --- a/contrib/cirrus/packer/fah_base-setup.sh +++ /dev/null @@ -1,45 +0,0 @@ - -# N/B: This script is not intended to be run by humans. It is used to configure the -# FAH base image for importing, so that it will boot in GCE. - -set -e - -# Load in library (copied by packer, before this script was run) -source $GOSRC/$SCRIPT_BASE/lib.sh - -install_ooe - -if [[ "$1" == "pre" ]] -then - echo "Upgrading Atomic Host" - setenforce 0 - ooe.sh atomic host upgrade - - echo "Configuring Repositories" - ooe.sh sudo tee /etc/yum.repos.d/ngompa-gce-oslogin.repo <> "SETUP_MARKER_FILEPATH" } trap exithandler EXIT -# Must be bash, always bash -[[ "$SHELL" =~ "bash" ]] || chsh -s /bin/bash - # Verify basic dependencies for depbin in go rsync unzip sha256sum curl make python3 git do @@ -39,8 +35,10 @@ case "${OS_REL_VER}" in ubuntu-18) ;; fedora-29) ;; fedora-28) ;; - centos-7) ;; - rhel-7) ;; + centos-7) # Current VM is an image-builder-image no local podman/testing + echo "No further setup required for VM image building" + exit 0 + ;; *) bad_os_id_ver ;; esac @@ -59,19 +57,27 @@ sudo install -D -m 755 $GOSRC/test/registries.conf \ # cri-o if installed will mess with testing in non-obvious ways rm -f /etc/cni/net.d/*cri* -go get github.com/onsi/ginkgo/ginkgo -go get github.com/onsi/gomega/... +make install.tools case "$SPECIALMODE" in + none) ;; # Do the normal thing rootless) - X=$(echo "export ROOTLESS_USER='some${RANDOM}dude'" | \ - tee -a "$HOME/$ENVLIB") && eval "$X" && echo "$X" - X=$(echo "export SPECIALMODE='$SPECIALMODE'"| \ - tee -a "$HOME/$ENVLIB") && eval "$X" && echo "$X" - setup_rootless + # Only do this once, even if ROOTLESS_USER (somehow) changes + if ! grep -q 'ROOTLESS_USER' /etc/environment + then + X=$(echo "export ROOTLESS_USER='${ROOTLESS_USER:-some${RANDOM}dude}'" | \ + tee -a /etc/environment) && eval "$X" && echo "$X" + X=$(echo "export SPECIALMODE='${SPECIALMODE}'" | \ + tee -a /etc/environment) && eval "$X" && echo "$X" + X=$(echo "export TEST_REMOTE_CLIENT='${TEST_REMOTE_CLIENT}'" | \ + tee -a /etc/environment) && eval "$X" && echo "$X" + setup_rootless + fi ;; in_podman) # Assumed to be Fedora dnf install -y podman buildah $SCRIPT_BASE/setup_container_environment.sh ;; + *) + die 111 "Unsupported \$SPECIAL_MODE: $SPECIALMODE" esac diff --git a/contrib/cirrus/system_test.sh b/contrib/cirrus/system_test.sh index 0da170d6c..a2cc1af05 100755 --- a/contrib/cirrus/system_test.sh +++ b/contrib/cirrus/system_test.sh @@ -8,12 +8,9 @@ req_env_var GOSRC OS_RELEASE_ID OS_RELEASE_VER set -x cd "$GOSRC" -case "${OS_RELEASE_ID}-${OS_RELEASE_VER}" in - ubuntu-18) ;& # Continue to the next item - fedora-28) ;& - fedora-29) ;& - centos-7) ;& - rhel-7) +case "${OS_RELEASE_ID}" in + ubuntu) ;& # Continue to the next item + fedora) make install.tools make make test-binaries diff --git a/hack/get_ci_vm.sh b/hack/get_ci_vm.sh index d0325b8ae..370cd8a5e 100755 --- a/hack/get_ci_vm.sh +++ b/hack/get_ci_vm.sh @@ -19,7 +19,6 @@ PROJECT="libpod-218412" GOSRC="/var/tmp/go/src/github.com/containers/libpod" GCLOUD_IMAGE=${GCLOUD_IMAGE:-quay.io/cevich/gcloud_centos:latest} GCLOUD_SUDO=${GCLOUD_SUDO-sudo} -ROOTLESS_USER="madcowdog" # Shared tmp directory between container and us TMPDIR=$(mktemp -d --tmpdir $(basename $0)_tmpdir_XXXXXX) @@ -48,11 +47,15 @@ showrun() { } cleanup() { + RET=$? set +e wait # set GCLOUD_DEBUG to leave tmpdir behind for postmortem test -z "$GCLOUD_DEBUG" && rm -rf $TMPDIR + + # Not always called from an exit handler, but should always exit when called + exit $RET } trap cleanup EXIT @@ -67,14 +70,15 @@ delvm() { image_hints() { egrep '[[:space:]]+[[:alnum:]].+_CACHE_IMAGE_NAME:[[:space:]+"[[:print:]]+"' \ "$LIBPODROOT/.cirrus.yml" | cut -d: -f 2 | tr -d '"[:blank:]' | \ - grep -v 'notready' | grep -v 'image-builder' | sort -u + grep -v 'notready' | sort -u } show_usage() { echo -e "\n${RED}ERROR: $1${NOR}" - echo -e "${YEL}Usage: $(basename $0) [-s | -p | -r] ${NOR}" - echo "Use -s / -p to select source or package based dependencies" - echo -e "Use -r to setup and run tests as a regular user.\n" + echo -e "${YEL}Usage: $(basename $0) [-m ] [-u ] ${NOR}" + echo "Use -m with a supported value documented in contrib/cirrus/README.md." + echo "With '-m rootless' must also specify -u with name of user to create & use" + echo "" if [[ -r ".cirrus.yml" ]] then echo -e "${YEL}Some possible image_name values (from .cirrus.yml):${NOR}" @@ -87,7 +91,7 @@ show_usage() { get_env_vars() { python -c ' import yaml -env=yaml.load(open(".cirrus.yml"))["env"] +env=yaml.load(open(".cirrus.yml"), Loader=yaml.SafeLoader)["env"] keys=[k for k in env if "ENCRYPTED" not in str(env[k])] for k,v in env.items(): v=str(v) @@ -99,28 +103,56 @@ for k,v in env.items(): parse_args(){ echo -e "$USAGE_WARNING" - if [[ -z "$1" ]] + if [[ "$USER" =~ "root" ]] then + show_usage "This script must be run as a regular user." + fi + + ENVS="$(get_env_vars)" + [[ "$#" -ge "1" ]] || \ show_usage "Must specify at least one command-line parameter." - elif [[ "$1" == "-p" ]] - then - echo -e "${YEL}Hint: Use -p for package-based dependencies or -s for source-based.${NOR}" - DEPS="PACKAGE_DEPS=true SOURCE_DEPS=false" - IMAGE_NAME="$2" - elif [[ "$1" == "-s" ]] + IMAGE_NAME="" + ROOTLESS_USER="" + SPECIALMODE="none" + for arg + do + if [[ "$SPECIALMODE" == "GRABNEXT" ]] && [[ "${arg:0:1}" != "-" ]] + then + SPECIALMODE="$arg" + echo -e "${YEL}Using \$SPECIALMODE=$SPECIALMODE.${NOR}" + continue + elif [[ "$ROOTLESS_USER" == "GRABNEXT" ]] && [[ "${arg:0:1}" != "-" ]] + then + ROOTLESS_USER="$arg" + echo -e "${YEL}Using \$ROOTLESS_USER=$ROOTLESS_USER.${NOR}" + continue + fi + case "$arg" in + -m) + SPECIALMODE="GRABNEXT" + ;; + -u) + ROOTLESS_USER="GRABNEXT" + ;; + *) + [[ "${arg:0:1}" != "-" ]] || \ + show_usage "Unknown command-line option '$arg'." + [[ -z "$IMAGE_NAME" ]] || \ + show_usage "Must specify exactly one image name, got '$IMAGE_NAME' and '$arg'." + IMAGE_NAME="$arg" + ;; + esac + done + + if [[ "$SPECIALMODE" == "GRABNEXT" ]] then - echo -e "${RED}Using source-based dependencies.${NOR}" - DEPS="PACKAGE_DEPS=false SOURCE_DEPS=true" - IMAGE_NAME="$2" - elif [[ "$1" == "-r" ]] + show_usage "Must specify argument to -m option." + fi + + if [[ "$ROOTLESS_USER" == "GRABNEXT" ]] then - DEPS="ROOTLESS_USER=$ROOTLESS_USER" - IMAGE_NAME="$2" - else # no -s or -p - echo -e "${RED}Using package-based dependencies.${NOR}" - DEPS="$(get_env_vars)" - IMAGE_NAME="$1" + show_usage "Must specify argument to -u option." fi if [[ -z "$IMAGE_NAME" ]] @@ -128,15 +160,33 @@ parse_args(){ show_usage "No image-name specified." fi - if [[ "$USER" =~ "root" ]] + if [[ "$SPECIALMODE" == "rootless" ]] && [[ -z "$ROOTLESS_USER" ]] then - show_usage "This script must be run as a regular user." + show_usage "With '-m rootless' must also pass -u of rootless user." + fi + + if echo "$IMAGE_NAME" | grep -q "image-builder-image" + then + echo -e "Creating an image-builder VM, I hope you know what you're doing.\n" + IBI_ARGS="--scopes=compute-rw,storage-rw,userinfo-email \"--min-cpu-platform=Intel Haswell\"" + SSHUSER="centos" + else + unset IBI_ARGS + SSHUSER="root" fi - SETUP_CMD="env $DEPS $GOSRC/contrib/cirrus/setup_environment.sh" + ENVS="$ENVS SPECIALMODE=\"$SPECIALMODE\"" + + [[ -z "$ROOTLESS_USER" ]] || \ + ENVS="$ENVS ROOTLESS_USER=$ROOTLESS_USER" + + SETUP_CMD="env $ENVS $GOSRC/contrib/cirrus/setup_environment.sh" VMNAME="${VMNAME:-${USER}-${IMAGE_NAME}}" - CREATE_CMD="$PGCLOUD compute instances create --zone=$ZONE --image=${IMAGE_NAME} --custom-cpu=$CPUS --custom-memory=$MEMORY --boot-disk-size=$DISK --labels=in-use-by=$USER $VMNAME" - SSH_CMD="$PGCLOUD compute ssh root@$VMNAME" + + CREATE_CMD="$PGCLOUD compute instances create --zone=$ZONE --image=${IMAGE_NAME} --custom-cpu=$CPUS --custom-memory=$MEMORY --boot-disk-size=$DISK --labels=in-use-by=$USER $IBI_ARGS $VMNAME" + + SSH_CMD="$PGCLOUD compute ssh $SSHUSER@$VMNAME" + CLEANUP_CMD="$PGCLOUD compute instances delete --zone $ZONE --delete-disks=all $VMNAME" } @@ -147,7 +197,7 @@ parse_args(){ cd "$LIBPODROOT" -parse_args $@ +parse_args "$@" # Ensure mount-points and data directories exist on host as $USER. Also prevents # permission-denied errors during cleanup() b/c `sudo podman` created mount-points @@ -188,7 +238,7 @@ showrun --background tar cjf $TMPDIR/$TARBALL --warning=no-file-changed -C $LIBP trap delvm INT # Allow deleting VM if CTRL-C during create # This fails if VM already exists: permit this usage to re-init -echo -e "\n${YEL}Trying to creating a VM named $VMNAME ${RED}(might take a minute/two. Errors ignored).${NOR}" +echo -e "\n${YEL}Trying to creating a VM named $VMNAME\n${RED}(might take a minute/two. Errors ignored).${NOR}" showrun $CREATE_CMD || true # allow re-running commands below when "delete: N" # Any subsequent failure should prompt for VM deletion @@ -208,19 +258,13 @@ then fi echo -e "${YEL}Got it${NOR}" -if $SSH_CMD --command "test -r /root/.bash_profile_original" -then - echo -e "\n${YEL}Resetting environment configuration${NOR}" - showrun $SSH_CMD --command "cp /root/.bash_profile_original /root/.bash_profile" -fi - echo -e "\n${YEL}Removing and re-creating $GOSRC on $VMNAME.${NOR}" showrun $SSH_CMD --command "rm -rf $GOSRC" showrun $SSH_CMD --command "mkdir -p $GOSRC" echo -e "\n${YEL}Transfering tarball to $VMNAME.${NOR}" wait -showrun $SCP_CMD $HOME/$TARBALL root@$VMNAME:/tmp/$TARBALL +showrun $SCP_CMD $HOME/$TARBALL $SSHUSER@$VMNAME:/tmp/$TARBALL echo -e "\n${YEL}Unpacking tarball into $GOSRC on $VMNAME.${NOR}" showrun $SSH_CMD --command "tar xjf /tmp/$TARBALL -C $GOSRC" @@ -231,9 +275,12 @@ showrun $SSH_CMD --command "rm -f /tmp/$TARBALL" echo -e "\n${YEL}Executing environment setup${NOR}" showrun $SSH_CMD --command "$SETUP_CMD" -echo -e "\n${YEL}Connecting to $VMNAME ${RED}(option to delete VM upon logout).${NOR}\n" -if [[ "$1" == "-r" ]] +echo -e "\n${YEL}Connecting to $VMNAME\n${RED}(option to delete VM upon logout).${NOR}\n" +if [[ -n "$ROOTLESS_USER" ]] then + echo "Re-chowning source files after transfer" + showrun $SSH_CMD --command "chown -R $ROOTLESS_USER $GOSRC" + echo "Connecting as user $ROOTLESS_USER" SSH_CMD="$PGCLOUD compute ssh $ROOTLESS_USER@$VMNAME" fi -showrun $SSH_CMD -- -t "cd $GOSRC && exec env $DEPS bash -il" +showrun $SSH_CMD -- -t "cd $GOSRC && exec env $ENVS bash -il" -- cgit v1.2.3-54-g00ecf From 1a2055ffac8b3673e0d1a4062ae64a87cb4f404f Mon Sep 17 00:00:00 2001 From: Chris Evich Date: Thu, 2 May 2019 14:38:59 -0400 Subject: Cirrus: Stub in F30 support New base-image boots, a cache-image builds, but more work is needed for it to be prime-time ready. This commit just adds some updates to the scafolding necessary to build the base-image. Future work will make F30 more of a reality. Also add log-collection scripts to test image verification task Signed-off-by: Chris Evich --- .cirrus.yml | 17 +++++++++++++++++ contrib/cirrus/lib.sh | 2 ++ contrib/cirrus/packer/Makefile | 6 ++++-- contrib/cirrus/packer/fedora_base-setup.sh | 5 ++--- contrib/cirrus/packer/libpod_base_images.yml | 13 +++++++------ 5 files changed, 32 insertions(+), 11 deletions(-) (limited to 'contrib/cirrus/packer') diff --git a/.cirrus.yml b/.cirrus.yml index 263593018..32b76ba79 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -24,6 +24,8 @@ env: # Command to prefix every output line with a timestamp # (can't do inline awk script, Cirrus-CI or YAML mangles quoting) TIMESTAMP: "awk --file ${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/timestamp.awk" + # Command to log critical filesystems, types, and sizes. + DFCMD: "df -lhTx tmpfs" CIRRUS_CLONE_DEPTH: 50 #### @@ -276,6 +278,7 @@ testing_task: unit_test_script: '$SCRIPT_BASE/unit_test.sh |& ${TIMESTAMP}' integration_test_script: '$SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP}' ginkgo_node_logs_script: 'cat $CIRRUS_WORKING_DIR/test/e2e/ginkgo-node-*.log || echo "Ginkgo node logs not found"' + df_script: '${DFCMD}' audit_log_script: 'cat /var/log/audit/audit.log || cat /var/log/kern.log' journalctl_b_script: 'journalctl -b' @@ -283,6 +286,7 @@ testing_task: failed_master_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_master_failure.sh' # Job has already failed, don't fail again and miss collecting data failed_ginkgo_node_logs_script: 'cat $CIRRUS_WORKING_DIR/test/e2e/ginkgo-node-*.log || echo "Ginkgo node logs not found"' + failed_df_script: '${DFCMD}' failed_audit_log_script: 'cat /var/log/audit/audit.log || cat /var/log/kern.log || echo "Uh oh, cat audit.log failed"' failed_journalctl_b_script: 'journalctl -b || echo "Uh oh, journalctl -b failed"' @@ -307,12 +311,14 @@ special_testing_task: setup_environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}' integration_test_script: '$SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP}' + df_script: '${DFCMD}' audit_log_script: 'cat /var/log/audit/audit.log || cat /var/log/kern.log' journalctl_b_script: 'journalctl -b' on_failure: failed_master_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_master_failure.sh' # Job has already failed, don't fail again and miss collecting data + failed_df_script: '${DFCMD}' failed_audit_log_script: 'cat /var/log/audit/audit.log || cat /var/log/kern.log || echo "Uh oh, cat audit.log failed"' failed_journalctl_b_script: 'journalctl -b || echo "Uh oh, journalctl -b failed"' @@ -402,7 +408,18 @@ verify_test_built_images_task: # "probably" work. A full round of testing will happen again after $*_CACHE_IMAGE_NAME # are updated in this or another PR (w/o '***CIRRUS: TEST IMAGES***'). environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}' + integration_test_script: '$SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP}' + ginkgo_node_logs_script: 'cat $CIRRUS_WORKING_DIR/test/e2e/ginkgo-node-*.log || echo "Ginkgo node logs not found"' + df_script: '${DFCMD}' + audit_log_script: 'cat /var/log/audit/audit.log || cat /var/log/kern.log' + journalctl_b_script: 'journalctl -b' + on_failure: + # Job has already failed, don't fail again and miss collecting data + failed_ginkgo_node_logs_script: 'cat $CIRRUS_WORKING_DIR/test/e2e/ginkgo-node-*.log || echo "Ginkgo node logs not found"' + failed_df_script: '${DFCMD}' + failed_audit_log_script: 'cat /var/log/audit/audit.log || cat /var/log/kern.log || echo "Uh oh, cat audit.log failed"' + failed_journalctl_b_script: 'journalctl -b || echo "Uh oh, journalctl -b failed"' # Build new cache-images for future PR testing, but only after a PR merge. diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh index d0a2e1579..334202aa9 100644 --- a/contrib/cirrus/lib.sh +++ b/contrib/cirrus/lib.sh @@ -60,7 +60,9 @@ PACKER_BUILDS="${PACKER_BUILDS:-ubuntu-18,fedora-29,fedora-28}" UBUNTU_BASE_IMAGE="ubuntu-1804-bionic-v20181203a" # Manually produced base-image names (see $SCRIPT_BASE/README.md) FEDORA_BASE_IMAGE="fedora-cloud-base-29-1-2-1541789245" +# FEDORA_BASE_IMAGE: "fedora-cloud-base-30-1-2-1556821664" PRIOR_FEDORA_BASE_IMAGE="fedora-cloud-base-28-1-1-1544474897" +# PRIOR_FEDORA_BASE_IMAGE="fedora-cloud-base-29-1-2-1541789245" BUILT_IMAGE_SUFFIX="${BUILT_IMAGE_SUFFIX:--$CIRRUS_REPO_NAME-${CIRRUS_BUILD_ID}}" # Safe env. vars. to transfer from root -> $ROOTLESS_USER (go env handled separetly) diff --git a/contrib/cirrus/packer/Makefile b/contrib/cirrus/packer/Makefile index 0ae0deca6..d03d22abe 100644 --- a/contrib/cirrus/packer/Makefile +++ b/contrib/cirrus/packer/Makefile @@ -3,7 +3,7 @@ # builder name(s) from applicable YAML file, # e.g for names see libpod_images.yml -PACKER_VER ?= 1.3.2 +PACKER_VER ?= 1.3.5 GOARCH=$(shell go env GOARCH) ARCH=$(uname -m) PACKER_DIST_FILENAME := packer_${PACKER_VER}_linux_${GOARCH}.zip @@ -78,6 +78,9 @@ ifndef GCP_PROJECT_ID endif ifndef GOOGLE_APPLICATION_CREDENTIALS $(error GOOGLE_APPLICATION_CREDENTIALS is undefined, expected absolute path to JSON file, like $HOME/.config/gcloud/legacy_credentials/*/adc.json) +endif +ifndef PACKER_BUILDS + $(error PACKER_BUILDS is undefined, expected builder-names CSV) endif PACKER_CACHE_DIR=/tmp ./packer build \ -force \ @@ -88,6 +91,5 @@ endif -var GOSRC=$(GOSRC) \ -var PACKER_BASE=$(PACKER_BASE) \ -var SCRIPT_BASE=$(SCRIPT_BASE) \ - -var POST_MERGE_BUCKET_SUFFIX=$(POST_MERGE_BUCKET_SUFFIX) \ -only $(PACKER_BUILDS) \ libpod_base_images.json diff --git a/contrib/cirrus/packer/fedora_base-setup.sh b/contrib/cirrus/packer/fedora_base-setup.sh index c0a1e422c..2e6d3eceb 100644 --- a/contrib/cirrus/packer/fedora_base-setup.sh +++ b/contrib/cirrus/packer/fedora_base-setup.sh @@ -15,9 +15,8 @@ install_ooe echo "Updating packages" ooe.sh dnf -y update -echo "Installing necessary packages and google services" -ooe.sh dnf -y copr enable ngompa/gce-oslogin -ooe.sh dnf -y install rng-tools google-compute-engine google-compute-engine-oslogin +echo "Installing necessary packages and google services" +ooe.sh dnf -y install rng-tools google-compute-engine-tools google-compute-engine-oslogin echo "Enabling services" ooe.sh systemctl enable rngd diff --git a/contrib/cirrus/packer/libpod_base_images.yml b/contrib/cirrus/packer/libpod_base_images.yml index 2caa28a04..560cb321c 100644 --- a/contrib/cirrus/packer/libpod_base_images.yml +++ b/contrib/cirrus/packer/libpod_base_images.yml @@ -13,13 +13,14 @@ variables: TTYDEV: # Latest Fedora release - FEDORA_IMAGE_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/29/Cloud/x86_64/images/Fedora-Cloud-Base-29-1.2.x86_64.qcow2" - FEDORA_CSUM_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/29/Cloud/x86_64/images/Fedora-Cloud-29-1.2-x86_64-CHECKSUM" - FEDORA_BASE_IMAGE_NAME: 'fedora-cloud-base-29-1-2' # Name to use in GCE + FEDORA_IMAGE_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/30/Cloud/x86_64/images/Fedora-Cloud-Base-30-1.2.x86_64.qcow2" + FEDORA_CSUM_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/30/Cloud/x86_64/images/Fedora-Cloud-30-1.2-x86_64-CHECKSUM" + FEDORA_BASE_IMAGE_NAME: 'fedora-cloud-base-30-1-2' + # Prior Fedora release - PRIOR_FEDORA_IMAGE_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/28/Cloud/x86_64/images/Fedora-Cloud-Base-28-1.1.x86_64.qcow2" - PRIOR_FEDORA_CSUM_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/28/Cloud/x86_64/images/Fedora-Cloud-28-1.1-x86_64-CHECKSUM" - PRIOR_FEDORA_BASE_IMAGE_NAME: 'fedora-cloud-base-28-1-1' # Name to use in GCE + PRIOR_FEDORA_IMAGE_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/29/Cloud/x86_64/images/Fedora-Cloud-Base-29-1.2.x86_64.qcow2" + PRIOR_FEDORA_CSUM_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/29/Cloud/x86_64/images/Fedora-Cloud-29-1.2-x86_64-CHECKSUM" + PRIOR_FEDORA_BASE_IMAGE_NAME: 'fedora-cloud-base-29-1-2' # Name to use in GCE # The name of the image in GCE used for packer build libpod_images.yml IBI_BASE_NAME: 'image-builder-image' -- cgit v1.2.3-54-g00ecf From bd0ac639bf1100c268d436bd44032b8fa46b9cc6 Mon Sep 17 00:00:00 2001 From: Chris Evich Date: Tue, 21 May 2019 10:56:47 -0400 Subject: Cirrus: Fix missing CRIO_COMMIT -> CONMON_COMMIT Signed-off-by: Chris Evich --- .cirrus.yml | 2 +- contrib/cirrus/build_vm_images.sh | 2 +- contrib/cirrus/packer/fedora_setup.sh | 2 +- contrib/cirrus/packer/libpod_images.yml | 4 ++-- contrib/cirrus/packer/ubuntu_setup.sh | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) (limited to 'contrib/cirrus/packer') diff --git a/.cirrus.yml b/.cirrus.yml index 32b76ba79..f3a0776db 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -43,7 +43,7 @@ env: # Git commits to use while building dependencies into cache-images FEDORA_CNI_COMMIT: "412b6d31280682bb4fab4446f113c22ff1886554" CNI_COMMIT: "7480240de9749f9a0a5c8614b17f1f03e0c06ab9" - CRIO_COMMIT: "7a283c391abb7bd25086a8ff91dbb36ebdd24466" + CONMON_COMMIT: "f02c053eb37010fc76d1e2966de7f2cb9f969ef2" CRIU_COMMIT: "c74b83cd49c00589c0c0468ba5fe685b67fdbd0a" # Special image w/ nested-libvirt + tools for creating new cache and base images IMAGE_BUILDER_CACHE_IMAGE_NAME: "image-builder-image-1541772081" diff --git a/contrib/cirrus/build_vm_images.sh b/contrib/cirrus/build_vm_images.sh index f4567ce74..805aba428 100755 --- a/contrib/cirrus/build_vm_images.sh +++ b/contrib/cirrus/build_vm_images.sh @@ -3,7 +3,7 @@ set -e source $(dirname $0)/lib.sh -ENV_VARS='CNI_COMMIT CRIO_COMMIT PACKER_BUILDS BUILT_IMAGE_SUFFIX UBUNTU_BASE_IMAGE FEDORA_BASE_IMAGE PRIOR_FEDORA_BASE_IMAGE SERVICE_ACCOUNT GCE_SSH_USERNAME GCP_PROJECT_ID PACKER_VER SCRIPT_BASE PACKER_BASE' +ENV_VARS='CNI_COMMIT CONMON_COMMIT PACKER_BUILDS BUILT_IMAGE_SUFFIX UBUNTU_BASE_IMAGE FEDORA_BASE_IMAGE PRIOR_FEDORA_BASE_IMAGE SERVICE_ACCOUNT GCE_SSH_USERNAME GCP_PROJECT_ID PACKER_VER SCRIPT_BASE PACKER_BASE' req_env_var $ENV_VARS # Must also be made available through make, into packer process export $ENV_VARS diff --git a/contrib/cirrus/packer/fedora_setup.sh b/contrib/cirrus/packer/fedora_setup.sh index 0e9867a89..e031129d7 100644 --- a/contrib/cirrus/packer/fedora_setup.sh +++ b/contrib/cirrus/packer/fedora_setup.sh @@ -8,7 +8,7 @@ set -e # Load in library (copied by packer, before this script was run) source /tmp/libpod/$SCRIPT_BASE/lib.sh -req_env_var SCRIPT_BASE FEDORA_CNI_COMMIT CNI_COMMIT CRIO_COMMIT CRIU_COMMIT +req_env_var SCRIPT_BASE FEDORA_CNI_COMMIT CNI_COMMIT CONMON_COMMIT CRIU_COMMIT install_ooe diff --git a/contrib/cirrus/packer/libpod_images.yml b/contrib/cirrus/packer/libpod_images.yml index 722d912c2..34d4db7fb 100644 --- a/contrib/cirrus/packer/libpod_images.yml +++ b/contrib/cirrus/packer/libpod_images.yml @@ -10,7 +10,7 @@ variables: # libpod dependencies to build and install into images FEDORA_CNI_COMMIT: "{{env `FEDORA_CNI_COMMIT`}}" CNI_COMMIT: "{{env `CNI_COMMIT`}}" - CRIO_COMMIT: "{{env `CRIO_COMMIT`}}" + CONMON_COMMIT: "{{env `CONMON_COMMIT`}}" CRIU_COMMIT: "{{env `CRIU_COMMIT`}}" RUNC_COMMIT: "{{env `RUNC_COMMIT`}}" @@ -74,7 +74,7 @@ provisioners: - 'GOSRC=/tmp/libpod' - 'CNI_COMMIT={{user `CNI_COMMIT`}}' - 'FEDORA_CNI_COMMIT={{user `FEDORA_CNI_COMMIT`}}' - - 'CRIO_COMMIT={{user `CRIO_COMMIT`}}' + - 'CONMON_COMMIT={{user `CONMON_COMMIT`}}' - 'CRIU_COMMIT={{user `CRIU_COMMIT`}}' - 'RUNC_COMMIT={{user `RUNC_COMMIT`}}' - 'SCRIPT_BASE={{user `SCRIPT_BASE`}}' diff --git a/contrib/cirrus/packer/ubuntu_setup.sh b/contrib/cirrus/packer/ubuntu_setup.sh index 45a987df0..56d7f962e 100644 --- a/contrib/cirrus/packer/ubuntu_setup.sh +++ b/contrib/cirrus/packer/ubuntu_setup.sh @@ -8,7 +8,7 @@ set -e # Load in library (copied by packer, before this script was run) source /tmp/libpod/$SCRIPT_BASE/lib.sh -req_env_var SCRIPT_BASE CNI_COMMIT CRIO_COMMIT CRIU_COMMIT +req_env_var SCRIPT_BASE CNI_COMMIT CONMON_COMMIT CRIU_COMMIT install_ooe -- cgit v1.2.3-54-g00ecf