diff options
Diffstat (limited to 'contrib/cirrus')
| -rw-r--r-- | contrib/cirrus/README.md | 73 | ||||
| -rwxr-xr-x | contrib/cirrus/build_vm_images.sh | 5 | ||||
| -rw-r--r-- | contrib/cirrus/lib.sh | 5 | ||||
| -rw-r--r-- | contrib/cirrus/packer/centos_setup.sh | 3 | ||||
| -rw-r--r-- | contrib/cirrus/packer/fedora_setup.sh | 3 | ||||
| -rw-r--r-- | contrib/cirrus/packer/libpod_images.yml | 6 | ||||
| -rw-r--r-- | contrib/cirrus/packer/rhel_setup.sh | 3 | ||||
| -rw-r--r-- | contrib/cirrus/packer/ubuntu_setup.sh | 3 | ||||
| -rwxr-xr-x | contrib/cirrus/rootless_test.sh | 6 | ||||
| -rwxr-xr-x | contrib/cirrus/setup_and_run_rootless.sh | 26 | ||||
| -rwxr-xr-x | contrib/cirrus/test/test_dot_cirrus_yaml.py | 78 |
11 files changed, 94 insertions, 117 deletions
diff --git a/contrib/cirrus/README.md b/contrib/cirrus/README.md index ea358d2d7..ce3ea7777 100644 --- a/contrib/cirrus/README.md +++ b/contrib/cirrus/README.md @@ -99,43 +99,64 @@ contents of the ``$SPECIALMODE`` environment variable. then execute `make localsystem` from the repository root. -### ``cache_images`` Task - -Modifying the contents of cache-images is done by making changes to -one or more of the ``./contrib/cirrus/packer/*_setup.sh`` files. Testing -those changes currently requires adding a temporary commit to a PR that -updates ``.cirrus.yml``: +### ``test_build_cache_images_task`` Task -* Remove all task sections except ``cache_images_task``. -* Remove the ``only_if`` condition and ``depends_on`` dependencies - -The new image names will be displayed at the end of output, assuming the build -is successful, at that point the temporary commit may be removed. Finally, -the new names may be used as ``image_name`` values in ``.cirrus.yml``. +Modifying the contents of cache-images is tested by making changes to +one or more of the ``./contrib/cirrus/packer/*_setup.sh`` files. Then +in the PR description, add the magic string: ``***CIRRUS: TEST IMAGES***`` ***N/B: Steps below are performed by automation*** -1. When a PR is merged (``$CIRRUS_BRANCH`` == ``master``), run another - round of the ``gating`` and ``testing`` tasks (above). - -2. Assuming tests pass, if the commit message contains the magic string - ``***CIRRUS: REBUILD IMAGES***``, then this task continues. Otherwise - simply mark the master branch as 'passed'. +1. ``setup_environment.sh``: Same as for other tasks. -3. ``setup_environment.sh``: Same as for other tasks. - -4. ``build_vm_images.sh``: Utilize [the packer tool](http://packer.io/docs/) +2. ``build_vm_images.sh``: Utilize [the packer tool](http://packer.io/docs/) to produce new VM images. Create a new VM from each base-image, connect to them with ``ssh``, and perform the steps as defined by the - ``$PACKER_BASE/libpod_images.json`` file: + ``$PACKER_BASE/libpod_images.yml`` file: 1. On a base-image VM, as root, copy the current state of the repository into ``/tmp/libpod``. 2. Execute distribution-specific scripts to prepare the image for - use by the ``integration_testing`` task (above). For example, - ``fedora_setup.sh``. - 3. If successful, shut down each VM and create a new GCE Image - named with the base image, and the commit sha of the merge. + use. For example, ``fedora_setup.sh``. + 3. If successful, shut down each VM and record the names, and dates + into a json manifest file. + 4. Move the manifest file, into a google storage bucket object. + This is a retained as a secondary method for tracking/auditing + creation of VM images, should it ever be needed. + +***Manual Steps:*** In order to utilize built images, their names must be upated +in ``.cirrus.yml``. For example, if the image ``blah-1234`` was produced above: + +```yaml +env: + #### + #### Cache-image names to test with + ### + BLAH_CACHE_IMAGE_NAME: "blah-1234" +``` + +A new pull-request with that change, will run tasks utilizing that image. + + +### ``test_built_images`` Task + +Only runs following successful ``test_build_cache_images_task`` task. Uses +images following the standard naming format, with execution of +the 'gate', 'testing' and 'rootless_testing' scripts. Validating the images +suitability for wide-spread use. + + +### ``cache_images`` Task + +Exactly the same as ``test_build_cache_images_task`` task, but only runs on +the master branch. Requires a magic string to be in the `HEAD` +commit message: ``***CIRRUS: BUILD IMAGES***`` + +When successful, the manifest file along with all VM disks, are moved +into a dedicated google storage bucket, separate from the one used by +`test_build_cache_images_task`. These may be used to create new cache-images for +PR testing by manually importing them as described above. + ### Base-images diff --git a/contrib/cirrus/build_vm_images.sh b/contrib/cirrus/build_vm_images.sh index 43eb3c057..cfdfd5d7d 100755 --- a/contrib/cirrus/build_vm_images.sh +++ b/contrib/cirrus/build_vm_images.sh @@ -46,7 +46,4 @@ make libpod_images \ URI="gs://packer-import${POST_MERGE_BUCKET_SUFFIX}/manifest${BUILT_IMAGE_SUFFIX}.json" gsutil cp packer-manifest.json "$URI" -echo "Finished." -echo "Any tarball URI's referenced above at at $URI" -echo "may be used to create VM images suitable for use in" -echo ".cirrus.yml as values for the 'image_name' keys." +echo "Finished. A JSON manifest of produced images is available at $URI" diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh index a285b133b..565646c12 100644 --- a/contrib/cirrus/lib.sh +++ b/contrib/cirrus/lib.sh @@ -19,6 +19,9 @@ CIRRUS_BASE_SHA=${CIRRUS_BASE_SHA:-HEAD} CIRRUS_CHANGE_IN_REPO=${CIRRUS_CHANGE_IN_REPO:-FETCH_HEAD} SPECIALMODE="${SPECIALMODE:-none}" export CONTAINER_RUNTIME=${CONTAINER_RUNTIME:-podman} +ROOTLESS_USER="${ROOTLESS_USER:-}" +ROOTLESS_UID="${ROOTLESS_UID:-}" +ROOTLESS_GID="${ROOTLESS_GID:-}" if ! [[ "$PATH" =~ "/usr/local/bin" ]] then @@ -200,7 +203,7 @@ setup_rootless() { su --login --command 'go env' $ROOTLESS_USER | \ while read envline do - X=$(echo "export $envline" | tee -a "/home/$ROOTLESS_USER/$ENVLIB") && echo "$X" + X=$(echo "export $envline" | tee -a "/home/$ROOTLESS_USER/.bash_profile") && echo "$X" done } diff --git a/contrib/cirrus/packer/centos_setup.sh b/contrib/cirrus/packer/centos_setup.sh index 91b1963c2..b86ba1a1c 100644 --- a/contrib/cirrus/packer/centos_setup.sh +++ b/contrib/cirrus/packer/centos_setup.sh @@ -36,6 +36,7 @@ ooe.sh sudo yum -y install \ golang-github-cpuguy83-go-md2man \ gpgme-devel \ iptables \ + jq \ libassuan-devel \ libcap-devel \ libnet \ @@ -77,6 +78,8 @@ install_criu install_packer_copied_files +sudo /tmp/libpod/hack/install_catatonit.sh + rh_finalize echo "SUCCESS!" diff --git a/contrib/cirrus/packer/fedora_setup.sh b/contrib/cirrus/packer/fedora_setup.sh index 18490fdac..fef74179d 100644 --- a/contrib/cirrus/packer/fedora_setup.sh +++ b/contrib/cirrus/packer/fedora_setup.sh @@ -36,6 +36,7 @@ ooe.sh sudo dnf install -y \ gpgme-devel \ iptables \ iproute \ + jq \ libassuan-devel \ libcap-devel \ libnet \ @@ -78,6 +79,8 @@ install_conmon install_packer_copied_files +sudo /tmp/libpod/hack/install_catatonit.sh + rh_finalize # N/B: Halts system! echo "SUCCESS!" diff --git a/contrib/cirrus/packer/libpod_images.yml b/contrib/cirrus/packer/libpod_images.yml index 30ad0723a..5c42fdae1 100644 --- a/contrib/cirrus/packer/libpod_images.yml +++ b/contrib/cirrus/packer/libpod_images.yml @@ -98,8 +98,4 @@ provisioners: - 'RHSM_COMMAND={{user `RHSM_COMMAND`}}' post-processors: - # Store VM disk in GCP storage, where it will expire based on a defined - # lifecycle. This prevents GCE from filling with disused images. - - - type: 'googlecompute-export' - paths: ['gs://packer-import{{user `POST_MERGE_BUCKET_SUFFIX`}}/{{build_name}}{{user `BUILT_IMAGE_SUFFIX`}}.tar.gz'] - - type: 'manifest' # writes packer-manifest.json + - - type: 'manifest' # writes packer-manifest.json diff --git a/contrib/cirrus/packer/rhel_setup.sh b/contrib/cirrus/packer/rhel_setup.sh index 45f5c3e9b..b6240be80 100644 --- a/contrib/cirrus/packer/rhel_setup.sh +++ b/contrib/cirrus/packer/rhel_setup.sh @@ -41,6 +41,7 @@ ooe.sh sudo yum -y install \ golang-github-cpuguy83-go-md2man \ gpgme-devel \ iptables \ + jq \ libassuan-devel \ libcap-devel \ libnet \ @@ -84,6 +85,8 @@ install_criu install_packer_copied_files +sudo /tmp/libpod/hack/install_catatonit.sh + rhel_exit_handler # release subscription! rh_finalize diff --git a/contrib/cirrus/packer/ubuntu_setup.sh b/contrib/cirrus/packer/ubuntu_setup.sh index d3ac8bddb..93dbd0fa0 100644 --- a/contrib/cirrus/packer/ubuntu_setup.sh +++ b/contrib/cirrus/packer/ubuntu_setup.sh @@ -45,6 +45,7 @@ ooe.sh sudo -E apt-get -qq install \ golang \ iproute2 \ iptables \ + jq \ libaio-dev \ libapparmor-dev \ libcap-dev \ @@ -101,6 +102,8 @@ install_buildah install_packer_copied_files +sudo /tmp/libpod/hack/install_catatonit.sh + install_varlink sudo curl https://raw.githubusercontent.com/projectatomic/registries/master/registries.fedora\ diff --git a/contrib/cirrus/rootless_test.sh b/contrib/cirrus/rootless_test.sh index eab06bac0..13386bb09 100755 --- a/contrib/cirrus/rootless_test.sh +++ b/contrib/cirrus/rootless_test.sh @@ -1,10 +1,10 @@ #!/bin/bash -set -e -source $HOME/.bash_profile +set -ex +source "$HOME/.bash_profile" +source $GOSRC/$SCRIPT_BASE/lib.sh cd $GOSRC -source $(dirname $0)/lib.sh req_env_var GOSRC OS_RELEASE_ID OS_RELEASE_VER diff --git a/contrib/cirrus/setup_and_run_rootless.sh b/contrib/cirrus/setup_and_run_rootless.sh new file mode 100755 index 000000000..7ca34a6ac --- /dev/null +++ b/contrib/cirrus/setup_and_run_rootless.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +set -ex + +source $(dirname $0)/lib.sh + +req_env_var " +CIRRUS_WORKING_DIR $CIRRUS_WORKING_DIR +GOSRC $GOSRC +SCRIPT_BASE $SCRIPT_BASE +ROOTLESS_USER $ROOTLESS_USER +ROOTLESS_UID $ROOTLESS_UID +ROOTLESS_GID $ROOTLESS_GID +" + +if run_rootless +then + die 86 "Error: Expected rootless env. vars not set or empty" +fi + +cd $GOSRC +setup_rootless + +ssh $ROOTLESS_USER@localhost \ + -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o CheckHostIP=no \ + $CIRRUS_WORKING_DIR/$SCRIPT_BASE/rootless_test.sh diff --git a/contrib/cirrus/test/test_dot_cirrus_yaml.py b/contrib/cirrus/test/test_dot_cirrus_yaml.py deleted file mode 100755 index 2894bc45e..000000000 --- a/contrib/cirrus/test/test_dot_cirrus_yaml.py +++ /dev/null @@ -1,78 +0,0 @@ -#!/bin/env python3 - -import sys -import os -import os.path -import unittest -import warnings -import yaml - -class TestCaseBase(unittest.TestCase): - - SCRIPT_PATH = os.path.realpath((os.path.dirname(sys.argv[0]))) - CIRRUS_WORKING_DIR = os.environ.get('CIRRUS_WORKING_DIR', - '{0}/../../../'.format(SCRIPT_PATH)) - - def setUp(self): - os.chdir(self.CIRRUS_WORKING_DIR) - - -class TestCirrusYAML(TestCaseBase): - - IMAGE_NAME_SUFFIX = '_CACHE_IMAGE_NAME' - ACTIVE_IMAGES_NAME = 'ACTIVE_CACHE_IMAGE_NAMES' - - def setUp(self): - TestCirrusYAML._cirrus = None - super().setUp() - - @property - def cirrus(self): - if TestCirrusYAML._cirrus is None: - with warnings.catch_warnings(): - warnings.filterwarnings("ignore",category=DeprecationWarning) - with open('.cirrus.yml', "r") as dot_cirrus_dot_yaml: - TestCirrusYAML._cirrus = yaml.load(dot_cirrus_dot_yaml) - return TestCirrusYAML._cirrus - - def _assert_get_cache_image_names(self, env): - inames = set([key for key in env.keys() - if key.endswith(self.IMAGE_NAME_SUFFIX)]) - self.assertNotEqual(inames, set()) - - ivalues = set([value for key, value in env.items() - if key in inames]) - self.assertNotEqual(ivalues, set()) - return ivalues - - def _assert_get_subdct(self, key, dct): - self.assertIn(key, dct) - return dct[key] - - def test_parse_yaml(self): - self.assertIsInstance(self.cirrus, dict) - - def test_active_cache_image_names(self): - env = self._assert_get_subdct('env', self.cirrus) - acin = self._assert_get_subdct(self.ACTIVE_IMAGES_NAME, env) - - for ivalue in self._assert_get_cache_image_names(env): - self.assertIn(ivalue, acin, - "The '{}' sub-key of 'env' should contain this among" - " its space-separated values." - "".format(self.ACTIVE_IMAGES_NAME)) - - - def test_cache_image_names_active(self): - env = self._assert_get_subdct('env', self.cirrus) - ivalues = self._assert_get_cache_image_names(env) - - for avalue in set(self._assert_get_subdct(self.ACTIVE_IMAGES_NAME, env).split()): - self.assertIn(avalue, ivalues, - "All space-separated values in the '{}' sub-key" - " of 'env' must also be used in a key with a '{}' suffix." - "".format(self.ACTIVE_IMAGES_NAME, self.IMAGE_NAME_SUFFIX)) - - -if __name__ == '__main__': - unittest.main(failfast=True, catchbreak=True, verbosity=0) |