Cirrus: Use imgts container to record metadata

Make use of the built imgts container image to track
VM image usage statistics for every automation run.

Also update and add small check to the gate test
that verifies expected formatting/content of the
`.cirrus.yml` file WRT VM Image names.

Signed-off-by: Chris Evich <cevich@redhat.com>

4 files changed, 122 insertions, 1 deletions

diff --git a/.cirrus.yml b/.cirrus.yml
index b473980ea..328b2e676 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -27,11 +27,17 @@ env:
     ####
     #### Cache-image names to test with
     ###
+    ACTIVE_CACHE_IMAGE_NAMES: >-
+        fedora-29-libpod-d6d53e40
+        fedora-28-libpod-d6d53e40
+        ubuntu-18-libpod-d6d53e40
+        rhel-7-libpod-7f4cd1f7
+        image-builder-image-1541772081
     FEDORA_CACHE_IMAGE_NAME: "fedora-29-libpod-d6d53e40"
     PRIOR_FEDORA_CACHE_IMAGE_NAME: "fedora-28-libpod-d6d53e40"
     UBUNTU_CACHE_IMAGE_NAME: "ubuntu-18-libpod-d6d53e40"
+    PRIOR_RHEL_CACHE_IMAGE_NAME: "rhel-7-libpod-7f4cd1f7"
     # RHEL_CACHE_IMAGE_NAME: "rhel-8-notready"
-    PRIOR_RHEL_CACHE_IMAGE_NAME: "rhel-7-libpod-d6d53e40"
     # CENTOS_CACHE_IMAGE_NAME: "centos-7-notready"
 
     ####
@@ -108,6 +114,7 @@ gating_task:
     gate_script:
         - '/usr/local/bin/entrypoint.sh validate'
         - '/usr/local/bin/entrypoint.sh lint'
+        - '${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/test/test_dot_cirrus_yaml.py'
 
     # This task runs `make vendor` followed by ./hack/tree_status.sh to check
     # whether the git tree is clean.  The reasoning for that is to make sure
@@ -149,6 +156,30 @@ build_each_commit_task:
         - env GOPATH=/var/tmp/go/ make build-all-new-commits GIT_BASE_BRANCH=origin/$CIRRUS_BASE_BRANCH
 
 
+# Update metadata on VM images referenced by this repository state
+meta_task:
+
+    depends_on:
+        - "gating"
+
+    container:
+        image: "quay.io/libpod/imgts:latest"  # see contrib/imgts
+        cpu: 1
+        memory: 1
+
+    env:
+        # Space-separated list of images used by this repository state
+        IMGNAMES: "${ACTIVE_CACHE_IMAGE_NAMES}"
+        BUILDID: "${CIRRUS_BUILD_ID}"
+        REPOREF: "${CIRRUS_CHANGE_IN_REPO}"
+        GCPJSON: ENCRYPTED[950d9c64ad78f7b1f0c7e499b42dc058d2b23aa67e38b315e68f557f2aba0bf83068d4734f7b1e1bdd22deabe99629df]
+        GCPNAME: ENCRYPTED[b05d469a0dba8cb479cb00cc7c1f6747c91d17622fba260a986b976aa6c817d4077eacffd4613d6d5f23afc4084fab1d]
+        GCPPROJECT: ENCRYPTED[7c80e728e046b1c76147afd156a32c1c57d4a1ac1eab93b7e68e718c61ca8564fc61fef815952b8ae0a64e7034b8fe4f]
+        CIRRUS_CLONE_DEPTH: 1  # source not used
+
+    script: /usr/local/bin/entrypoint.sh
+
+
 # This task does the unit and integration testing for every platform
 testing_task:
 
diff --git a/contrib/cirrus/README.md b/contrib/cirrus/README.md
index 0d91301c6..e3b3182ec 100644
--- a/contrib/cirrus/README.md
+++ b/contrib/cirrus/README.md
@@ -33,6 +33,17 @@ task (pass or fail) is set based on the exit status of the last script to execut
    the vendor.conf, the code and the vendored packages in ./vendor are in sync
    at all times.
 
+### ``meta`` Task
+
+***N/B: Steps below are performed by automation***
+
+1. Launch a container built from definition in ``./contrib/imgts``.
+
+2. Update VM Image metadata to help track usage across all automation.
+
+4. Always exits successfully unless there's a major problem.
+
+
 ### ``testing`` Task
 
 ***N/B: Steps below are performed by automation***
diff --git a/contrib/cirrus/test/test_dot_cirrus_yaml.py b/contrib/cirrus/test/test_dot_cirrus_yaml.py
new file mode 100755
index 000000000..2894bc45e
--- /dev/null
+++ b/contrib/cirrus/test/test_dot_cirrus_yaml.py
@@ -0,0 +1,78 @@
+#!/bin/env python3
+
+import sys
+import os
+import os.path
+import unittest
+import warnings
+import yaml
+
+class TestCaseBase(unittest.TestCase):
+
+    SCRIPT_PATH = os.path.realpath((os.path.dirname(sys.argv[0])))
+    CIRRUS_WORKING_DIR = os.environ.get('CIRRUS_WORKING_DIR',
+                                        '{0}/../../../'.format(SCRIPT_PATH))
+
+    def setUp(self):
+        os.chdir(self.CIRRUS_WORKING_DIR)
+
+
+class TestCirrusYAML(TestCaseBase):
+
+    IMAGE_NAME_SUFFIX = '_CACHE_IMAGE_NAME'
+    ACTIVE_IMAGES_NAME = 'ACTIVE_CACHE_IMAGE_NAMES'
+
+    def setUp(self):
+        TestCirrusYAML._cirrus = None
+        super().setUp()
+
+    @property
+    def cirrus(self):
+        if TestCirrusYAML._cirrus is None:
+            with warnings.catch_warnings():
+                warnings.filterwarnings("ignore",category=DeprecationWarning)
+                with open('.cirrus.yml', "r") as dot_cirrus_dot_yaml:
+                    TestCirrusYAML._cirrus = yaml.load(dot_cirrus_dot_yaml)
+        return TestCirrusYAML._cirrus
+
+    def _assert_get_cache_image_names(self, env):
+        inames = set([key for key in env.keys()
+                      if key.endswith(self.IMAGE_NAME_SUFFIX)])
+        self.assertNotEqual(inames, set())
+
+        ivalues = set([value for key, value in env.items()
+                       if key in inames])
+        self.assertNotEqual(ivalues, set())
+        return ivalues
+
+    def _assert_get_subdct(self, key, dct):
+        self.assertIn(key, dct)
+        return dct[key]
+
+    def test_parse_yaml(self):
+        self.assertIsInstance(self.cirrus, dict)
+
+    def test_active_cache_image_names(self):
+        env = self._assert_get_subdct('env', self.cirrus)
+        acin = self._assert_get_subdct(self.ACTIVE_IMAGES_NAME, env)
+
+        for ivalue in self._assert_get_cache_image_names(env):
+            self.assertIn(ivalue, acin,
+                          "The '{}' sub-key of 'env' should contain this among"
+                          " its space-separated values."
+                          "".format(self.ACTIVE_IMAGES_NAME))
+
+
+    def test_cache_image_names_active(self):
+        env = self._assert_get_subdct('env', self.cirrus)
+        ivalues = self._assert_get_cache_image_names(env)
+
+        for avalue in set(self._assert_get_subdct(self.ACTIVE_IMAGES_NAME, env).split()):
+            self.assertIn(avalue, ivalues,
+                          "All space-separated values in the '{}' sub-key"
+                          " of 'env' must also be used in a key with a '{}' suffix."
+                          "".format(self.ACTIVE_IMAGES_NAME, self.IMAGE_NAME_SUFFIX))
+
+
+if __name__ == '__main__':
+    unittest.main(failfast=True, catchbreak=True, verbosity=0)
diff --git a/contrib/gate/Dockerfile b/contrib/gate/Dockerfile
index 4d88ae9a6..16d5eda67 100644
--- a/contrib/gate/Dockerfile
+++ b/contrib/gate/Dockerfile
@@ -29,6 +29,7 @@ RUN dnf -y install \
       python3-dateutil \
       python3-psutil \
       python3-pytoml \
+      python3-pyyaml \
       python3-varlink \
       skopeo-containers \
       slirp4netns \