summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--.cirrus.yml239
-rw-r--r--contrib/cirrus/README.md109
-rwxr-xr-xcontrib/cirrus/build_vm_images.sh10
-rwxr-xr-xcontrib/cirrus/integration_test.sh26
-rw-r--r--contrib/cirrus/lib.sh266
-rwxr-xr-xcontrib/cirrus/lib.sh.t17
-rwxr-xr-xcontrib/cirrus/notice_master_failure.sh2
-rw-r--r--contrib/cirrus/packer/Makefile19
-rw-r--r--contrib/cirrus/packer/centos_setup.sh82
-rw-r--r--contrib/cirrus/packer/fah_base-setup.sh45
-rw-r--r--contrib/cirrus/packer/fah_setup.sh21
-rw-r--r--contrib/cirrus/packer/fedora_base-setup.sh5
-rw-r--r--contrib/cirrus/packer/fedora_setup.sh12
-rw-r--r--contrib/cirrus/packer/libpod_base_images.yml66
-rw-r--r--contrib/cirrus/packer/libpod_images.yml28
-rw-r--r--contrib/cirrus/packer/rhel_base-setup.sh78
-rw-r--r--contrib/cirrus/packer/rhel_setup.sh91
-rw-r--r--contrib/cirrus/packer/ubuntu_setup.sh14
-rwxr-xr-xcontrib/cirrus/rootless_test.sh12
-rwxr-xr-xcontrib/cirrus/setup_environment.sh142
-rwxr-xr-xcontrib/cirrus/system_test.sh11
-rwxr-xr-xcontrib/cirrus/test/test_dot_cirrus_yaml.py78
-rwxr-xr-xcontrib/cirrus/unit_test.sh4
-rwxr-xr-xhack/get_ci_vm.sh127
-rw-r--r--test/registries.conf2
25 files changed, 561 insertions, 945 deletions
diff --git a/.cirrus.yml b/.cirrus.yml
index fe09ea988..f3a0776db 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -13,61 +13,40 @@ env:
####
#### Global variables used for all tasks
####
- # File to update in home-dir with task-specific env. var values
- ENVLIB: ".bash_profile"
# Overrides default location (/tmp/cirrus) for repo clone
+ GOPATH: "/var/tmp/go"
+ GOSRC: "/var/tmp/go/src/github.com/containers/libpod"
CIRRUS_WORKING_DIR: "/var/tmp/go/src/github.com/containers/libpod"
- # Required so $ENVLIB gets loaded
+ # The default is 'sh' if unspecified
CIRRUS_SHELL: "/bin/bash"
# Save a little typing (path relative to $CIRRUS_WORKING_DIR)
SCRIPT_BASE: "./contrib/cirrus"
- PACKER_BASE: "./contrib/cirrus/packer"
- CIRRUS_CLONE_DEPTH: 200
# Command to prefix every output line with a timestamp
# (can't do inline awk script, Cirrus-CI or YAML mangles quoting)
TIMESTAMP: "awk --file ${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/timestamp.awk"
+ # Command to log critical filesystems, types, and sizes.
+ DFCMD: "df -lhTx tmpfs"
+ CIRRUS_CLONE_DEPTH: 50
####
#### Cache-image names to test with
###
- ACTIVE_CACHE_IMAGE_NAMES: >-
- fedora-29-libpod-548c1c05
- fedora-28-libpod-548c1c05
- ubuntu-18-libpod-548c1c05
- rhel-7-libpod-548c1c05
- image-builder-image-1541772081
- FEDORA_CACHE_IMAGE_NAME: "fedora-29-libpod-548c1c05"
- PRIOR_FEDORA_CACHE_IMAGE_NAME: "fedora-28-libpod-548c1c05"
- UBUNTU_CACHE_IMAGE_NAME: "ubuntu-18-libpod-548c1c05"
- PRIOR_RHEL_CACHE_IMAGE_NAME: "rhel-7-libpod-548c1c05"
- # RHEL_CACHE_IMAGE_NAME: "rhel-8-notready"
- # CENTOS_CACHE_IMAGE_NAME: "centos-7-notready"
+ FEDORA_CACHE_IMAGE_NAME: "fedora-29-libpod-5170730531028992"
+ PRIOR_FEDORA_CACHE_IMAGE_NAME: "fedora-28-libpod-5170730531028992"
+ UBUNTU_CACHE_IMAGE_NAME: "ubuntu-18-libpod-5170730531028992"
####
#### Variables for composing new cache-images (used in PR testing) from
#### base-images (pre-existing in GCE)
####
+ BUILT_IMAGE_SUFFIX: "-${CIRRUS_REPO_NAME}-${CIRRUS_BUILD_ID}"
# Git commits to use while building dependencies into cache-images
FEDORA_CNI_COMMIT: "412b6d31280682bb4fab4446f113c22ff1886554"
CNI_COMMIT: "7480240de9749f9a0a5c8614b17f1f03e0c06ab9"
- CRIO_COMMIT: "7a283c391abb7bd25086a8ff91dbb36ebdd24466"
+ CONMON_COMMIT: "f02c053eb37010fc76d1e2966de7f2cb9f969ef2"
CRIU_COMMIT: "c74b83cd49c00589c0c0468ba5fe685b67fdbd0a"
- RUNC_COMMIT: "029124da7af7360afa781a0234d1b083550f797c"
- # CSV of cache-image names to build (see $PACKER_BASE/libpod_images.json)
- PACKER_BUILDS: "ubuntu-18,fedora-29,fedora-28,rhel-7" # TODO: rhel-8,centos-7
- # Version of packer to use
- PACKER_VER: "1.3.2"
# Special image w/ nested-libvirt + tools for creating new cache and base images
IMAGE_BUILDER_CACHE_IMAGE_NAME: "image-builder-image-1541772081"
- # Google-maintained base-image names
- UBUNTU_BASE_IMAGE: "ubuntu-1804-bionic-v20181203a"
- CENTOS_BASE_IMAGE: "centos-7-v20181113"
- # Manually produced base-image names (see $SCRIPT_BASE/README.md)
- FEDORA_BASE_IMAGE: "fedora-cloud-base-29-1-2-1541789245"
- PRIOR_FEDORA_BASE_IMAGE: "fedora-cloud-base-28-1-1-1544474897"
- FAH_BASE_IMAGE: "fedora-atomichost-29-20181025-1-1541787861"
- # RHEL image must be imported, google bills extra for their native image.
- RHEL_BASE_IMAGE: "rhel-guest-image-7-6-210-x86-64-qcow2-1548099756"
####
#### Default to NOT operating in any special-case testing mode
@@ -80,8 +59,6 @@ env:
####
# Freenode IRC credentials for posting status messages
IRCID: ENCRYPTED[e87bba62a8e924dc70bdb2b66b16f6ab4a60d2870e6e5534ae9e2b0076f483c71c84091c655ca239101e6816c5ec0883]
- # Command to register a RHEL VM to install/update packages
- RHSM_COMMAND: ENCRYPTED[5caa5ff8c5370c3d25c7a1a28168501ab0fa2e5e3b627926f6eaba02b3fed965a7638a6151657809661f8c905c7dc187]
# Needed to build GCE images, within a GCE VM
SERVICE_ACCOUNT: ENCRYPTED[99e9a0b1c23f8dd29e83dfdf164f064cfd17afd9b895ca3b5e4c41170bd4290a8366fe2ad8e7a210b9f751711d1d002a]
# User ID for cirrus to ssh into VMs
@@ -89,21 +66,16 @@ env:
# Name where this repositories cloud resources are located
GCP_PROJECT_ID: ENCRYPTED[7c80e728e046b1c76147afd156a32c1c57d4a1ac1eab93b7e68e718c61ca8564fc61fef815952b8ae0a64e7034b8fe4f]
- # Space separated list of environment variables to unset before testing
- UNSET_ENV_VARS: >-
- GCP_PROJECT_ID GCE_SSH_USERNAME SERVICE_ACCOUNT RHSM_COMMAND BUILT_IMAGE_SUFFIX
- IRCID RHEL_BASE_IMAGE FAH_BASE_IMAGE FEDORA_BASE_IMAGE CENTOS_BASE_IMAGE
- UBUNTU_BASE_IMAGE PACKER_VER PACKER_BUILDS RUNC_COMMIT CRIU_COMMIT
- CRIO_COMMIT CNI_COMMIT FEDORA_CNI_COMMIT PACKER_BASE SCRIPT_BASE
- CIRRUS_SHELL CIRRUS_WORKING_DIR ENVLIB BUILT_IMAGE_SUFFIX CIRRUS_CI
- CI_NODE_INDEX CI_NODE_TOTAL CIRRUS_BASE_BRANCH CIRRUS_BASE_SHA
- CIRRUS_BRANCH CIRRUS_BUILD_ID CIRRUS_CHANGE_IN_REPO CIRRUS_CLONE_DEPTH
- CIRRUS_COMMIT_MESSAGE CIRRUS_CHANGE_MESSAGE CIRRUS_REPO_CLONE_HOST
- CIRRUS_DEFAULT_BRANCH CIRRUS_PR CIRRUS_TAG CIRRUS_OS CIRRUS_TASK_NAME
- CIRRUS_TASK_ID CIRRUS_REPO_NAME CIRRUS_REPO_OWNER CIRRUS_REPO_FULL_NAME
- CIRRUS_REPO_CLONE_URL CIRRUS_SHELL CIRRUS_USER_COLLABORATOR CIRRUS_USER_PERMISSION
- CIRRUS_WORKING_DIR CIRRUS_HTTP_CACHE_HOST PACKER_BUILDS BUILT_IMAGE_SUFFIX
- XDG_DATA_DIRS XDG_RUNTIME_DIR XDG_SESSION_ID ROOTLESS_USER
+
+# Default VM to use unless set or modified by task
+gce_instance:
+ image_project: "libpod-218412"
+ zone: "us-central1-a" # Required by Cirrus for the time being
+ cpu: 2
+ memory: "4Gb"
+ disk: 200
+ # A matrix could be used here, for now just one VM
+ image_name: "${FEDORA_CACHE_IMAGE_NAME}"
# Every *_task runs in parallel in separate VMsd. The name prefix only for reference
@@ -113,6 +85,7 @@ gating_task:
env:
CIRRUS_WORKING_DIR: "/usr/src/libpod"
+ GOPATH: "/go"
GOSRC: "/go/src/github.com/containers/libpod"
# Runs within Cirrus's "community cluster"
@@ -131,7 +104,6 @@ gating_task:
- '/usr/local/bin/entrypoint.sh install.tools |& ${TIMESTAMP}'
- '/usr/local/bin/entrypoint.sh validate |& ${TIMESTAMP}'
- '/usr/local/bin/entrypoint.sh lint |& ${TIMESTAMP}'
- - '${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/test/test_dot_cirrus_yaml.py |& ${TIMESTAMP}'
# This task builds Podman with different buildtags to ensure the build does
# not break. It also verifies all sub-commands have man pages.
@@ -157,11 +129,15 @@ gating_task:
# in sync at all times.
vendor_task:
+ only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\*\*\*\s*CIRRUS:\s*TEST\s*IMAGES\s*\*\*\*.*'
+
depends_on:
- "gating"
env:
CIRRUS_WORKING_DIR: "/usr/src/libpod"
+ GOPATH: "/go"
+ GOSRC: "/go/src/github.com/containers/libpod"
# Runs within Cirrus's "community cluster"
container:
@@ -174,7 +150,7 @@ vendor_task:
vendor_script:
- '/usr/local/bin/entrypoint.sh .install.vndr |& ${TIMESTAMP}'
- '/usr/local/bin/entrypoint.sh vendor |& ${TIMESTAMP}'
- - 'cd /go/src/github.com/containers/libpod && ./hack/tree_status.sh |& ${TIMESTAMP}'
+ - 'cd ${GOSRC} && ./hack/tree_status.sh |& ${TIMESTAMP}'
on_failure:
failed_master_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_master_failure.sh |& ${TIMESTAMP}'
@@ -184,11 +160,15 @@ vendor_task:
# whether the git tree is clean.
varlink_api_task:
+ only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\*\*\*\s*CIRRUS:\s*TEST\s*IMAGES\s*\*\*\*.*'
+
depends_on:
- "gating"
env:
CIRRUS_WORKING_DIR: "/usr/src/libpod"
+ GOPATH: "/go"
+ GOSRC: "/go/src/github.com/containers/libpod"
# Used by tree_status.sh
SUGGESTION: 'remove API.md, then "make varlink_api_generate" and commit changes.'
@@ -200,9 +180,9 @@ varlink_api_task:
timeout_in: 10m
- vendor_script:
- - '/usr/local/bin/entrypoint.sh varlink_api_generate'
- - 'cd /go/src/github.com/containers/libpod && ./hack/tree_status.sh'
+ api_md_script:
+ - '/usr/local/bin/entrypoint.sh varlink_api_generate |& ${TIMESTAMP}'
+ - 'cd ${GOSRC} && ./hack/tree_status.sh |& ${TIMESTAMP}'
on_failure:
failed_master_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_master_failure.sh'
@@ -216,7 +196,8 @@ build_each_commit_task:
- "varlink_api"
# $CIRRUS_BASE_BRANCH is only set when testing a PR
- only_if: $CIRRUS_BRANCH != 'master'
+ only_if: $CIRRUS_BRANCH != 'master' &&
+ $CIRRUS_CHANGE_MESSAGE !=~ '.*\*\*\*\s*CIRRUS:\s*TEST\s*IMAGES\s*\*\*\*.*'
gce_instance:
image_project: "libpod-218412"
@@ -250,7 +231,11 @@ meta_task:
env:
# Space-separated list of images used by this repository state
- IMGNAMES: "${ACTIVE_CACHE_IMAGE_NAMES}"
+ IMGNAMES: >-
+ ${FEDORA_CACHE_IMAGE_NAME}
+ ${PRIOR_FEDORA_CACHE_IMAGE_NAME}
+ ${UBUNTU_CACHE_IMAGE_NAME}
+ ${IMAGE_BUILDER_CACHE_IMAGE_NAME}
BUILDID: "${CIRRUS_BUILD_ID}"
REPOREF: "${CIRRUS_CHANGE_IN_REPO}"
GCPJSON: ENCRYPTED[950d9c64ad78f7b1f0c7e499b42dc058d2b23aa67e38b315e68f557f2aba0bf83068d4734f7b1e1bdd22deabe99629df]
@@ -258,6 +243,8 @@ meta_task:
GCPPROJECT: ENCRYPTED[7c80e728e046b1c76147afd156a32c1c57d4a1ac1eab93b7e68e718c61ca8564fc61fef815952b8ae0a64e7034b8fe4f]
CIRRUS_CLONE_DEPTH: 1 # source not used
+ timeout_in: 10m
+
script: '/usr/local/bin/entrypoint.sh |& ${TIMESTAMP}'
@@ -270,38 +257,28 @@ testing_task:
- "vendor"
- "build_each_commit"
- env:
- matrix:
- TEST_REMOTE_CLIENT: true
- TEST_REMOTE_CLIENT: false
+ # Only test build cache-images, if that's what's requested
+ only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\*\*\*\s*CIRRUS:\s*TEST\s*IMAGES\s*\*\*\*.*'
gce_instance:
- image_project: "libpod-218412"
- zone: "us-central1-a" # Required by Cirrus for the time being
- cpu: 2
- memory: "4Gb"
- disk: 200 # see https://developers.google.com/compute/docs/disks#performance
- # Generate multiple parallel tasks, covering all possible
- # 'matrix' combinations.
matrix:
# Images are generated separately, from build_images_task (below)
image_name: "${FEDORA_CACHE_IMAGE_NAME}"
image_name: "${PRIOR_FEDORA_CACHE_IMAGE_NAME}"
image_name: "${UBUNTU_CACHE_IMAGE_NAME}"
- # TODO: Make these work (also optional_testing_task below)
- # image_name: "${PRIOR_RHEL_CACHE_IMAGE_NAME}"
- # image_name: "${RHEL_CACHE_IMAGE_NAME}"
- # image_name: "${CENTOS_CACHE_IMAGE_NAME}"
-
timeout_in: 120m
- # Every *_script runs in sequence, for each task. The name prefix is for
- # WebUI reference. The values may be strings...
+ env:
+ matrix:
+ TEST_REMOTE_CLIENT: true
+ TEST_REMOTE_CLIENT: false
+
setup_environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}'
unit_test_script: '$SCRIPT_BASE/unit_test.sh |& ${TIMESTAMP}'
integration_test_script: '$SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP}'
ginkgo_node_logs_script: 'cat $CIRRUS_WORKING_DIR/test/e2e/ginkgo-node-*.log || echo "Ginkgo node logs not found"'
+ df_script: '${DFCMD}'
audit_log_script: 'cat /var/log/audit/audit.log || cat /var/log/kern.log'
journalctl_b_script: 'journalctl -b'
@@ -309,6 +286,7 @@ testing_task:
failed_master_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_master_failure.sh'
# Job has already failed, don't fail again and miss collecting data
failed_ginkgo_node_logs_script: 'cat $CIRRUS_WORKING_DIR/test/e2e/ginkgo-node-*.log || echo "Ginkgo node logs not found"'
+ failed_df_script: '${DFCMD}'
failed_audit_log_script: 'cat /var/log/audit/audit.log || cat /var/log/kern.log || echo "Uh oh, cat audit.log failed"'
failed_journalctl_b_script: 'journalctl -b || echo "Uh oh, journalctl -b failed"'
@@ -322,30 +300,25 @@ special_testing_task:
- "vendor"
- "build_each_commit"
- gce_instance:
- image_project: "libpod-218412"
- zone: "us-central1-a" # Required by Cirrus for the time being
- cpu: 2
- memory: "4Gb"
- disk: 200
- # A matrix could be used here, for now just one VM
- image_name: "${FEDORA_CACHE_IMAGE_NAME}"
+ only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\*\*\*\s*CIRRUS:\s*TEST\s*IMAGES\s*\*\*\*.*'
env:
matrix:
SPECIALMODE: 'rootless' # See docs
SPECIALMODE: 'in_podman' # See docs
- timeout_in: 120m
+ timeout_in: 60m
setup_environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}'
integration_test_script: '$SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP}'
+ df_script: '${DFCMD}'
audit_log_script: 'cat /var/log/audit/audit.log || cat /var/log/kern.log'
journalctl_b_script: 'journalctl -b'
on_failure:
failed_master_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_master_failure.sh'
# Job has already failed, don't fail again and miss collecting data
+ failed_df_script: '${DFCMD}'
failed_audit_log_script: 'cat /var/log/audit/audit.log || cat /var/log/kern.log || echo "Uh oh, cat audit.log failed"'
failed_journalctl_b_script: 'journalctl -b || echo "Uh oh, journalctl -b failed"'
@@ -354,23 +327,22 @@ special_testing_task:
# necessary to execute them within a PR to validate changes.
optional_testing_task:
+ depends_on:
+ - "gating"
+
# Only run system tests in PRs (not on merge) if magic string is present
# in the PR description. Post-merge system testing is assumed to happen
# later from OS distribution's build systems.
only_if: >-
$CIRRUS_BRANCH != 'master' &&
+ $CIRRUS_CHANGE_MESSAGE !=~ '.*\*\*\*\s*CIRRUS:\s*TEST\s*IMAGES\s*\*\*\*.*' &&
$CIRRUS_CHANGE_MESSAGE =~ '.*\*\*\*\s*CIRRUS:\s*SYSTEM\s*TEST\s*\*\*\*.*'
gce_instance:
- image_project: "libpod-218412"
matrix:
image_name: "${FEDORA_CACHE_IMAGE_NAME}"
image_name: "${PRIOR_FEDORA_CACHE_IMAGE_NAME}"
image_name: "${UBUNTU_CACHE_IMAGE_NAME}"
- # TODO: Make these work (also testing_task above)
- # image_name: "${RHEL_CACHE_IMAGE_NAME}"
- # image_name: "${PRIOR_RHEL_CACHE_IMAGE_NAME}"
- # image_name: "${CENTOS_CACHE_IMAGE_NAME}"
timeout_in: 60m
@@ -378,13 +350,85 @@ optional_testing_task:
system_test_script: '$SCRIPT_BASE/system_test.sh |& ${TIMESTAMP}'
+# Test building of new cache-images for future PR testing, in this PR.
+test_build_cache_images_task:
+
+ only_if: >-
+ $CIRRUS_BRANCH != 'master' &&
+ $CIRRUS_CHANGE_MESSAGE =~ '.*\*\*\*\s*CIRRUS:\s*TEST\s*IMAGES\s*\*\*\*.*' &&
+ $CIRRUS_CHANGE_MESSAGE !=~ '.*\*\*\*\s*CIRRUS:\s*SYSTEM\s*TEST\s*\*\*\*.*'
+
+ depends_on:
+ - "gating"
+
+ # VMs created by packer are not cleaned up by cirrus, must allow task to complete
+ auto_cancellation: $CI != "true"
+
+ gce_instance:
+ image_project: "libpod-218412"
+ zone: "us-central1-a"
+ cpu: 4
+ memory: "4Gb"
+ disk: 200
+ image_name: "${IMAGE_BUILDER_CACHE_IMAGE_NAME}"
+ scopes: # required for image building
+ - compute
+ - devstorage.full_control
+
+ environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}'
+ build_vm_images_script: '$SCRIPT_BASE/build_vm_images.sh |& ${TIMESTAMP}'
+
+
+# Test building of new cache-images for future PR testing, in this PR.
+verify_test_built_images_task:
+
+ only_if: >-
+ $CIRRUS_BRANCH != 'master' &&
+ $CIRRUS_CHANGE_MESSAGE =~ '.*\*\*\*\s*CIRRUS:\s*TEST\s*IMAGES\s*\*\*\*.*' &&
+ $CIRRUS_CHANGE_MESSAGE !=~ '.*\*\*\*\s*CIRRUS:\s*SYSTEM\s*TEST\s*\*\*\*.*'
+
+
+ depends_on:
+ - "gating"
+ - "test_build_cache_images"
+
+ gce_instance:
+ matrix:
+ # Images are generated separately, from build_images_task (below)
+ image_name: "fedora-28${BUILT_IMAGE_SUFFIX}"
+ image_name: "fedora-29${BUILT_IMAGE_SUFFIX}"
+ image_name: "ubuntu-18${BUILT_IMAGE_SUFFIX}"
+
+ env:
+ matrix:
+ TEST_REMOTE_CLIENT: true
+ TEST_REMOTE_CLIENT: false
+
+ # Note: A truncated form of normal testing. It only needs to confirm new images
+ # "probably" work. A full round of testing will happen again after $*_CACHE_IMAGE_NAME
+ # are updated in this or another PR (w/o '***CIRRUS: TEST IMAGES***').
+ environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}'
+
+ integration_test_script: '$SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP}'
+ ginkgo_node_logs_script: 'cat $CIRRUS_WORKING_DIR/test/e2e/ginkgo-node-*.log || echo "Ginkgo node logs not found"'
+ df_script: '${DFCMD}'
+ audit_log_script: 'cat /var/log/audit/audit.log || cat /var/log/kern.log'
+ journalctl_b_script: 'journalctl -b'
+ on_failure:
+ # Job has already failed, don't fail again and miss collecting data
+ failed_ginkgo_node_logs_script: 'cat $CIRRUS_WORKING_DIR/test/e2e/ginkgo-node-*.log || echo "Ginkgo node logs not found"'
+ failed_df_script: '${DFCMD}'
+ failed_audit_log_script: 'cat /var/log/audit/audit.log || cat /var/log/kern.log || echo "Uh oh, cat audit.log failed"'
+ failed_journalctl_b_script: 'journalctl -b || echo "Uh oh, journalctl -b failed"'
+
+
# Build new cache-images for future PR testing, but only after a PR merge.
# The cache-images save install/setup time needed test every PR. The 'active' images
# are selected by the 'image_name' items tasks above. Currently this requires
# manually updating the names, but this could be automated (see comment below).
-cache_images_task:
+build_cache_images_task:
# Only produce new cache-images after a PR merge, and if a magic string
- # is present in the most recent commit-message.
+ # is present in the most recent ___commit-message___.
only_if: >-
$CIRRUS_BRANCH == 'master' &&
$CIRRUS_CHANGE_MESSAGE =~ '.*\*\*\*\s*CIRRUS:\s*REBUILD\s*IMAGES\s*\*\*\*.*'
@@ -393,6 +437,7 @@ cache_images_task:
depends_on:
- "gating"
- "testing"
+ - "rootless_testing"
# VMs created by packer are not cleaned up by cirrus
auto_cancellation: $CI != "true"
@@ -408,6 +453,7 @@ cache_images_task:
scopes:
- compute
- devstorage.full_control
+
environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}'
build_vm_images_script: '$SCRIPT_BASE/build_vm_images.sh |& ${TIMESTAMP}'
@@ -433,19 +479,24 @@ success_task:
depends_on: # ignores any dependent task conditions
- "gating"
- - "varlink_api"
+ - "build_each_commit"
- "vendor"
- - "build_each_commit_task"
+ - "varlink_api"
- "testing"
- - "rootless_testing_task"
+ - "special_testing"
- "optional_testing"
+ - "test_build_cache_images"
+ - "verify_test_built_images"
+ - "build_cache_images"
env:
CIRRUS_WORKING_DIR: "/usr/src/libpod"
+ GOPATH: "/go"
+ GOSRC: "/go/src/github.com/containers/libpod"
container:
image: "quay.io/libpod/gate:latest"
cpu: 1
memory: 1
- success_script: '$SCRIPT_BASE/success.sh |& ${TIMESTAMP}'
+ success_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/success.sh |& ${TIMESTAMP}'
diff --git a/contrib/cirrus/README.md b/contrib/cirrus/README.md
index ea358d2d7..69d8653fe 100644
--- a/contrib/cirrus/README.md
+++ b/contrib/cirrus/README.md
@@ -99,43 +99,88 @@ contents of the ``$SPECIALMODE`` environment variable.
then execute `make localsystem` from the repository root.
-### ``cache_images`` Task
+### ``test_build_cache_images_task`` Task
-Modifying the contents of cache-images is done by making changes to
-one or more of the ``./contrib/cirrus/packer/*_setup.sh`` files. Testing
-those changes currently requires adding a temporary commit to a PR that
-updates ``.cirrus.yml``:
-
-* Remove all task sections except ``cache_images_task``.
-* Remove the ``only_if`` condition and ``depends_on`` dependencies
-
-The new image names will be displayed at the end of output, assuming the build
-is successful, at that point the temporary commit may be removed. Finally,
-the new names may be used as ``image_name`` values in ``.cirrus.yml``.
+Modifying the contents of cache-images is tested by making changes to
+one or more of the ``./contrib/cirrus/packer/*_setup.sh`` files. Then
+in the PR description, add the magic string: ``***CIRRUS: TEST IMAGES***``
***N/B: Steps below are performed by automation***
-1. When a PR is merged (``$CIRRUS_BRANCH`` == ``master``), run another
- round of the ``gating`` and ``testing`` tasks (above).
-
-2. Assuming tests pass, if the commit message contains the magic string
- ``***CIRRUS: REBUILD IMAGES***``, then this task continues. Otherwise
- simply mark the master branch as 'passed'.
+1. ``setup_environment.sh``: Same as for other tasks.
-3. ``setup_environment.sh``: Same as for other tasks.
-
-4. ``build_vm_images.sh``: Utilize [the packer tool](http://packer.io/docs/)
+2. ``build_vm_images.sh``: Utilize [the packer tool](http://packer.io/docs/)
to produce new VM images. Create a new VM from each base-image, connect
to them with ``ssh``, and perform the steps as defined by the
- ``$PACKER_BASE/libpod_images.json`` file:
+ ``$PACKER_BASE/libpod_images.yml`` file:
1. On a base-image VM, as root, copy the current state of the repository
into ``/tmp/libpod``.
2. Execute distribution-specific scripts to prepare the image for
- use by the ``integration_testing`` task (above). For example,
- ``fedora_setup.sh``.
- 3. If successful, shut down each VM and create a new GCE Image
- named with the base image, and the commit sha of the merge.
+ use. For example, ``fedora_setup.sh``.
+ 3. If successful, shut down each VM and record the names, and dates
+ into a json manifest file.
+ 4. Move the manifest file, into a google storage bucket object.
+ This is a retained as a secondary method for tracking/auditing
+ creation of VM images, should it ever be needed.
+
+### ``verify_test_built_images`` Task
+
+Only runs following successful ``test_build_cache_images_task`` task. Uses
+images following the standard naming format; ***however, only runs a limited
+sub-set of automated tests***. Validating newly built images fully, requires
+updating ``.cirrus.yml``.
+
+***Manual Steps:*** Assuming `verify_test_built_images` passes, then
+you'll find the new image names displayed at the end of the
+`test_build_cache_images_task` in the `build_vm_images` output.
+For example:
+
+
+```
+...cut...
+==> Builds finished. The artifacts of successful builds are:
+--> ubuntu-18: A disk image was created: ubuntu-18-libpod-5699523102900224
+--> ubuntu-18:
+--> fedora-29: A disk image was created: fedora-29-libpod-5699523102900224
+--> fedora-29:
+--> fedora-28: A disk image was created: fedora-28-libpod-5699523102900224
+```
+
+Now edit `.cirrus.yml`, updating the `*_IMAGE_NAME` lines to reflect the
+images from above:
+
+
+```yaml
+env:
+ ...cut...
+ ####
+ #### Cache-image names to test with
+ ###
+ FEDORA_CACHE_IMAGE_NAME: "fedora-29-libpod-5699523102900224"
+ PRIOR_FEDORA_CACHE_IMAGE_NAME: "fedora-28-libpod-5699523102900224"
+ UBUNTU_CACHE_IMAGE_NAME: "ubuntu-18-libpod-5699523102900224"
+ ...cut...
+```
+
+***NOTE:*** If re-using the same PR with new images in `.cirrus.yml`,
+take care to also *update the PR description* to remove
+the magic ``***CIRRUS: TEST IMAGES***`` string. Keeping it and
+`--force` pushing would needlessly cause Cirrus-CI to build
+and test images again.
+
+
+### ``build_cache_images`` Task *(Deprecated)*
+
+Exactly the same as ``test_build_cache_images_task`` task, but only runs on
+the master branch. Requires a magic string to be in the `HEAD`
+commit message: ``***CIRRUS: BUILD IMAGES***``
+
+When successful, the manifest file along with all VM disks, are moved
+into a dedicated google storage bucket, separate from the one used by
+`test_build_cache_images_task`. These may be used to create new cache-images for
+PR testing by manually importing them as described above.
+
### Base-images
@@ -170,15 +215,6 @@ the ``cache_images`` Task) some input parameters are required:
or [end-user
credentials](https://cloud.google.com/docs/authentication/end-user#creating_your_client_credentials)
-* ``RHEL_IMAGE_FILE`` and ``RHEL_CSUM_FILE`` complete paths
- to a `rhel-server-ec2-*.raw.xz` and it's cooresponding
- checksum file. These must be supplied manually because
- they're not available directly via URL like other images.
-
-* ``RHSM_COMMAND`` contains the complete string needed to register
- the VM for installing package dependencies. The VM will be de-registered
- upon completion.
-
* Optionally, CSV's may be specified to ``PACKER_BUILDS``
to limit the base-images produced. For example,
``PACKER_BUILDS=fedora,image-builder-image``.
@@ -224,9 +260,6 @@ When ready, change to the ``packer`` sub-directory, and build the images:
$ cd libpod/contrib/cirrus/packer
$ make libpod_base_images GCP_PROJECT_ID=<VALUE> \
GOOGLE_APPLICATION_CREDENTIALS=<VALUE> \
- RHEL_IMAGE_FILE=<VALUE> \
- RHEL_CSUM_FILE=<VALUE> \
- RHSM_COMMAND=<VALUE> \
PACKER_BUILDS=<OPTIONAL>
```
diff --git a/contrib/cirrus/build_vm_images.sh b/contrib/cirrus/build_vm_images.sh
index 43eb3c057..805aba428 100755
--- a/contrib/cirrus/build_vm_images.sh
+++ b/contrib/cirrus/build_vm_images.sh
@@ -3,7 +3,10 @@
set -e
source $(dirname $0)/lib.sh
-req_env_var CNI_COMMIT CRIO_COMMIT RUNC_COMMIT PACKER_BUILDS BUILT_IMAGE_SUFFIX CENTOS_BASE_IMAGE UBUNTU_BASE_IMAGE FEDORA_BASE_IMAGE FAH_BASE_IMAGE RHEL_BASE_IMAGE RHSM_COMMAND SERVICE_ACCOUNT GCE_SSH_USERNAME GCP_PROJECT_ID PACKER_VER SCRIPT_BASE PACKER_BASE
+ENV_VARS='CNI_COMMIT CONMON_COMMIT PACKER_BUILDS BUILT_IMAGE_SUFFIX UBUNTU_BASE_IMAGE FEDORA_BASE_IMAGE PRIOR_FEDORA_BASE_IMAGE SERVICE_ACCOUNT GCE_SSH_USERNAME GCP_PROJECT_ID PACKER_VER SCRIPT_BASE PACKER_BASE'
+req_env_var $ENV_VARS
+# Must also be made available through make, into packer process
+export $ENV_VARS
show_env_vars
@@ -46,7 +49,4 @@ make libpod_images \
URI="gs://packer-import${POST_MERGE_BUCKET_SUFFIX}/manifest${BUILT_IMAGE_SUFFIX}.json"
gsutil cp packer-manifest.json "$URI"
-echo "Finished."
-echo "Any tarball URI's referenced above at at $URI"
-echo "may be used to create VM images suitable for use in"
-echo ".cirrus.yml as values for the 'image_name' keys."
+echo "Finished. A JSON manifest of produced images is available at $URI"
diff --git a/contrib/cirrus/integration_test.sh b/contrib/cirrus/integration_test.sh
index c7d381318..e7f582b42 100755
--- a/contrib/cirrus/integration_test.sh
+++ b/contrib/cirrus/integration_test.sh
@@ -9,7 +9,6 @@ cd "$GOSRC"
if [[ "$SPECIALMODE" == "in_podman" ]]
then
- set -x
${CONTAINER_RUNTIME} run --rm --privileged --net=host \
-v $GOSRC:$GOSRC:Z \
--workdir $GOSRC \
@@ -26,30 +25,19 @@ then
elif [[ "$SPECIALMODE" == "rootless" ]]
then
req_env_var ROOTLESS_USER
- set -x
- ssh $ROOTLESS_USER@localhost \
+
+ if [[ "$USER" == "$ROOTLESS_USER" ]]
+ then
+ $GOSRC/$SCRIPT_BASE/rootless_test.sh
+ else
+ ssh $ROOTLESS_USER@localhost \
-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o CheckHostIP=no \
$GOSRC/$SCRIPT_BASE/rootless_test.sh
- exit $?
+ fi
else
- set -x
make
make install PREFIX=/usr ETCDIR=/etc
make test-binaries
- make install.tools
- clean_env
-
- case "${OS_RELEASE_ID}-${OS_RELEASE_VER}" in
- ubuntu-18) ;;
- fedora-29) ;& # Continue to the next item
- fedora-28) ;&
- centos-7) ;&
- rhel-7)
- make podman-remote
- install bin/podman-remote /usr/bin
- ;;
- *) bad_os_id_ver ;;
- esac
if [[ "$TEST_REMOTE_CLIENT" == "true" ]]
then
make remoteintegration
diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh
index a285b133b..334202aa9 100644
--- a/contrib/cirrus/lib.sh
+++ b/contrib/cirrus/lib.sh
@@ -3,35 +3,92 @@
# Library of common, shared utility functions. This file is intended
# to be sourced by other scripts, not called directly.
+# Global details persist here
+source /etc/environment # not always loaded under all circumstances
+
# Under some contexts these values are not set, make sure they are.
-export USER="$(whoami)"
-export HOME="$(getent passwd $USER | cut -d : -f 6)"
-
-# These are normally set by cirrus, but can't be for VMs setup by hack/get_ci_vm.sh
-# Pick some reasonable defaults
-ENVLIB=${ENVLIB:-.bash_profile}
-CIRRUS_WORKING_DIR="${CIRRUS_WORKING_DIR:-/var/tmp/go/src/github.com/containers/libpod}"
-GOSRC="${GOSRC:-$CIRRUS_WORKING_DIR}"
+USER="$(whoami)"
+HOME="$(getent passwd $USER | cut -d : -f 6)"
+[[ -n "$UID" ]] || UID=$(getent passwd $USER | cut -d : -f 3)
+GID=$(getent passwd $USER | cut -d : -f 4)
+
+# Essential default paths, many are overriden when executing under Cirrus-CI
+export GOPATH="${GOPATH:-/var/tmp/go}"
+if type -P go &> /dev/null
+then
+ # required for go 1.12+
+ export GOCACHE="${GOCACHE:-$HOME/.cache/go-build}"
+ eval "$(go env)"
+ # required by make and other tools
+ export $(go env | cut -d '=' -f 1)
+
+ # Ensure compiled tooling is reachable
+ export PATH="$PATH:$GOPATH/bin"
+fi
+CIRRUS_WORKING_DIR="${CIRRUS_WORKING_DIR:-$GOPATH/src/github.com/containers/libpod}"
+export GOSRC="${GOSRC:-$CIRRUS_WORKING_DIR}"
+export PATH="$HOME/bin:$GOPATH/bin:/usr/local/bin:$PATH"
+export LD_LIBRARY_PATH="/usr/local/lib${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}"
+TIMESTAMPS_FILEPATH="${TIMESTAMPS_FILEPATH:-/var/tmp/timestamps}"
+SETUP_MARKER_FILEPATH="${SETUP_MARKER_FILEPATH:-/var/tmp/.setup_environment_sh_complete}"
+# Saves typing / in case location ever moves
SCRIPT_BASE=${SCRIPT_BASE:-./contrib/cirrus}
PACKER_BASE=${PACKER_BASE:-./contrib/cirrus/packer}
-CIRRUS_BUILD_ID=${CIRRUS_BUILD_ID:-DEADBEEF} # a human
-CIRRUS_BASE_SHA=${CIRRUS_BASE_SHA:-HEAD}
-CIRRUS_CHANGE_IN_REPO=${CIRRUS_CHANGE_IN_REPO:-FETCH_HEAD}
-SPECIALMODE="${SPECIALMODE:-none}"
-export CONTAINER_RUNTIME=${CONTAINER_RUNTIME:-podman}
-if ! [[ "$PATH" =~ "/usr/local/bin" ]]
+cd $GOSRC
+if type -P git &> /dev/null
then
- export PATH="$PATH:/usr/local/bin"
+ CIRRUS_CHANGE_IN_REPO=${CIRRUS_CHANGE_IN_REPO:-$(git show-ref --hash=8 HEAD || date +%s)}
+else # pick something unique and obviously not from Cirrus
+ CIRRUS_CHANGE_IN_REPO=${CIRRUS_CHANGE_IN_REPO:-no_git_$(date +%s)}
fi
-# In ci/testing environment, ensure variables are always loaded
-if [[ -r "$HOME/$ENVLIB" ]] && [[ -n "$CI" ]]
+# Defaults when not running under CI
+export CI="${CI:-false}"
+CIRRUS_CI="${CIRRUS_CI:-false}"
+CONTINUOUS_INTEGRATION="${CONTINUOUS_INTEGRATION:-false}"
+CIRRUS_REPO_NAME=${CIRRUS_REPO_NAME:-libpod}
+CIRRUS_BASE_SHA=${CIRRUS_BASE_SHA:-unknown$(date +%s)} # difficult to reliably discover
+CIRRUS_BUILD_ID=${CIRRUS_BUILD_ID:-$RANDOM$(date +%s)} # must be short and unique
+# Vars. for image-building
+PACKER_VER="1.3.5"
+# CSV of cache-image names to build (see $PACKER_BASE/libpod_images.json)
+
+# Base-images rarely change, define them here so they're out of the way.
+PACKER_BUILDS="${PACKER_BUILDS:-ubuntu-18,fedora-29,fedora-28}"
+# Google-maintained base-image names
+UBUNTU_BASE_IMAGE="ubuntu-1804-bionic-v20181203a"
+# Manually produced base-image names (see $SCRIPT_BASE/README.md)
+FEDORA_BASE_IMAGE="fedora-cloud-base-29-1-2-1541789245"
+# FEDORA_BASE_IMAGE: "fedora-cloud-base-30-1-2-1556821664"
+PRIOR_FEDORA_BASE_IMAGE="fedora-cloud-base-28-1-1-1544474897"
+# PRIOR_FEDORA_BASE_IMAGE="fedora-cloud-base-29-1-2-1541789245"
+BUILT_IMAGE_SUFFIX="${BUILT_IMAGE_SUFFIX:--$CIRRUS_REPO_NAME-${CIRRUS_BUILD_ID}}"
+
+# Safe env. vars. to transfer from root -> $ROOTLESS_USER (go env handled separetly)
+ROOTLESS_ENV_RE='(CIRRUS_.+)|(ROOTLESS_.+)|(.+_IMAGE.*)|(.+_BASE)|(.*DIRPATH)|(.*FILEPATH)|(SOURCE.*)|(DEPEND.*)|(.+_DEPS_.+)|(OS_REL.*)|(.+_ENV_RE)|(TRAVIS)|(CI.+)'
+# Unsafe env. vars for display
+SECRET_ENV_RE='(IRCID)|(ACCOUNT)|(^GC[EP]..+)|(SSH)'
+
+SPECIALMODE="${SPECIALMODE:-none}"
+TEST_REMOTE_CLIENT="${TEST_REMOTE_CLIENT:-false}"
+export CONTAINER_RUNTIME=${CONTAINER_RUNTIME:-podman}
+
+# When running as root, this may be empty or not, as a user, it MUST be set.
+if [[ "$USER" == "root" ]]
then
- # Make sure this is always loaded
- source "$HOME/$ENVLIB"
+ ROOTLESS_USER="${ROOTLESS_USER:-}"
+else
+ ROOTLESS_USER="${ROOTLESS_USER:-$USER}"
fi
+# GCE image-name compatible string representation of distribution name
+OS_RELEASE_ID="$(source /etc/os-release; echo $ID)"
+# GCE image-name compatible string representation of distribution _major_ version
+OS_RELEASE_VER="$(source /etc/os-release; echo $VERSION_ID | cut -d '.' -f 1)"
+# Combined to ease soe usage
+OS_REL_VER="${OS_RELEASE_ID}-${OS_RELEASE_VER}"
+
# Pass in a list of one or more envariable names; exit non-zero with
# helpful error message if any value is empty
req_env_var() {
@@ -57,81 +114,30 @@ req_env_var() {
done
}
-# Some env. vars may contain secrets. Display values for known "safe"
-# and useful variables.
-# ref: https://cirrus-ci.org/guide/writing-tasks/#environment-variables
show_env_vars() {
- # This is almost always multi-line, print it separately
- echo "export CIRRUS_CHANGE_MESSAGE=$CIRRUS_CHANGE_MESSAGE"
- echo "
-BUILDTAGS $BUILDTAGS
-BUILT_IMAGE_SUFFIX $BUILT_IMAGE_SUFFIX
-ROOTLESS_USER $ROOTLESS_USER
-CI $CI
-CIRRUS_CI $CIRRUS_CI
-CI_NODE_INDEX $CI_NODE_INDEX
-CI_NODE_TOTAL $CI_NODE_TOTAL
-CONTINUOUS_INTEGRATION $CONTINUOUS_INTEGRATION
-CIRRUS_BASE_BRANCH $CIRRUS_BASE_BRANCH
-CIRRUS_BASE_SHA $CIRRUS_BASE_SHA
-CIRRUS_BRANCH $CIRRUS_BRANCH
-CIRRUS_BUILD_ID $CIRRUS_BUILD_ID
-CIRRUS_CHANGE_IN_REPO $CIRRUS_CHANGE_IN_REPO
-CIRRUS_CLONE_DEPTH $CIRRUS_CLONE_DEPTH
-CIRRUS_DEFAULT_BRANCH $CIRRUS_DEFAULT_BRANCH
-CIRRUS_PR $CIRRUS_PR
-CIRRUS_TAG $CIRRUS_TAG
-CIRRUS_OS $CIRRUS_OS
-OS $OS
-CIRRUS_TASK_NAME $CIRRUS_TASK_NAME
-CIRRUS_TASK_ID $CIRRUS_TASK_ID
-CIRRUS_REPO_NAME $CIRRUS_REPO_NAME
-CIRRUS_REPO_OWNER $CIRRUS_REPO_OWNER
-CIRRUS_REPO_FULL_NAME $CIRRUS_REPO_FULL_NAME
-CIRRUS_REPO_CLONE_URL $CIRRUS_REPO_CLONE_URL
-CIRRUS_SHELL $CIRRUS_SHELL
-CIRRUS_USER_COLLABORATOR $CIRRUS_USER_COLLABORATOR
-CIRRUS_USER_PERMISSION $CIRRUS_USER_PERMISSION
-CIRRUS_WORKING_DIR $CIRRUS_WORKING_DIR
-CIRRUS_HTTP_CACHE_HOST $CIRRUS_HTTP_CACHE_HOST
-SPECIALMODE $SPECIALMODE
-$(go env)
-PACKER_BUILDS $PACKER_BUILDS
- " | while read NAME VALUE
+ echo "Showing selection of environment variable definitions:"
+ _ENV_VAR_NAMES=$(awk 'BEGIN{for(v in ENVIRON) print v}' | \
+ egrep -v "(^PATH$)|(^BASH_FUNC)|(^[[:punct:][:space:]]+)|$SECRET_ENV_RE" | \
+ sort -u)
+ for _env_var_name in $_ENV_VAR_NAMES
do
- [[ -z "$NAME" ]] || echo "export $NAME=\"$VALUE\""
+ # Supports older BASH versions
+ printf " ${_env_var_name}=%q\n" "$(printenv $_env_var_name)"
done
echo ""
echo "##### $(go version) #####"
echo ""
}
-# Unset environment variables not needed for testing purposes
-clean_env() {
- req_env_var UNSET_ENV_VARS
- echo "Unsetting $(echo $UNSET_ENV_VARS | wc -w) environment variables"
- unset -v UNSET_ENV_VARS $UNSET_ENV_VARS || true # don't fail on read-only
-}
-
die() {
- echo "${2:-FATAL ERROR (but no message given!) in ${FUNCNAME[1]}()}"
+ echo "************************************************"
+ echo ">>>>> ${2:-FATAL ERROR (but no message given!) in ${FUNCNAME[1]}()}"
+ echo "************************************************"
exit ${1:-1}
}
-# Return a GCE image-name compatible string representation of distribution name
-os_release_id() {
- eval "$(egrep -m 1 '^ID=' /etc/os-release | tr -d \' | tr -d \")"
- echo "$ID"
-}
-
-# Return a GCE image-name compatible string representation of distribution major version
-os_release_ver() {
- eval "$(egrep -m 1 '^VERSION_ID=' /etc/os-release | tr -d \' | tr -d \")"
- echo "$VERSION_ID" | cut -d '.' -f 1
-}
-
bad_os_id_ver() {
- echo "Unknown/Unsupported distro. $OS_RELEASE_ID and/or version $OS_RELEASE_VER for $ARGS"
+ echo "Unknown/Unsupported distro. $OS_RELEASE_ID and/or version $OS_RELEASE_VER for $(basename $0)"
exit 42
}
@@ -140,8 +146,8 @@ stub() {
}
ircmsg() {
- req_env_var CIRRUS_TASK_ID
- [[ -n "$*" ]] || die 9 "ircmsg() invoked without args"
+ req_env_var CIRRUS_TASK_ID IRCID
+ [[ -n "$*" ]] || die 9 "ircmsg() invoked without message text argument"
# Sometimes setup_environment.sh didn't run
SCRIPT="$(dirname $0)/podbot.py"
NICK="podbot_$CIRRUS_TASK_ID"
@@ -153,8 +159,9 @@ ircmsg() {
}
setup_rootless() {
- req_env_var ROOTLESS_USER GOSRC ENVLIB
+ req_env_var ROOTLESS_USER GOSRC
+ # Only do this once
if passwd --status $ROOTLESS_USER
then
echo "Updating $ROOTLESS_USER user permissions on possibly changed libpod code"
@@ -162,12 +169,7 @@ setup_rootless() {
return 0
fi
- # Only do this once
cd $GOSRC
- make install.catatonit
- go get github.com/onsi/ginkgo/ginkgo
- go get github.com/onsi/gomega/...
-
# Guarantee independence from specific values
ROOTLESS_UID=$[RANDOM+1000]
ROOTLESS_GID=$[RANDOM+1000]
@@ -177,7 +179,8 @@ setup_rootless() {
chown -R $ROOTLESS_USER:$ROOTLESS_USER "$GOSRC"
echo "creating ssh keypair for $USER"
- ssh-keygen -P "" -f $HOME/.ssh/id_rsa
+ [[ -r "$HOME/.ssh/id_rsa" ]] || \
+ ssh-keygen -P "" -f "$HOME/.ssh/id_rsa"
echo "Allowing ssh key for $ROOTLESS_USER"
(umask 077 && mkdir "/home/$ROOTLESS_USER/.ssh")
@@ -192,16 +195,19 @@ setup_rootless() {
echo "${ROOTLESS_USER}:$[ROOTLESS_UID * 100]:65536" | \
tee -a /etc/subuid >> /etc/subgid
- echo "Copying $HOME/$ENVLIB"
- install -o $ROOTLESS_USER -g $ROOTLESS_USER -m 0700 \
- "$HOME/$ENVLIB" "/home/$ROOTLESS_USER/$ENVLIB"
-
- echo "Configuring user's go environment variables"
- su --login --command 'go env' $ROOTLESS_USER | \
- while read envline
- do
- X=$(echo "export $envline" | tee -a "/home/$ROOTLESS_USER/$ENVLIB") && echo "$X"
- done
+ # Env. vars set by Cirrus and setup_environment.sh must be explicitly
+ # transfered to the test-user.
+ echo "Configuring rootless user's environment variables:"
+ echo "# Added by $GOSRC/$SCRIPT_PATH/lib.sh setup_rootless()"
+ _ENV_VAR_NAMES=$(awk 'BEGIN{for(v in ENVIRON) print v}' | \
+ egrep -v "(^PATH$)|(^BASH_FUNC)|(^[[:punct:][:space:]]+)|$SECRET_ENV_RE" | \
+ egrep "$ROOTLESS_ENV_RE" | \
+ sort -u)
+ for _env_var_name in $_ENV_VAR_NAMES
+ do
+ # Works with older versions of bash
+ printf "${_env_var_name}=%q\n" "$(printenv $_env_var_name)" >> "/home/$ROOTLESS_USER/.bashrc"
+ done
}
# Helper/wrapper script to only show stderr/stdout on non-zero exit
@@ -239,6 +245,7 @@ install_cni_plugins() {
}
install_runc_from_git(){
+ req_env_var GOPATH OS_RELEASE_ID RUNC_COMMIT
wd=$(pwd)
DEST="$GOPATH/src/github.com/opencontainers/runc"
rm -rf "$DEST"
@@ -246,13 +253,17 @@ install_runc_from_git(){
cd "$DEST"
ooe.sh git fetch origin --tags
ooe.sh git checkout -q "$RUNC_COMMIT"
- ooe.sh make static BUILDTAGS="seccomp apparmor selinux"
+ if [[ "${OS_RELEASE_ID}" == "ubuntu" ]]
+ then
+ ooe.sh make static BUILDTAGS="seccomp apparmor"
+ else
+ ooe.sh make BUILDTAGS="seccomp selinux"
+ fi
sudo install -m 755 runc /usr/bin/runc
cd $wd
}
install_runc(){
- OS_RELEASE_ID=$(os_release_id)
echo "Installing RunC from commit $RUNC_COMMIT"
echo "Platform is $OS_RELEASE_ID"
req_env_var GOPATH RUNC_COMMIT OS_RELEASE_ID
@@ -298,8 +309,6 @@ install_conmon(){
}
install_criu(){
- OS_RELEASE_ID=$(os_release_id)
- OS_RELEASE_VER=$(os_release_ver)
echo "Installing CRIU"
echo "Installing CRIU from commit $CRIU_COMMIT"
echo "Platform is $OS_RELEASE_ID"
@@ -309,21 +318,6 @@ install_criu(){
ooe.sh sudo -E add-apt-repository -y ppa:criu/ppa
ooe.sh sudo -E apt-get -qq -y update
ooe.sh sudo -E apt-get -qq -y install criu
- elif [[ ( "$OS_RELEASE_ID" =~ "centos" || "$OS_RELEASE_ID" =~ "rhel" ) && "$OS_RELEASE_VER" =~ "7"* ]]; then
- echo "Configuring Repositories for latest CRIU"
- ooe.sh sudo tee /etc/yum.repos.d/adrian-criu-el7.repo <<EOF
-[adrian-criu-el7]
-name=Copr repo for criu-el7 owned by adrian
-baseurl=https://copr-be.cloud.fedoraproject.org/results/adrian/criu-el7/epel-7-$basearch/
-type=rpm-md
-skip_if_unavailable=True
-gpgcheck=1
-gpgkey=https://copr-be.cloud.fedoraproject.org/results/adrian/criu-el7/pubkey.gpg
-repo_gpgcheck=0
-enabled=1
-enabled_metadata=1
-EOF
- ooe.sh sudo yum -y install criu
elif [[ "$OS_RELEASE_ID" =~ "fedora" ]]; then
echo "Using CRIU from distribution"
else
@@ -338,16 +332,6 @@ EOF
fi
}
-install_packer_copied_files(){
- # Install cni config, policy and registry config
- sudo install -D -m 755 /tmp/libpod/cni/87-podman-bridge.conflist \
- /etc/cni/net.d/87-podman-bridge.conflist
- sudo install -D -m 755 /tmp/libpod/test/policy.json \
- /etc/containers/policy.json
- sudo install -D -m 755 /tmp/libpod/test/redhat_sigstore.yaml \
- /etc/containers/registries.d/registry.access.redhat.com.yaml
-}
-
install_varlink() {
echo "Installing varlink from the cheese-factory"
ooe.sh sudo -H pip3 install varlink
@@ -376,7 +360,7 @@ rh_finalize(){
fi
echo "Resetting to fresh-state for usage as cloud-image."
PKG=$(type -P dnf || type -P yum || echo "")
- [[ -z "$PKG" ]] || sudo $PKG clean all # not on atomic
+ sudo $PKG clean all
sudo rm -rf /var/cache/{yum,dnf}
sudo rm -f /etc/udev/rules.d/*-persistent-*.rules
sudo touch /.unconfigured # force firstboot to run
@@ -389,25 +373,3 @@ ubuntu_finalize(){
sudo rm -rf /var/cache/apt
_finalize
}
-
-rhel_exit_handler() {
- set +ex
- req_env_var GOPATH RHSMCMD
- cd /
- sudo rm -rf "$RHSMCMD"
- sudo rm -rf "$GOPATH"
- sudo subscription-manager remove --all
- sudo subscription-manager unregister
- sudo subscription-manager clean
-}
-
-rhsm_enable() {
- req_env_var RHSM_COMMAND
- export GOPATH="$(mktemp -d)"
- export RHSMCMD="$(mktemp)"
- trap "rhel_exit_handler" EXIT
- # Avoid logging sensitive details
- echo "$RHSM_COMMAND" > "$RHSMCMD"
- ooe.sh sudo bash "$RHSMCMD"
- sudo rm -rf "$RHSMCMD"
-}
diff --git a/contrib/cirrus/lib.sh.t b/contrib/cirrus/lib.sh.t
index ce51f8ad2..1f05b3bb5 100755
--- a/contrib/cirrus/lib.sh.t
+++ b/contrib/cirrus/lib.sh.t
@@ -10,12 +10,13 @@ rc=0
function check_result {
testnum=$(expr $testnum + 1)
- if [ "$1" = "$2" ]; then
- echo "ok $testnum $3 = $1"
+ MSG=$(echo "$1" | tr -d '*>\012'|sed -e 's/^ \+//')
+ if [ "$MSG" = "$2" ]; then
+ echo "ok $testnum $3 = $MSG"
else
echo "not ok $testnum $3"
echo "# expected: $2"
- echo "# actual: $1"
+ echo "# actual: $MSG"
rc=1
fi
}
@@ -64,11 +65,17 @@ test_rev '' 1 'FATAL: req_env_var: invoked without arguments'
unset FOO BAR
test_rev FOO 9 'FATAL: test_rev() requires $FOO to be non-empty'
test_rev BAR 9 'FATAL: test_rev() requires $BAR to be non-empty'
-
-# OK if desired envariable is unset
+# OK if desired envariable was unset
FOO=1
test_rev FOO 0 ''
+# OK if multiple vars are non-empty
+FOO="stuff"
+BAR="things"
+ENV_VARS="FOO BAR"
+test_rev "$ENV_VARS" 0 ''
+unset BAR
+
# ...but error if any single desired one is unset
test_rev "FOO BAR" 9 'FATAL: test_rev() requires $BAR to be non-empty'
diff --git a/contrib/cirrus/notice_master_failure.sh b/contrib/cirrus/notice_master_failure.sh
index 4b09331d3..1fc15cdf9 100755
--- a/contrib/cirrus/notice_master_failure.sh
+++ b/contrib/cirrus/notice_master_failure.sh
@@ -12,7 +12,7 @@ NOR="$(echo -n -e '\x0f')"
if [[ "$CIRRUS_BRANCH" =~ "master" ]]
then
BURL="https://cirrus-ci.com/build/$CIRRUS_BUILD_ID"
- ircmsg "${RED}[Action Recommended]: ${NOR}Post-merge testing ${RED}$CIRRUS_BRANCH failed${NOR} in $CIRRUS_TASK_NAME on $(os_release_id)-$(os_release_ver): $BURL. Please investigate, and re-run if appropriate."
+ ircmsg "${RED}[Action Recommended]: ${NOR}Post-merge testing ${RED}$CIRRUS_BRANCH failed${NOR} in $CIRRUS_TASK_NAME on $(OS_RELEASE_ID)-$(OS_RELEASE_VER): $BURL. Please investigate, and re-run if appropriate."
fi
# This script assumed to be executed on failure
diff --git a/contrib/cirrus/packer/Makefile b/contrib/cirrus/packer/Makefile
index 91a1dfeef..d03d22abe 100644
--- a/contrib/cirrus/packer/Makefile
+++ b/contrib/cirrus/packer/Makefile
@@ -3,7 +3,7 @@
# builder name(s) from applicable YAML file,
# e.g for names see libpod_images.yml
-PACKER_VER ?= 1.3.2
+PACKER_VER ?= 1.3.5
GOARCH=$(shell go env GOARCH)
ARCH=$(uname -m)
PACKER_DIST_FILENAME := packer_${PACKER_VER}_linux_${GOARCH}.zip
@@ -49,6 +49,7 @@ ifndef PACKER_BUILDS
$(error PACKER_BUILDS is undefined, expected builder-names CSV)
endif
./packer build -only=${PACKER_BUILDS} \
+ -force \
-var GOSRC=$(GOSRC) \
-var PACKER_BASE=$(PACKER_BASE) \
-var SCRIPT_BASE=$(SCRIPT_BASE) \
@@ -78,16 +79,11 @@ endif
ifndef GOOGLE_APPLICATION_CREDENTIALS
$(error GOOGLE_APPLICATION_CREDENTIALS is undefined, expected absolute path to JSON file, like $HOME/.config/gcloud/legacy_credentials/*/adc.json)
endif
-ifndef RHEL_IMAGE_FILE
- $(error RHEL_IMAGE_FILE is undefined, expected full path to a rhel-server-ec2-*.raw.xz file)
-endif
-ifndef RHEL_CSUM_FILE
- $(error RHEL_CSUM_FILE is undefined, expected full path to a rhel-server-ec2-*.raw.xz.SHA256SUM file)
-endif
-ifndef RHSM_COMMAND
- $(error RHSM_COMMAND is undefined, expected string required for temporarily registering VM)
+ifndef PACKER_BUILDS
+ $(error PACKER_BUILDS is undefined, expected builder-names CSV)
endif
PACKER_CACHE_DIR=/tmp ./packer build \
+ -force \
-var TIMESTAMP=$(TIMESTAMP) \
-var TTYDEV=$(TTYDEV) \
-var GCP_PROJECT_ID=$(GCP_PROJECT_ID) \
@@ -95,10 +91,5 @@ endif
-var GOSRC=$(GOSRC) \
-var PACKER_BASE=$(PACKER_BASE) \
-var SCRIPT_BASE=$(SCRIPT_BASE) \
- -var RHEL_BASE_IMAGE_NAME=$(shell basename $(RHEL_IMAGE_FILE) | tr -d '[[:space:]]' | sed -r -e 's/\.${ARCH}\.raw\.xz//' | tr '[[:upper:]]' '[[:lower:]]' | tr '[[:punct:]]' '-') \
- -var RHEL_IMAGE_FILE=$(RHEL_IMAGE_FILE) \
- -var RHEL_CSUM_FILE=$(RHEL_CSUM_FILE) \
- -var 'RHSM_COMMAND=$(RHSM_COMMAND)' \
- -var POST_MERGE_BUCKET_SUFFIX=$(POST_MERGE_BUCKET_SUFFIX) \
-only $(PACKER_BUILDS) \
libpod_base_images.json
diff --git a/contrib/cirrus/packer/centos_setup.sh b/contrib/cirrus/packer/centos_setup.sh
deleted file mode 100644
index 91b1963c2..000000000
--- a/contrib/cirrus/packer/centos_setup.sh
+++ /dev/null
@@ -1,82 +0,0 @@
-#!/bin/bash
-
-# This script is called by packer on the subject CentOS VM, to setup the podman
-# build/test environment. It's not intended to be used outside of this context.
-
-set -e
-
-# Load in library (copied by packer, before this script was run)
-source /tmp/libpod/$SCRIPT_BASE/lib.sh
-
-req_env_var SCRIPT_BASE CNI_COMMIT CRIO_COMMIT CRIU_COMMIT
-
-install_ooe
-
-export GOPATH="$(mktemp -d)"
-trap "sudo rm -rf $GOPATH" EXIT
-
-ooe.sh sudo yum -y update
-
-ooe.sh sudo yum -y install centos-release-scl epel-release
-
-ooe.sh sudo yum -y install \
- PyYAML \
- atomic-registries \
- bats \
- btrfs-progs-devel \
- bzip2 \
- device-mapper-devel \
- emacs-nox \
- findutils \
- glib2-devel \
- glibc-static \
- gnupg \
- golang \
- golang-github-cpuguy83-go-md2man \
- golang-github-cpuguy83-go-md2man \
- gpgme-devel \
- iptables \
- libassuan-devel \
- libcap-devel \
- libnet \
- libnet-devel \
- libnl3-devel \
- libseccomp-devel \
- libselinux-devel \
- lsof \
- make \
- nmap-ncat \
- ostree-devel \
- protobuf \
- protobuf-c \
- protobuf-c-devel \
- protobuf-compiler \
- protobuf-devel \
- protobuf-python \
- python \
- python2-future \
- python3-dateutil \
- python3-psutil \
- python3-pytoml \
- runc \
- skopeo-containers \
- unzip \
- vim \
- which \
- xz
-
-install_scl_git
-
-install_cni_plugins
-
-install_buildah
-
-install_conmon
-
-install_criu
-
-install_packer_copied_files
-
-rh_finalize
-
-echo "SUCCESS!"
diff --git a/contrib/cirrus/packer/fah_base-setup.sh b/contrib/cirrus/packer/fah_base-setup.sh
deleted file mode 100644
index 606c4f336..000000000
--- a/contrib/cirrus/packer/fah_base-setup.sh
+++ /dev/null
@@ -1,45 +0,0 @@
-
-# N/B: This script is not intended to be run by humans. It is used to configure the
-# FAH base image for importing, so that it will boot in GCE.
-
-set -e
-
-# Load in library (copied by packer, before this script was run)
-source $GOSRC/$SCRIPT_BASE/lib.sh
-
-install_ooe
-
-if [[ "$1" == "pre" ]]
-then
- echo "Upgrading Atomic Host"
- setenforce 0
- ooe.sh atomic host upgrade
-
- echo "Configuring Repositories"
- ooe.sh sudo tee /etc/yum.repos.d/ngompa-gce-oslogin.repo <<EOF
-[ngompa-gce-oslogin]
-name=Copr repo for gce-oslogin owned by ngompa
-baseurl=https://copr-be.cloud.fedoraproject.org/results/ngompa/gce-oslogin/fedora-\$releasever-\$basearch/
-type=rpm-md
-skip_if_unavailable=True
-gpgcheck=1
-gpgkey=https://copr-be.cloud.fedoraproject.org/results/ngompa/gce-oslogin/pubkey.gpg
-repo_gpgcheck=0
-enabled=1
-enabled_metadata=1
-EOF
- echo "Installing necessary packages and google services"
- # Google services are enabled by default, upon install.
- ooe.sh rpm-ostree install rng-tools google-compute-engine google-compute-engine-oslogin
- echo "Rebooting..."
- systemctl reboot # Required for upgrade + package installs to be active
-elif [[ "$1" == "post" ]]
-then
- echo "Enabling necessary services"
- systemctl enable rngd # Must reboot before enabling
- rh_finalize
- echo "SUCCESS!"
-else
- echo "Expected to be called with 'pre' or 'post'"
- exit 6
-fi
diff --git a/contrib/cirrus/packer/fah_setup.sh b/contrib/cirrus/packer/fah_setup.sh
deleted file mode 100644
index 18c4db0af..000000000
--- a/contrib/cirrus/packer/fah_setup.sh
+++ /dev/null
@@ -1,21 +0,0 @@
-#!/bin/bash
-
-# This script is called by packer on the subject fah VM, to setup the podman
-# build/test environment. It's not intended to be used outside of this context.
-
-set -e
-
-# Load in library (copied by packer, before this script was run)
-source /tmp/libpod/$SCRIPT_BASE/lib.sh
-
-req_env_var SCRIPT_BASE
-
-install_ooe
-
-ooe.sh sudo atomic host upgrade
-
-ooe.sh sudo rpm-ostree uninstall cloud-init
-
-rh_finalize
-
-echo "SUCCESS!"
diff --git a/contrib/cirrus/packer/fedora_base-setup.sh b/contrib/cirrus/packer/fedora_base-setup.sh
index c0a1e422c..2e6d3eceb 100644
--- a/contrib/cirrus/packer/fedora_base-setup.sh
+++ b/contrib/cirrus/packer/fedora_base-setup.sh
@@ -15,9 +15,8 @@ install_ooe
echo "Updating packages"
ooe.sh dnf -y update
-echo "Installing necessary packages and google services"
-ooe.sh dnf -y copr enable ngompa/gce-oslogin
-ooe.sh dnf -y install rng-tools google-compute-engine google-compute-engine-oslogin
+echo "Installing necessary packages and google services"
+ooe.sh dnf -y install rng-tools google-compute-engine-tools google-compute-engine-oslogin
echo "Enabling services"
ooe.sh systemctl enable rngd
diff --git a/contrib/cirrus/packer/fedora_setup.sh b/contrib/cirrus/packer/fedora_setup.sh
index 36a65eb71..e031129d7 100644
--- a/contrib/cirrus/packer/fedora_setup.sh
+++ b/contrib/cirrus/packer/fedora_setup.sh
@@ -8,7 +8,7 @@ set -e
# Load in library (copied by packer, before this script was run)
source /tmp/libpod/$SCRIPT_BASE/lib.sh
-req_env_var SCRIPT_BASE FEDORA_CNI_COMMIT CNI_COMMIT CRIO_COMMIT CRIU_COMMIT RUNC_COMMIT
+req_env_var SCRIPT_BASE FEDORA_CNI_COMMIT CNI_COMMIT CONMON_COMMIT CRIU_COMMIT
install_ooe
@@ -22,6 +22,7 @@ ooe.sh sudo dnf install -y \
bats \
btrfs-progs-devel \
bzip2 \
+ criu \
device-mapper-devel \
emacs-nox \
findutils \
@@ -35,6 +36,7 @@ ooe.sh sudo dnf install -y \
gpgme-devel \
iptables \
iproute \
+ jq \
libassuan-devel \
libcap-devel \
libnet \
@@ -68,16 +70,14 @@ ooe.sh sudo dnf install -y \
install_varlink
+install_conmon
+
CNI_COMMIT=$FEDORA_CNI_COMMIT
install_cni_plugins
install_buildah
-install_conmon
-
-install_criu
-
-install_packer_copied_files
+sudo /tmp/libpod/hack/install_catatonit.sh
rh_finalize # N/B: Halts system!
diff --git a/contrib/cirrus/packer/libpod_base_images.yml b/contrib/cirrus/packer/libpod_base_images.yml
index bf568b40e..560cb321c 100644
--- a/contrib/cirrus/packer/libpod_base_images.yml
+++ b/contrib/cirrus/packer/libpod_base_images.yml
@@ -11,24 +11,16 @@ variables:
TIMESTAMP:
# Required for output from qemu builders
TTYDEV:
- # RHEL images require click-through agreements to obtain (required)
- RHEL_BASE_IMAGE_NAME:
- RHEL_IMAGE_FILE:
- RHEL_CSUM_FILE:
- # RHEL requires a subscription to install/update packages
- RHSM_COMMAND:
# Latest Fedora release
- FEDORA_IMAGE_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/29/Cloud/x86_64/images/Fedora-Cloud-Base-29-1.2.x86_64.qcow2"
- FEDORA_CSUM_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/29/Cloud/x86_64/images/Fedora-Cloud-29-1.2-x86_64-CHECKSUM"
- FEDORA_BASE_IMAGE_NAME: 'fedora-cloud-base-29-1-2' # Name to use in GCE
+ FEDORA_IMAGE_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/30/Cloud/x86_64/images/Fedora-Cloud-Base-30-1.2.x86_64.qcow2"
+ FEDORA_CSUM_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/30/Cloud/x86_64/images/Fedora-Cloud-30-1.2-x86_64-CHECKSUM"
+ FEDORA_BASE_IMAGE_NAME: 'fedora-cloud-base-30-1-2'
+
# Prior Fedora release
- PRIOR_FEDORA_IMAGE_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/28/Cloud/x86_64/images/Fedora-Cloud-Base-28-1.1.x86_64.qcow2"
- PRIOR_FEDORA_CSUM_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/28/Cloud/x86_64/images/Fedora-Cloud-28-1.1-x86_64-CHECKSUM"
- PRIOR_FEDORA_BASE_IMAGE_NAME: 'fedora-cloud-base-28-1-1' # Name to use in GCE
- FAH_IMAGE_URL: "https://dl.fedoraproject.org/pub/alt/atomic/stable/Fedora-Atomic-29-20181025.1/AtomicHost/x86_64/images/Fedora-AtomicHost-29-20181025.1.x86_64.qcow2"
- FAH_CSUM_URL: "https://dl.fedoraproject.org/pub/alt/atomic/stable/Fedora-Atomic-29-20181025.1/AtomicHost/x86_64/images/Fedora-AtomicHost-29-20181025.1-x86_64-CHECKSUM"
- FAH_BASE_IMAGE_NAME: 'fedora-atomichost-29-20181025-1' # Name to use in GCE
+ PRIOR_FEDORA_IMAGE_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/29/Cloud/x86_64/images/Fedora-Cloud-Base-29-1.2.x86_64.qcow2"
+ PRIOR_FEDORA_CSUM_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/29/Cloud/x86_64/images/Fedora-Cloud-29-1.2-x86_64-CHECKSUM"
+ PRIOR_FEDORA_BASE_IMAGE_NAME: 'fedora-cloud-base-29-1-2' # Name to use in GCE
# The name of the image in GCE used for packer build libpod_images.yml
IBI_BASE_NAME: 'image-builder-image'
@@ -45,28 +37,9 @@ variables:
sensitive-variables:
- 'GOOGLE_APPLICATION_CREDENTIALS'
- 'GCP_PROJECT_ID'
- - 'RHSM_COMMAND'
# What images to produce in which cloud
builders:
- - name: '{{user `IBI_BASE_NAME`}}'
- type: 'googlecompute'
- image_name: '{{user `IBI_BASE_NAME`}}-{{user `TIMESTAMP`}}'
- image_family: '{{user `IBI_BASE_NAME`}}'
- source_image_project_id: 'centos-cloud'
- source_image_family: 'centos-7'
- project_id: '{{user `GCP_PROJECT_ID`}}'
- account_file: '{{user `GOOGLE_APPLICATION_CREDENTIALS`}}'
- communicator: 'ssh'
- ssh_username: 'centos'
- ssh_pty: 'true'
- # The only supported zone in Cirrus-CI, as of addition of this comment
- zone: 'us-central1-a'
- # Enable nested virtualization in case it's ever needed
- image_licenses:
- - 'https://www.googleapis.com/compute/v1/projects/vm-options/global/licenses/enable-vmx'
- min_cpu_platform: "Intel Broadwell" # nested-virt requirement
-
- &nested_virt
name: 'fedora'
type: 'qemu'
@@ -109,18 +82,6 @@ builders:
iso_url: '{{user `PRIOR_FEDORA_IMAGE_URL`}}'
iso_checksum_url: '{{user `PRIOR_FEDORA_CSUM_URL`}}'
- - <<: *nested_virt
- name: 'fah'
- iso_url: '{{user `FAH_IMAGE_URL`}}'
- iso_checksum_url: '{{user `FAH_CSUM_URL`}}'
- disk_size: 10240
-
- - <<: *nested_virt
- name: 'rhel'
- iso_url: 'file://{{user `RHEL_IMAGE_FILE`}}'
- iso_checksum_url: 'file://{{user `RHEL_CSUM_FILE`}}'
- disk_size: 10240
-
provisioners:
- type: 'shell'
inline:
@@ -150,7 +111,6 @@ provisioners:
- 'GOSRC=/tmp/libpod'
- 'SCRIPT_BASE={{user `SCRIPT_BASE`}}'
- 'PACKER_BASE={{user `PACKER_BASE`}}'
- - 'RHSM_COMMAND={{user `RHSM_COMMAND`}}'
- <<: *shell_script
inline: ['{{user `GOSRC`}}/{{user `PACKER_BASE`}}/{{build_name}}_base-setup.sh']
@@ -161,7 +121,7 @@ provisioners:
post-processors:
- - type: "compress"
- only: ['fedora', 'prior-fedora', 'fah', 'rhel']
+ only: ['fedora', 'prior-fedora']
output: '/tmp/{{build_name}}/disk.raw.tar.gz'
format: '.tar.gz'
compression_level: 9
@@ -180,14 +140,4 @@ post-processors:
image_name: "{{user `PRIOR_FEDORA_BASE_IMAGE_NAME`}}-{{user `TIMESTAMP`}}"
image_description: 'Based on {{user `PRIOR_FEDORA_IMAGE_URL`}}'
image_family: '{{user `PRIOR_FEDORA_BASE_IMAGE_NAME`}}'
- - <<: *gcp_import
- only: ['fah']
- image_name: "{{user `FAH_BASE_IMAGE_NAME`}}-{{user `TIMESTAMP`}}"
- image_description: 'Based on {{user `FAH_IMAGE_URL`}}'
- image_family: '{{user `FAH_BASE_IMAGE_NAME`}}'
- - <<: *gcp_import
- only: ['rhel']
- image_name: "{{user `RHEL_BASE_IMAGE_NAME`}}-{{user `TIMESTAMP`}}"
- image_description: 'Based on {{user `RHEL_IMAGE_FILE`}}'
- image_family: '{{user `RHEL_BASE_IMAGE_NAME`}}'
- type: 'manifest'
diff --git a/contrib/cirrus/packer/libpod_images.yml b/contrib/cirrus/packer/libpod_images.yml
index 30ad0723a..34d4db7fb 100644
--- a/contrib/cirrus/packer/libpod_images.yml
+++ b/contrib/cirrus/packer/libpod_images.yml
@@ -3,17 +3,14 @@
# All of these are required
variables:
# Names of GCE Base images to start from, in .cirrus.yml
- RHEL_BASE_IMAGE: '{{env `RHEL_BASE_IMAGE`}}'
- CENTOS_BASE_IMAGE: '{{env `CENTOS_BASE_IMAGE`}}'
UBUNTU_BASE_IMAGE: '{{env `UBUNTU_BASE_IMAGE`}}'
FEDORA_BASE_IMAGE: '{{env `FEDORA_BASE_IMAGE`}}'
PRIOR_FEDORA_BASE_IMAGE: '{{env `PRIOR_FEDORA_BASE_IMAGE`}}'
- FAH_BASE_IMAGE: '{{env `FAH_BASE_IMAGE`}}'
# libpod dependencies to build and install into images
FEDORA_CNI_COMMIT: "{{env `FEDORA_CNI_COMMIT`}}"
CNI_COMMIT: "{{env `CNI_COMMIT`}}"
- CRIO_COMMIT: "{{env `CRIO_COMMIT`}}"
+ CONMON_COMMIT: "{{env `CONMON_COMMIT`}}"
CRIU_COMMIT: "{{env `CRIU_COMMIT`}}"
RUNC_COMMIT: "{{env `RUNC_COMMIT`}}"
@@ -25,7 +22,6 @@ variables:
# Protected credentials, decrypted by Cirrus at runtime
GCE_SSH_USERNAME: '{{env `GCE_SSH_USERNAME`}}'
GCP_PROJECT_ID: '{{env `GCP_PROJECT_ID`}}'
- RHSM_COMMAND: '{{env `RHSM_COMMAND`}}'
SERVICE_ACCOUNT: '{{env `SERVICE_ACCOUNT`}}'
GOOGLE_APPLICATION_CREDENTIALS: '{{env `GOOGLE_APPLICATION_CREDENTIALS`}}'
@@ -37,7 +33,6 @@ variables:
sensitive-variables:
- 'GCE_SSH_USERNAME'
- 'GCP_PROJECT_ID'
- - 'RHSM_COMMAND'
- 'SERVICE_ACCOUNT'
# What images to produce in which cloud
@@ -60,14 +55,6 @@ builders:
# v----- is a YAML alias, allows partial re-use of the anchor object
- <<: *gce_hosted_image
- name: 'rhel-7'
- source_image: '{{user `RHEL_BASE_IMAGE`}}'
-
- - <<: *gce_hosted_image
- name: 'centos-7'
- source_image: '{{user `CENTOS_BASE_IMAGE`}}'
-
- - <<: *gce_hosted_image
name: 'fedora-29'
source_image: '{{user `FEDORA_BASE_IMAGE`}}'
@@ -75,10 +62,6 @@ builders:
name: 'fedora-28'
source_image: '{{user `PRIOR_FEDORA_BASE_IMAGE`}}'
- - <<: *gce_hosted_image
- name: 'fah-29'
- source_image: '{{user `FAH_BASE_IMAGE`}}'
-
# The brains of the operation, making actual modifications to the base-image.
provisioners:
- type: 'file'
@@ -91,15 +74,10 @@ provisioners:
- 'GOSRC=/tmp/libpod'
- 'CNI_COMMIT={{user `CNI_COMMIT`}}'
- 'FEDORA_CNI_COMMIT={{user `FEDORA_CNI_COMMIT`}}'
- - 'CRIO_COMMIT={{user `CRIO_COMMIT`}}'
+ - 'CONMON_COMMIT={{user `CONMON_COMMIT`}}'
- 'CRIU_COMMIT={{user `CRIU_COMMIT`}}'
- 'RUNC_COMMIT={{user `RUNC_COMMIT`}}'
- 'SCRIPT_BASE={{user `SCRIPT_BASE`}}'
- - 'RHSM_COMMAND={{user `RHSM_COMMAND`}}'
post-processors:
- # Store VM disk in GCP storage, where it will expire based on a defined
- # lifecycle. This prevents GCE from filling with disused images.
- - - type: 'googlecompute-export'
- paths: ['gs://packer-import{{user `POST_MERGE_BUCKET_SUFFIX`}}/{{build_name}}{{user `BUILT_IMAGE_SUFFIX`}}.tar.gz']
- - type: 'manifest' # writes packer-manifest.json
+ - - type: 'manifest' # writes packer-manifest.json
diff --git a/contrib/cirrus/packer/rhel_base-setup.sh b/contrib/cirrus/packer/rhel_base-setup.sh
deleted file mode 100644
index 8d5892d7d..000000000
--- a/contrib/cirrus/packer/rhel_base-setup.sh
+++ /dev/null
@@ -1,78 +0,0 @@
-#!/bin/bash
-
-# N/B: This script is not intended to be run by humans. It is used to configure the
-# rhel base image for importing, so that it will boot in GCE
-
-set -e
-
-[[ "$1" == "post" ]] || exit 0 # pre stage is not needed
-
-# Load in library (copied by packer, before this script was run)
-source $GOSRC/$SCRIPT_BASE/lib.sh
-
-req_env_var RHSM_COMMAND
-
-install_ooe
-
-rhsm_enable
-
-echo "Setting up repos"
-# Frequently needed
-ooe.sh sudo yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
-
-# Required for google to manage ssh keys
-ooe.sh sudo tee /etc/yum.repos.d/google-cloud-sdk.repo << EOM
-[google-cloud-compute]
-name=google-cloud-compute
-baseurl=https://packages.cloud.google.com/yum/repos/google-cloud-compute-el7-x86_64
-enabled=1
-gpgcheck=1
-repo_gpgcheck=1
-gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg
- https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
-EOM
-
-echo "Updating all packages"
-ooe.sh sudo yum -y update
-
-echo "Installing/removing packages"
-ooe.sh sudo yum -y install rng-tools google-compute-engine google-compute-engine-oslogin
-
-echo "Enabling critical services"
-ooe.sh sudo systemctl enable \
- rngd \
- google-accounts-daemon \
- google-clock-skew-daemon \
- google-instance-setup \
- google-network-daemon \
- google-shutdown-scripts \
- google-startup-scripts
-
-rhel_exit_handler # release subscription!
-
-echo "Configuring boot"
-cat << "EOF" | sudo tee /etc/default/grub
-GRUB_TIMEOUT=0
-GRUB_DISTRIBUTOR="$(sed 's, release .*$,,g' /etc/system-release)"
-GRUB_DEFAULT=saved
-GRUB_DISABLE_SUBMENU=true
-GRUB_TERMINAL="serial console"
-GRUB_SERIAL_COMMAND="serial --speed=38400"
-GRUB_CMDLINE_LINUX="crashkernel=auto console=ttyS0,38400n8"
-GRUB_DISABLE_RECOVERY="true"
-EOF
-sudo grub2-mkconfig -o /boot/grub2/grub.cfg
-
-echo "Configuring networking"
-ooe.sh sudo nmcli connection modify 'System eth0' 802-3-ethernet.mtu 1460
-ooe.sh sudo nmcli connection modify 'System eth0' connection.autoconnect yes
-ooe.sh sudo nmcli connection modify 'System eth0' connection.autoconnect-priority
-ooe.sh sudo nmcli connection modify 'System eth0' ipv4.method auto
-ooe.sh sudo nmcli connection modify 'System eth0' ipv4.dhcp-send-hostname yes
-ooe.sh sudo nmcli connection modify 'System eth0' ipv4.dhcp-timeout 0
-ooe.sh sudo nmcli connection modify 'System eth0' ipv4.never-default no
-ooe.sh /usr/bin/google_instance_setup
-
-rh_finalize
-
-echo "SUCCESS!"
diff --git a/contrib/cirrus/packer/rhel_setup.sh b/contrib/cirrus/packer/rhel_setup.sh
deleted file mode 100644
index 45f5c3e9b..000000000
--- a/contrib/cirrus/packer/rhel_setup.sh
+++ /dev/null
@@ -1,91 +0,0 @@
-#!/bin/bash
-
-# This script is called by packer on the subject CentOS VM, to setup the podman
-# build/test environment. It's not intended to be used outside of this context.
-
-set -e
-
-# Load in library (copied by packer, before this script was run)
-source /tmp/libpod/$SCRIPT_BASE/lib.sh
-
-req_env_var SCRIPT_BASE CNI_COMMIT CRIO_COMMIT CRIU_COMMIT RHSM_COMMAND
-
-install_ooe
-
-rhsm_enable
-
-ooe.sh sudo yum -y erase "rh-amazon-rhui-client*"
-ooe.sh sudo subscription-manager repos "--disable=*"
-ooe.sh sudo subscription-manager repos \
- --enable=rhel-7-server-rpms \
- --enable=rhel-7-server-optional-rpms \
- --enable=rhel-7-server-extras-rpms \
- --enable=rhel-server-rhscl-7-rpms
-
-ooe.sh sudo yum -y update
-
-ooe.sh sudo yum -y install \
- PyYAML \
- atomic-registries \
- bats \
- btrfs-progs-devel \
- bzip2 \
- device-mapper-devel \
- emacs-nox \
- findutils \
- glib2-devel \
- glibc-static \
- gnupg \
- golang \
- golang-github-cpuguy83-go-md2man \
- golang-github-cpuguy83-go-md2man \
- gpgme-devel \
- iptables \
- libassuan-devel \
- libcap-devel \
- libnet \
- libnet-devel \
- libnl3-devel \
- libseccomp-devel \
- libselinux-devel \
- lsof \
- make \
- nmap-ncat \
- ostree-devel \
- protobuf \
- protobuf-c \
- protobuf-c-devel \
- protobuf-compiler \
- protobuf-devel \
- protobuf-python \
- python \
- python2-future \
- python2-pyyaml \
- python34-dateutil \
- python34-psutil \
- python34-pytoml \
- python34-PyYAML \
- runc \
- skopeo-containers \
- unzip \
- vim \
- which \
- xz
-
-install_scl_git
-
-install_cni_plugins
-
-install_buildah
-
-install_conmon
-
-install_criu
-
-install_packer_copied_files
-
-rhel_exit_handler # release subscription!
-
-rh_finalize
-
-echo "SUCCESS!"
diff --git a/contrib/cirrus/packer/ubuntu_setup.sh b/contrib/cirrus/packer/ubuntu_setup.sh
index d3ac8bddb..56d7f962e 100644
--- a/contrib/cirrus/packer/ubuntu_setup.sh
+++ b/contrib/cirrus/packer/ubuntu_setup.sh
@@ -8,7 +8,7 @@ set -e
# Load in library (copied by packer, before this script was run)
source /tmp/libpod/$SCRIPT_BASE/lib.sh
-req_env_var SCRIPT_BASE CNI_COMMIT CRIO_COMMIT CRIU_COMMIT RUNC_COMMIT
+req_env_var SCRIPT_BASE CNI_COMMIT CONMON_COMMIT CRIU_COMMIT
install_ooe
@@ -26,6 +26,8 @@ ooe.sh sudo -E apt-get -qq install software-properties-common
# Required to have Go 1.11 on Ubuntu 18.0.4
ooe.sh sudo -E add-apt-repository --yes ppa:longsleep/golang-backports
+ooe.sh sudo -E add-apt-repository --yes ppa:projectatomic/ppa
+ooe.sh sudo -E add-apt-repository --yes ppa:criu/ppa
ooe.sh sudo -E apt-get -qq update || sudo -E apt-get -qq update
ooe.sh sudo -E apt-get -qq install \
@@ -36,6 +38,8 @@ ooe.sh sudo -E apt-get -qq install \
bison \
btrfs-tools \
build-essential \
+ cri-o-runc \
+ criu \
curl \
e2fslibs-dev \
emacs-nox \
@@ -45,6 +49,7 @@ ooe.sh sudo -E apt-get -qq install \
golang \
iproute2 \
iptables \
+ jq \
libaio-dev \
libapparmor-dev \
libcap-dev \
@@ -89,20 +94,17 @@ ooe.sh sudo sed -re "$SEDCMD" -i /etc/default/grub.d/*
ooe.sh sudo sed -re "$SEDCMD" -i /etc/default/grub
ooe.sh sudo update-grub
-install_runc
-
install_conmon
-install_criu
-
install_cni_plugins
install_buildah
-install_packer_copied_files
+sudo /tmp/libpod/hack/install_catatonit.sh
install_varlink
+sudo mkdir -p /etc/containers
sudo curl https://raw.githubusercontent.com/projectatomic/registries/master/registries.fedora\
-o /etc/containers/registries.conf
diff --git a/contrib/cirrus/rootless_test.sh b/contrib/cirrus/rootless_test.sh
index eab06bac0..3b668034b 100755
--- a/contrib/cirrus/rootless_test.sh
+++ b/contrib/cirrus/rootless_test.sh
@@ -1,22 +1,26 @@
#!/bin/bash
set -e
-source $HOME/.bash_profile
-cd $GOSRC
source $(dirname $0)/lib.sh
-req_env_var GOSRC OS_RELEASE_ID OS_RELEASE_VER
-
if [[ "$UID" == "0" ]]
then
echo "Error: Expected to be running as a regular user"
exit 1
fi
+# Ensure environment setup correctly
+req_env_var GOSRC ROOTLESS_USER
+
echo "."
echo "Hello, my name is $USER and I live in $PWD can I be your friend?"
+echo "."
+
+export PODMAN_VARLINK_ADDRESS=unix:/tmp/podman-$(id -u)
+show_env_vars
+set -x
cd "$GOSRC"
make
make varlink_generate
diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh
index 4dbd56ed9..f40405e8d 100755
--- a/contrib/cirrus/setup_environment.sh
+++ b/contrib/cirrus/setup_environment.sh
@@ -4,11 +4,18 @@ set -e
source $(dirname $0)/lib.sh
-req_env_var USER HOME ENVLIB SCRIPT_BASE CIRRUS_BUILD_ID
+req_env_var USER HOME GOSRC SCRIPT_BASE SETUP_MARKER_FILEPATH
-[[ "$SHELL" =~ "bash" ]] || chsh -s /bin/bash
-
-cd "$CIRRUS_WORKING_DIR" # for clarity of initial conditions
+# Ensure this script only executes successfully once and always logs ending timestamp
+[[ ! -e "$SETUP_MARKER_FILEPATH" ]] || exit 0
+exithandler() {
+ RET=$?
+ set +e
+ show_env_vars
+ echo "$(basename $0) exit status: $RET"
+ [[ "$RET" -eq "0" ]] && date +%s >> "SETUP_MARKER_FILEPATH"
+}
+trap exithandler EXIT
# Verify basic dependencies
for depbin in go rsync unzip sha256sum curl make python3 git
@@ -19,71 +26,70 @@ do
fi
done
-# Setup env. vars common to all tasks/scripts/platforms and
-# ensure they return for every following script execution.
-MARK="# Added by $0, manual changes will be lost."
-touch "$HOME/$ENVLIB"
-if ! grep -q "$MARK" "$HOME/$ENVLIB"
-then
- cp "$HOME/$ENVLIB" "$HOME/${ENVLIB}_original"
- # N/B: Single-quote items evaluated every time, double-quotes only once (right now).
- for envstr in \
- "$MARK" \
- "export EPOCH_TEST_COMMIT=\"$CIRRUS_BASE_SHA\"" \
- "export HEAD=\"$CIRRUS_CHANGE_IN_REPO\"" \
- "export TRAVIS=\"1\"" \
- "export GOSRC=\"$CIRRUS_WORKING_DIR\"" \
- "export OS_RELEASE_ID=\"$(os_release_id)\"" \
- "export OS_RELEASE_VER=\"$(os_release_ver)\"" \
- "export OS_REL_VER=\"$(os_release_id)-$(os_release_ver)\"" \
- "export TEST_REMOTE_CLIENT=\"$TEST_REMOTE_CLIENT\"" \
- "export BUILT_IMAGE_SUFFIX=\"-$CIRRUS_REPO_NAME-${CIRRUS_CHANGE_IN_REPO:0:8}\"" \
- "export GOPATH=\"/var/tmp/go\"" \
- 'export PATH="$HOME/bin:$GOPATH/bin:/usr/local/bin:$PATH"' \
- 'export LD_LIBRARY_PATH="/usr/local/lib${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}"'
- do
- # Make permanent in later shells, and set in current shell
- X=$(echo "$envstr" | tee -a "$HOME/$ENVLIB") && eval "$X" && echo "$X"
- done
+# Sometimes environment setup needs to vary between distros
+# Note: This should only be used for environment variables, and temporary workarounds.
+# Anything externally dependent, should be made fixed-in-time by adding to
+# contrib/cirrus/packer/*_setup.sh to be incorporated into VM cache-images
+# (see docs).
+case "${OS_REL_VER}" in
+ ubuntu-18) ;;
+ fedora-29)
+ # Occasionally, and seemingly only on F29 the root disk fails to expand
+ # upon boot. When this happens, any number of failures could occur if
+ # space runs out. Until there is time to investigate the actual cause,
+ # workaround this problem by detecting it and acting accordingly.
+ REMAINING=$(df /dev/sda1 | tail -1 | awk '{print $4}')
+ if [[ "$REMAINING" -lt "100000000" ]] # .cirrus.yml specifies 200gig
+ then
+ echo "Fixing failure to expand root filesystem"
+ growpart /dev/sda 1 # device guaranteed by cloud provider
+ resize2fs /dev/sda1 # growpart & resuze guaranteed by base-image
+ fi
+ ;;
+ fedora-28) ;;
+ centos-7) # Current VM is an image-builder-image no local podman/testing
+ echo "No further setup required for VM image building"
+ exit 0
+ ;;
+ *) bad_os_id_ver ;;
+esac
- # Some setup needs to vary between distros
- case "${OS_RELEASE_ID}-${OS_RELEASE_VER}" in
- ubuntu-18)
- # Always install runc on Ubuntu
- install_runc_from_git
- ;;
- fedora-29)
- CON_SEL="https://kojipkgs.fedoraproject.org/packages/container-selinux/2.100/1.git3b78187.fc29/noarch/container-selinux-2.100-1.git3b78187.fc29.noarch.rpm"
- echo ">>>>> OVERRIDING container-selinux WITH $CON_SEL <<<<<"
- dnf -y install $CON_SEL
- echo ">>>>> OVERRIDING criu and selinux-policy with latest package <<<<<"
- dnf -y upgrade criu selinux-policy
- ;& # Continue to the next item
- fedora-28)
- echo ">>>>> OVERRIDING source-built runc with latest package <<<<<"
- dnf update -y runc
- ;& # Continue to the next item
- centos-7) ;&
- rhel-7)
- ;;
- *) bad_os_id_ver ;;
- esac
+cd "${GOSRC}/"
+# Reload to incorporate any changes from above
+source "$SCRIPT_BASE/lib.sh"
- cd "${GOSRC}/"
- # Reload to incorporate any changes from above
- source "$SCRIPT_BASE/lib.sh"
+echo "Installing cni config, policy and registry config"
+req_env_var GOSRC
+sudo install -D -m 755 $GOSRC/cni/87-podman-bridge.conflist \
+ /etc/cni/net.d/87-podman-bridge.conflist
+sudo install -D -m 755 $GOSRC/test/policy.json \
+ /etc/containers/policy.json
+sudo install -D -m 755 $GOSRC/test/registries.conf \
+ /etc/containers/registries.conf
+# cri-o if installed will mess with testing in non-obvious ways
+rm -f /etc/cni/net.d/*cri*
- case "$SPECIALMODE" in
- rootless)
- X=$(echo "export ROOTLESS_USER='some${RANDOM}dude'" | \
- tee -a "$HOME/$ENVLIB") && eval "$X" && echo "$X"
- setup_rootless
- ;;
- in_podman) # Assumed to be Fedora
- dnf install -y podman buildah
- $SCRIPT_BASE/setup_container_environment.sh
- ;;
- esac
-fi
+make install.tools
-show_env_vars
+case "$SPECIALMODE" in
+ none) ;; # Do the normal thing
+ rootless)
+ # Only do this once, even if ROOTLESS_USER (somehow) changes
+ if ! grep -q 'ROOTLESS_USER' /etc/environment
+ then
+ X=$(echo "export ROOTLESS_USER='${ROOTLESS_USER:-some${RANDOM}dude}'" | \
+ tee -a /etc/environment) && eval "$X" && echo "$X"
+ X=$(echo "export SPECIALMODE='${SPECIALMODE}'" | \
+ tee -a /etc/environment) && eval "$X" && echo "$X"
+ X=$(echo "export TEST_REMOTE_CLIENT='${TEST_REMOTE_CLIENT}'" | \
+ tee -a /etc/environment) && eval "$X" && echo "$X"
+ setup_rootless
+ fi
+ ;;
+ in_podman) # Assumed to be Fedora
+ dnf install -y podman buildah
+ $SCRIPT_BASE/setup_container_environment.sh
+ ;;
+ *)
+ die 111 "Unsupported \$SPECIAL_MODE: $SPECIALMODE"
+esac
diff --git a/contrib/cirrus/system_test.sh b/contrib/cirrus/system_test.sh
index dd5ef511d..a2cc1af05 100755
--- a/contrib/cirrus/system_test.sh
+++ b/contrib/cirrus/system_test.sh
@@ -5,17 +5,12 @@ source $(dirname $0)/lib.sh
req_env_var GOSRC OS_RELEASE_ID OS_RELEASE_VER
-clean_env
-
set -x
cd "$GOSRC"
-case "${OS_RELEASE_ID}-${OS_RELEASE_VER}" in
- ubuntu-18) ;& # Continue to the next item
- fedora-28) ;&
- fedora-29) ;&
- centos-7) ;&
- rhel-7)
+case "${OS_RELEASE_ID}" in
+ ubuntu) ;& # Continue to the next item
+ fedora)
make install.tools
make
make test-binaries
diff --git a/contrib/cirrus/test/test_dot_cirrus_yaml.py b/contrib/cirrus/test/test_dot_cirrus_yaml.py
deleted file mode 100755
index 2894bc45e..000000000
--- a/contrib/cirrus/test/test_dot_cirrus_yaml.py
+++ /dev/null
@@ -1,78 +0,0 @@
-#!/bin/env python3
-
-import sys
-import os
-import os.path
-import unittest
-import warnings
-import yaml
-
-class TestCaseBase(unittest.TestCase):
-
- SCRIPT_PATH = os.path.realpath((os.path.dirname(sys.argv[0])))
- CIRRUS_WORKING_DIR = os.environ.get('CIRRUS_WORKING_DIR',
- '{0}/../../../'.format(SCRIPT_PATH))
-
- def setUp(self):
- os.chdir(self.CIRRUS_WORKING_DIR)
-
-
-class TestCirrusYAML(TestCaseBase):
-
- IMAGE_NAME_SUFFIX = '_CACHE_IMAGE_NAME'
- ACTIVE_IMAGES_NAME = 'ACTIVE_CACHE_IMAGE_NAMES'
-
- def setUp(self):
- TestCirrusYAML._cirrus = None
- super().setUp()
-
- @property
- def cirrus(self):
- if TestCirrusYAML._cirrus is None:
- with warnings.catch_warnings():
- warnings.filterwarnings("ignore",category=DeprecationWarning)
- with open('.cirrus.yml', "r") as dot_cirrus_dot_yaml:
- TestCirrusYAML._cirrus = yaml.load(dot_cirrus_dot_yaml)
- return TestCirrusYAML._cirrus
-
- def _assert_get_cache_image_names(self, env):
- inames = set([key for key in env.keys()
- if key.endswith(self.IMAGE_NAME_SUFFIX)])
- self.assertNotEqual(inames, set())
-
- ivalues = set([value for key, value in env.items()
- if key in inames])
- self.assertNotEqual(ivalues, set())
- return ivalues
-
- def _assert_get_subdct(self, key, dct):
- self.assertIn(key, dct)
- return dct[key]
-
- def test_parse_yaml(self):
- self.assertIsInstance(self.cirrus, dict)
-
- def test_active_cache_image_names(self):
- env = self._assert_get_subdct('env', self.cirrus)
- acin = self._assert_get_subdct(self.ACTIVE_IMAGES_NAME, env)
-
- for ivalue in self._assert_get_cache_image_names(env):
- self.assertIn(ivalue, acin,
- "The '{}' sub-key of 'env' should contain this among"
- " its space-separated values."
- "".format(self.ACTIVE_IMAGES_NAME))
-
-
- def test_cache_image_names_active(self):
- env = self._assert_get_subdct('env', self.cirrus)
- ivalues = self._assert_get_cache_image_names(env)
-
- for avalue in set(self._assert_get_subdct(self.ACTIVE_IMAGES_NAME, env).split()):
- self.assertIn(avalue, ivalues,
- "All space-separated values in the '{}' sub-key"
- " of 'env' must also be used in a key with a '{}' suffix."
- "".format(self.ACTIVE_IMAGES_NAME, self.IMAGE_NAME_SUFFIX))
-
-
-if __name__ == '__main__':
- unittest.main(failfast=True, catchbreak=True, verbosity=0)
diff --git a/contrib/cirrus/unit_test.sh b/contrib/cirrus/unit_test.sh
index 0e8c9e2e2..202663fb7 100755
--- a/contrib/cirrus/unit_test.sh
+++ b/contrib/cirrus/unit_test.sh
@@ -3,9 +3,7 @@
set -e
source $(dirname $0)/lib.sh
-req_env_var GOSRC OS_RELEASE_ID OS_RELEASE_VER
-
-clean_env
+req_env_var GOSRC
set -x
cd "$GOSRC"
diff --git a/hack/get_ci_vm.sh b/hack/get_ci_vm.sh
index d0325b8ae..370cd8a5e 100755
--- a/hack/get_ci_vm.sh
+++ b/hack/get_ci_vm.sh
@@ -19,7 +19,6 @@ PROJECT="libpod-218412"
GOSRC="/var/tmp/go/src/github.com/containers/libpod"
GCLOUD_IMAGE=${GCLOUD_IMAGE:-quay.io/cevich/gcloud_centos:latest}
GCLOUD_SUDO=${GCLOUD_SUDO-sudo}
-ROOTLESS_USER="madcowdog"
# Shared tmp directory between container and us
TMPDIR=$(mktemp -d --tmpdir $(basename $0)_tmpdir_XXXXXX)
@@ -48,11 +47,15 @@ showrun() {
}
cleanup() {
+ RET=$?
set +e
wait
# set GCLOUD_DEBUG to leave tmpdir behind for postmortem
test -z "$GCLOUD_DEBUG" && rm -rf $TMPDIR
+
+ # Not always called from an exit handler, but should always exit when called
+ exit $RET
}
trap cleanup EXIT
@@ -67,14 +70,15 @@ delvm() {
image_hints() {
egrep '[[:space:]]+[[:alnum:]].+_CACHE_IMAGE_NAME:[[:space:]+"[[:print:]]+"' \
"$LIBPODROOT/.cirrus.yml" | cut -d: -f 2 | tr -d '"[:blank:]' | \
- grep -v 'notready' | grep -v 'image-builder' | sort -u
+ grep -v 'notready' | sort -u
}
show_usage() {
echo -e "\n${RED}ERROR: $1${NOR}"
- echo -e "${YEL}Usage: $(basename $0) [-s | -p | -r] <image_name>${NOR}"
- echo "Use -s / -p to select source or package based dependencies"
- echo -e "Use -r to setup and run tests as a regular user.\n"
+ echo -e "${YEL}Usage: $(basename $0) [-m <SPECIALMODE>] [-u <ROOTLESS_USER> ] <image_name>${NOR}"
+ echo "Use -m <SPECIALMODE> with a supported value documented in contrib/cirrus/README.md."
+ echo "With '-m rootless' must also specify -u <ROOTLESS_USER> with name of user to create & use"
+ echo ""
if [[ -r ".cirrus.yml" ]]
then
echo -e "${YEL}Some possible image_name values (from .cirrus.yml):${NOR}"
@@ -87,7 +91,7 @@ show_usage() {
get_env_vars() {
python -c '
import yaml
-env=yaml.load(open(".cirrus.yml"))["env"]
+env=yaml.load(open(".cirrus.yml"), Loader=yaml.SafeLoader)["env"]
keys=[k for k in env if "ENCRYPTED" not in str(env[k])]
for k,v in env.items():
v=str(v)
@@ -99,28 +103,56 @@ for k,v in env.items():
parse_args(){
echo -e "$USAGE_WARNING"
- if [[ -z "$1" ]]
+ if [[ "$USER" =~ "root" ]]
then
+ show_usage "This script must be run as a regular user."
+ fi
+
+ ENVS="$(get_env_vars)"
+ [[ "$#" -ge "1" ]] || \
show_usage "Must specify at least one command-line parameter."
- elif [[ "$1" == "-p" ]]
- then
- echo -e "${YEL}Hint: Use -p for package-based dependencies or -s for source-based.${NOR}"
- DEPS="PACKAGE_DEPS=true SOURCE_DEPS=false"
- IMAGE_NAME="$2"
- elif [[ "$1" == "-s" ]]
+ IMAGE_NAME=""
+ ROOTLESS_USER=""
+ SPECIALMODE="none"
+ for arg
+ do
+ if [[ "$SPECIALMODE" == "GRABNEXT" ]] && [[ "${arg:0:1}" != "-" ]]
+ then
+ SPECIALMODE="$arg"
+ echo -e "${YEL}Using \$SPECIALMODE=$SPECIALMODE.${NOR}"
+ continue
+ elif [[ "$ROOTLESS_USER" == "GRABNEXT" ]] && [[ "${arg:0:1}" != "-" ]]
+ then
+ ROOTLESS_USER="$arg"
+ echo -e "${YEL}Using \$ROOTLESS_USER=$ROOTLESS_USER.${NOR}"
+ continue
+ fi
+ case "$arg" in
+ -m)
+ SPECIALMODE="GRABNEXT"
+ ;;
+ -u)
+ ROOTLESS_USER="GRABNEXT"
+ ;;
+ *)
+ [[ "${arg:0:1}" != "-" ]] || \
+ show_usage "Unknown command-line option '$arg'."
+ [[ -z "$IMAGE_NAME" ]] || \
+ show_usage "Must specify exactly one image name, got '$IMAGE_NAME' and '$arg'."
+ IMAGE_NAME="$arg"
+ ;;
+ esac
+ done
+
+ if [[ "$SPECIALMODE" == "GRABNEXT" ]]
then
- echo -e "${RED}Using source-based dependencies.${NOR}"
- DEPS="PACKAGE_DEPS=false SOURCE_DEPS=true"
- IMAGE_NAME="$2"
- elif [[ "$1" == "-r" ]]
+ show_usage "Must specify argument to -m option."
+ fi
+
+ if [[ "$ROOTLESS_USER" == "GRABNEXT" ]]
then
- DEPS="ROOTLESS_USER=$ROOTLESS_USER"
- IMAGE_NAME="$2"
- else # no -s or -p
- echo -e "${RED}Using package-based dependencies.${NOR}"
- DEPS="$(get_env_vars)"
- IMAGE_NAME="$1"
+ show_usage "Must specify argument to -u option."
fi
if [[ -z "$IMAGE_NAME" ]]
@@ -128,15 +160,33 @@ parse_args(){
show_usage "No image-name specified."
fi
- if [[ "$USER" =~ "root" ]]
+ if [[ "$SPECIALMODE" == "rootless" ]] && [[ -z "$ROOTLESS_USER" ]]
then
- show_usage "This script must be run as a regular user."
+ show_usage "With '-m rootless' must also pass -u <username> of rootless user."
+ fi
+
+ if echo "$IMAGE_NAME" | grep -q "image-builder-image"
+ then
+ echo -e "Creating an image-builder VM, I hope you know what you're doing.\n"
+ IBI_ARGS="--scopes=compute-rw,storage-rw,userinfo-email \"--min-cpu-platform=Intel Haswell\""
+ SSHUSER="centos"
+ else
+ unset IBI_ARGS
+ SSHUSER="root"
fi
- SETUP_CMD="env $DEPS $GOSRC/contrib/cirrus/setup_environment.sh"
+ ENVS="$ENVS SPECIALMODE=\"$SPECIALMODE\""
+
+ [[ -z "$ROOTLESS_USER" ]] || \
+ ENVS="$ENVS ROOTLESS_USER=$ROOTLESS_USER"
+
+ SETUP_CMD="env $ENVS $GOSRC/contrib/cirrus/setup_environment.sh"
VMNAME="${VMNAME:-${USER}-${IMAGE_NAME}}"
- CREATE_CMD="$PGCLOUD compute instances create --zone=$ZONE --image=${IMAGE_NAME} --custom-cpu=$CPUS --custom-memory=$MEMORY --boot-disk-size=$DISK --labels=in-use-by=$USER $VMNAME"
- SSH_CMD="$PGCLOUD compute ssh root@$VMNAME"
+
+ CREATE_CMD="$PGCLOUD compute instances create --zone=$ZONE --image=${IMAGE_NAME} --custom-cpu=$CPUS --custom-memory=$MEMORY --boot-disk-size=$DISK --labels=in-use-by=$USER $IBI_ARGS $VMNAME"
+
+ SSH_CMD="$PGCLOUD compute ssh $SSHUSER@$VMNAME"
+
CLEANUP_CMD="$PGCLOUD compute instances delete --zone $ZONE --delete-disks=all $VMNAME"
}
@@ -147,7 +197,7 @@ parse_args(){
cd "$LIBPODROOT"
-parse_args $@
+parse_args "$@"
# Ensure mount-points and data directories exist on host as $USER. Also prevents
# permission-denied errors during cleanup() b/c `sudo podman` created mount-points
@@ -188,7 +238,7 @@ showrun --background tar cjf $TMPDIR/$TARBALL --warning=no-file-changed -C $LIBP
trap delvm INT # Allow deleting VM if CTRL-C during create
# This fails if VM already exists: permit this usage to re-init
-echo -e "\n${YEL}Trying to creating a VM named $VMNAME ${RED}(might take a minute/two. Errors ignored).${NOR}"
+echo -e "\n${YEL}Trying to creating a VM named $VMNAME\n${RED}(might take a minute/two. Errors ignored).${NOR}"
showrun $CREATE_CMD || true # allow re-running commands below when "delete: N"
# Any subsequent failure should prompt for VM deletion
@@ -208,19 +258,13 @@ then
fi
echo -e "${YEL}Got it${NOR}"
-if $SSH_CMD --command "test -r /root/.bash_profile_original"
-then
- echo -e "\n${YEL}Resetting environment configuration${NOR}"
- showrun $SSH_CMD --command "cp /root/.bash_profile_original /root/.bash_profile"
-fi
-
echo -e "\n${YEL}Removing and re-creating $GOSRC on $VMNAME.${NOR}"
showrun $SSH_CMD --command "rm -rf $GOSRC"
showrun $SSH_CMD --command "mkdir -p $GOSRC"
echo -e "\n${YEL}Transfering tarball to $VMNAME.${NOR}"
wait
-showrun $SCP_CMD $HOME/$TARBALL root@$VMNAME:/tmp/$TARBALL
+showrun $SCP_CMD $HOME/$TARBALL $SSHUSER@$VMNAME:/tmp/$TARBALL
echo -e "\n${YEL}Unpacking tarball into $GOSRC on $VMNAME.${NOR}"
showrun $SSH_CMD --command "tar xjf /tmp/$TARBALL -C $GOSRC"
@@ -231,9 +275,12 @@ showrun $SSH_CMD --command "rm -f /tmp/$TARBALL"
echo -e "\n${YEL}Executing environment setup${NOR}"
showrun $SSH_CMD --command "$SETUP_CMD"
-echo -e "\n${YEL}Connecting to $VMNAME ${RED}(option to delete VM upon logout).${NOR}\n"
-if [[ "$1" == "-r" ]]
+echo -e "\n${YEL}Connecting to $VMNAME\n${RED}(option to delete VM upon logout).${NOR}\n"
+if [[ -n "$ROOTLESS_USER" ]]
then
+ echo "Re-chowning source files after transfer"
+ showrun $SSH_CMD --command "chown -R $ROOTLESS_USER $GOSRC"
+ echo "Connecting as user $ROOTLESS_USER"
SSH_CMD="$PGCLOUD compute ssh $ROOTLESS_USER@$VMNAME"
fi
-showrun $SSH_CMD -- -t "cd $GOSRC && exec env $DEPS bash -il"
+showrun $SSH_CMD -- -t "cd $GOSRC && exec env $ENVS bash -il"
diff --git a/test/registries.conf b/test/registries.conf
index 6c9d39bbc..bb7072d45 100644
--- a/test/registries.conf
+++ b/test/registries.conf
@@ -1,5 +1,5 @@
[registries.search]
-registries = ['docker.io', 'quay.io']
+registries = ['docker.io', 'quay.io', 'registry.fedoraproject.org']
[registries.insecure]
registries = []