From 95141f88d4e6f994c2972a63be19c82ef759ea5b Mon Sep 17 00:00:00 2001 From: Chris Evich Date: Tue, 30 Apr 2019 13:09:32 -0400 Subject: Cirrus: fixups based on review feedback Also remove disused distros (RHEL/CentOS/FAH) and fix get_ci_vm script Signed-off-by: Chris Evich --- contrib/cirrus/packer/libpod_base_images.yml | 53 +--------------------------- 1 file changed, 1 insertion(+), 52 deletions(-) (limited to 'contrib/cirrus/packer/libpod_base_images.yml') diff --git a/contrib/cirrus/packer/libpod_base_images.yml b/contrib/cirrus/packer/libpod_base_images.yml index bf568b40e..2caa28a04 100644 --- a/contrib/cirrus/packer/libpod_base_images.yml +++ b/contrib/cirrus/packer/libpod_base_images.yml @@ -11,12 +11,6 @@ variables: TIMESTAMP: # Required for output from qemu builders TTYDEV: - # RHEL images require click-through agreements to obtain (required) - RHEL_BASE_IMAGE_NAME: - RHEL_IMAGE_FILE: - RHEL_CSUM_FILE: - # RHEL requires a subscription to install/update packages - RHSM_COMMAND: # Latest Fedora release FEDORA_IMAGE_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/29/Cloud/x86_64/images/Fedora-Cloud-Base-29-1.2.x86_64.qcow2" @@ -26,9 +20,6 @@ variables: PRIOR_FEDORA_IMAGE_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/28/Cloud/x86_64/images/Fedora-Cloud-Base-28-1.1.x86_64.qcow2" PRIOR_FEDORA_CSUM_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/28/Cloud/x86_64/images/Fedora-Cloud-28-1.1-x86_64-CHECKSUM" PRIOR_FEDORA_BASE_IMAGE_NAME: 'fedora-cloud-base-28-1-1' # Name to use in GCE - FAH_IMAGE_URL: "https://dl.fedoraproject.org/pub/alt/atomic/stable/Fedora-Atomic-29-20181025.1/AtomicHost/x86_64/images/Fedora-AtomicHost-29-20181025.1.x86_64.qcow2" - FAH_CSUM_URL: "https://dl.fedoraproject.org/pub/alt/atomic/stable/Fedora-Atomic-29-20181025.1/AtomicHost/x86_64/images/Fedora-AtomicHost-29-20181025.1-x86_64-CHECKSUM" - FAH_BASE_IMAGE_NAME: 'fedora-atomichost-29-20181025-1' # Name to use in GCE # The name of the image in GCE used for packer build libpod_images.yml IBI_BASE_NAME: 'image-builder-image' @@ -45,28 +36,9 @@ variables: sensitive-variables: - 'GOOGLE_APPLICATION_CREDENTIALS' - 'GCP_PROJECT_ID' - - 'RHSM_COMMAND' # What images to produce in which cloud builders: - - name: '{{user `IBI_BASE_NAME`}}' - type: 'googlecompute' - image_name: '{{user `IBI_BASE_NAME`}}-{{user `TIMESTAMP`}}' - image_family: '{{user `IBI_BASE_NAME`}}' - source_image_project_id: 'centos-cloud' - source_image_family: 'centos-7' - project_id: '{{user `GCP_PROJECT_ID`}}' - account_file: '{{user `GOOGLE_APPLICATION_CREDENTIALS`}}' - communicator: 'ssh' - ssh_username: 'centos' - ssh_pty: 'true' - # The only supported zone in Cirrus-CI, as of addition of this comment - zone: 'us-central1-a' - # Enable nested virtualization in case it's ever needed - image_licenses: - - 'https://www.googleapis.com/compute/v1/projects/vm-options/global/licenses/enable-vmx' - min_cpu_platform: "Intel Broadwell" # nested-virt requirement - - &nested_virt name: 'fedora' type: 'qemu' @@ -109,18 +81,6 @@ builders: iso_url: '{{user `PRIOR_FEDORA_IMAGE_URL`}}' iso_checksum_url: '{{user `PRIOR_FEDORA_CSUM_URL`}}' - - <<: *nested_virt - name: 'fah' - iso_url: '{{user `FAH_IMAGE_URL`}}' - iso_checksum_url: '{{user `FAH_CSUM_URL`}}' - disk_size: 10240 - - - <<: *nested_virt - name: 'rhel' - iso_url: 'file://{{user `RHEL_IMAGE_FILE`}}' - iso_checksum_url: 'file://{{user `RHEL_CSUM_FILE`}}' - disk_size: 10240 - provisioners: - type: 'shell' inline: @@ -150,7 +110,6 @@ provisioners: - 'GOSRC=/tmp/libpod' - 'SCRIPT_BASE={{user `SCRIPT_BASE`}}' - 'PACKER_BASE={{user `PACKER_BASE`}}' - - 'RHSM_COMMAND={{user `RHSM_COMMAND`}}' - <<: *shell_script inline: ['{{user `GOSRC`}}/{{user `PACKER_BASE`}}/{{build_name}}_base-setup.sh'] @@ -161,7 +120,7 @@ provisioners: post-processors: - - type: "compress" - only: ['fedora', 'prior-fedora', 'fah', 'rhel'] + only: ['fedora', 'prior-fedora'] output: '/tmp/{{build_name}}/disk.raw.tar.gz' format: '.tar.gz' compression_level: 9 @@ -180,14 +139,4 @@ post-processors: image_name: "{{user `PRIOR_FEDORA_BASE_IMAGE_NAME`}}-{{user `TIMESTAMP`}}" image_description: 'Based on {{user `PRIOR_FEDORA_IMAGE_URL`}}' image_family: '{{user `PRIOR_FEDORA_BASE_IMAGE_NAME`}}' - - <<: *gcp_import - only: ['fah'] - image_name: "{{user `FAH_BASE_IMAGE_NAME`}}-{{user `TIMESTAMP`}}" - image_description: 'Based on {{user `FAH_IMAGE_URL`}}' - image_family: '{{user `FAH_BASE_IMAGE_NAME`}}' - - <<: *gcp_import - only: ['rhel'] - image_name: "{{user `RHEL_BASE_IMAGE_NAME`}}-{{user `TIMESTAMP`}}" - image_description: 'Based on {{user `RHEL_IMAGE_FILE`}}' - image_family: '{{user `RHEL_BASE_IMAGE_NAME`}}' - type: 'manifest' -- cgit v1.2.3-54-g00ecf From 1a2055ffac8b3673e0d1a4062ae64a87cb4f404f Mon Sep 17 00:00:00 2001 From: Chris Evich Date: Thu, 2 May 2019 14:38:59 -0400 Subject: Cirrus: Stub in F30 support New base-image boots, a cache-image builds, but more work is needed for it to be prime-time ready. This commit just adds some updates to the scafolding necessary to build the base-image. Future work will make F30 more of a reality. Also add log-collection scripts to test image verification task Signed-off-by: Chris Evich --- .cirrus.yml | 17 +++++++++++++++++ contrib/cirrus/lib.sh | 2 ++ contrib/cirrus/packer/Makefile | 6 ++++-- contrib/cirrus/packer/fedora_base-setup.sh | 5 ++--- contrib/cirrus/packer/libpod_base_images.yml | 13 +++++++------ 5 files changed, 32 insertions(+), 11 deletions(-) (limited to 'contrib/cirrus/packer/libpod_base_images.yml') diff --git a/.cirrus.yml b/.cirrus.yml index 263593018..32b76ba79 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -24,6 +24,8 @@ env: # Command to prefix every output line with a timestamp # (can't do inline awk script, Cirrus-CI or YAML mangles quoting) TIMESTAMP: "awk --file ${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/timestamp.awk" + # Command to log critical filesystems, types, and sizes. + DFCMD: "df -lhTx tmpfs" CIRRUS_CLONE_DEPTH: 50 #### @@ -276,6 +278,7 @@ testing_task: unit_test_script: '$SCRIPT_BASE/unit_test.sh |& ${TIMESTAMP}' integration_test_script: '$SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP}' ginkgo_node_logs_script: 'cat $CIRRUS_WORKING_DIR/test/e2e/ginkgo-node-*.log || echo "Ginkgo node logs not found"' + df_script: '${DFCMD}' audit_log_script: 'cat /var/log/audit/audit.log || cat /var/log/kern.log' journalctl_b_script: 'journalctl -b' @@ -283,6 +286,7 @@ testing_task: failed_master_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_master_failure.sh' # Job has already failed, don't fail again and miss collecting data failed_ginkgo_node_logs_script: 'cat $CIRRUS_WORKING_DIR/test/e2e/ginkgo-node-*.log || echo "Ginkgo node logs not found"' + failed_df_script: '${DFCMD}' failed_audit_log_script: 'cat /var/log/audit/audit.log || cat /var/log/kern.log || echo "Uh oh, cat audit.log failed"' failed_journalctl_b_script: 'journalctl -b || echo "Uh oh, journalctl -b failed"' @@ -307,12 +311,14 @@ special_testing_task: setup_environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}' integration_test_script: '$SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP}' + df_script: '${DFCMD}' audit_log_script: 'cat /var/log/audit/audit.log || cat /var/log/kern.log' journalctl_b_script: 'journalctl -b' on_failure: failed_master_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_master_failure.sh' # Job has already failed, don't fail again and miss collecting data + failed_df_script: '${DFCMD}' failed_audit_log_script: 'cat /var/log/audit/audit.log || cat /var/log/kern.log || echo "Uh oh, cat audit.log failed"' failed_journalctl_b_script: 'journalctl -b || echo "Uh oh, journalctl -b failed"' @@ -402,7 +408,18 @@ verify_test_built_images_task: # "probably" work. A full round of testing will happen again after $*_CACHE_IMAGE_NAME # are updated in this or another PR (w/o '***CIRRUS: TEST IMAGES***'). environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}' + integration_test_script: '$SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP}' + ginkgo_node_logs_script: 'cat $CIRRUS_WORKING_DIR/test/e2e/ginkgo-node-*.log || echo "Ginkgo node logs not found"' + df_script: '${DFCMD}' + audit_log_script: 'cat /var/log/audit/audit.log || cat /var/log/kern.log' + journalctl_b_script: 'journalctl -b' + on_failure: + # Job has already failed, don't fail again and miss collecting data + failed_ginkgo_node_logs_script: 'cat $CIRRUS_WORKING_DIR/test/e2e/ginkgo-node-*.log || echo "Ginkgo node logs not found"' + failed_df_script: '${DFCMD}' + failed_audit_log_script: 'cat /var/log/audit/audit.log || cat /var/log/kern.log || echo "Uh oh, cat audit.log failed"' + failed_journalctl_b_script: 'journalctl -b || echo "Uh oh, journalctl -b failed"' # Build new cache-images for future PR testing, but only after a PR merge. diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh index d0a2e1579..334202aa9 100644 --- a/contrib/cirrus/lib.sh +++ b/contrib/cirrus/lib.sh @@ -60,7 +60,9 @@ PACKER_BUILDS="${PACKER_BUILDS:-ubuntu-18,fedora-29,fedora-28}" UBUNTU_BASE_IMAGE="ubuntu-1804-bionic-v20181203a" # Manually produced base-image names (see $SCRIPT_BASE/README.md) FEDORA_BASE_IMAGE="fedora-cloud-base-29-1-2-1541789245" +# FEDORA_BASE_IMAGE: "fedora-cloud-base-30-1-2-1556821664" PRIOR_FEDORA_BASE_IMAGE="fedora-cloud-base-28-1-1-1544474897" +# PRIOR_FEDORA_BASE_IMAGE="fedora-cloud-base-29-1-2-1541789245" BUILT_IMAGE_SUFFIX="${BUILT_IMAGE_SUFFIX:--$CIRRUS_REPO_NAME-${CIRRUS_BUILD_ID}}" # Safe env. vars. to transfer from root -> $ROOTLESS_USER (go env handled separetly) diff --git a/contrib/cirrus/packer/Makefile b/contrib/cirrus/packer/Makefile index 0ae0deca6..d03d22abe 100644 --- a/contrib/cirrus/packer/Makefile +++ b/contrib/cirrus/packer/Makefile @@ -3,7 +3,7 @@ # builder name(s) from applicable YAML file, # e.g for names see libpod_images.yml -PACKER_VER ?= 1.3.2 +PACKER_VER ?= 1.3.5 GOARCH=$(shell go env GOARCH) ARCH=$(uname -m) PACKER_DIST_FILENAME := packer_${PACKER_VER}_linux_${GOARCH}.zip @@ -78,6 +78,9 @@ ifndef GCP_PROJECT_ID endif ifndef GOOGLE_APPLICATION_CREDENTIALS $(error GOOGLE_APPLICATION_CREDENTIALS is undefined, expected absolute path to JSON file, like $HOME/.config/gcloud/legacy_credentials/*/adc.json) +endif +ifndef PACKER_BUILDS + $(error PACKER_BUILDS is undefined, expected builder-names CSV) endif PACKER_CACHE_DIR=/tmp ./packer build \ -force \ @@ -88,6 +91,5 @@ endif -var GOSRC=$(GOSRC) \ -var PACKER_BASE=$(PACKER_BASE) \ -var SCRIPT_BASE=$(SCRIPT_BASE) \ - -var POST_MERGE_BUCKET_SUFFIX=$(POST_MERGE_BUCKET_SUFFIX) \ -only $(PACKER_BUILDS) \ libpod_base_images.json diff --git a/contrib/cirrus/packer/fedora_base-setup.sh b/contrib/cirrus/packer/fedora_base-setup.sh index c0a1e422c..2e6d3eceb 100644 --- a/contrib/cirrus/packer/fedora_base-setup.sh +++ b/contrib/cirrus/packer/fedora_base-setup.sh @@ -15,9 +15,8 @@ install_ooe echo "Updating packages" ooe.sh dnf -y update -echo "Installing necessary packages and google services" -ooe.sh dnf -y copr enable ngompa/gce-oslogin -ooe.sh dnf -y install rng-tools google-compute-engine google-compute-engine-oslogin +echo "Installing necessary packages and google services" +ooe.sh dnf -y install rng-tools google-compute-engine-tools google-compute-engine-oslogin echo "Enabling services" ooe.sh systemctl enable rngd diff --git a/contrib/cirrus/packer/libpod_base_images.yml b/contrib/cirrus/packer/libpod_base_images.yml index 2caa28a04..560cb321c 100644 --- a/contrib/cirrus/packer/libpod_base_images.yml +++ b/contrib/cirrus/packer/libpod_base_images.yml @@ -13,13 +13,14 @@ variables: TTYDEV: # Latest Fedora release - FEDORA_IMAGE_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/29/Cloud/x86_64/images/Fedora-Cloud-Base-29-1.2.x86_64.qcow2" - FEDORA_CSUM_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/29/Cloud/x86_64/images/Fedora-Cloud-29-1.2-x86_64-CHECKSUM" - FEDORA_BASE_IMAGE_NAME: 'fedora-cloud-base-29-1-2' # Name to use in GCE + FEDORA_IMAGE_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/30/Cloud/x86_64/images/Fedora-Cloud-Base-30-1.2.x86_64.qcow2" + FEDORA_CSUM_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/30/Cloud/x86_64/images/Fedora-Cloud-30-1.2-x86_64-CHECKSUM" + FEDORA_BASE_IMAGE_NAME: 'fedora-cloud-base-30-1-2' + # Prior Fedora release - PRIOR_FEDORA_IMAGE_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/28/Cloud/x86_64/images/Fedora-Cloud-Base-28-1.1.x86_64.qcow2" - PRIOR_FEDORA_CSUM_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/28/Cloud/x86_64/images/Fedora-Cloud-28-1.1-x86_64-CHECKSUM" - PRIOR_FEDORA_BASE_IMAGE_NAME: 'fedora-cloud-base-28-1-1' # Name to use in GCE + PRIOR_FEDORA_IMAGE_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/29/Cloud/x86_64/images/Fedora-Cloud-Base-29-1.2.x86_64.qcow2" + PRIOR_FEDORA_CSUM_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/29/Cloud/x86_64/images/Fedora-Cloud-29-1.2-x86_64-CHECKSUM" + PRIOR_FEDORA_BASE_IMAGE_NAME: 'fedora-cloud-base-29-1-2' # Name to use in GCE # The name of the image in GCE used for packer build libpod_images.yml IBI_BASE_NAME: 'image-builder-image' -- cgit v1.2.3-54-g00ecf