From 02bc3c9962f8d80a239bba1a6c8bd97d1c5ba44c Mon Sep 17 00:00:00 2001 From: Chris Evich Date: Mon, 8 Oct 2018 09:06:10 -0400 Subject: Fix Cirrus/Packer VM image building An invalid GCE value is being passed to packer, preventing it from building VM images. Fix this, and centralize the definition of the image name suffix by setting it at ``setup_environment.sh`` call-time, rather encoding inside packer's `libpod_images.json`. This makes the value available for use by other scripts. Also, switch the unique component of the name, to be based on the commit-sha being tested. This will improve traceability, since the git history is more permanent than the `CIRRUS_BUILD_ID` env. var. The later is subject to log-rotation, destroying evidence of the images source state. Signed-off-by: Chris Evich --- contrib/cirrus/packer/libpod_images.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'contrib/cirrus/packer/libpod_images.json') diff --git a/contrib/cirrus/packer/libpod_images.json b/contrib/cirrus/packer/libpod_images.json index 82a41ca25..ec2641282 100644 --- a/contrib/cirrus/packer/libpod_images.json +++ b/contrib/cirrus/packer/libpod_images.json @@ -15,7 +15,7 @@ "SERVICE_ACCOUNT": "{{env `SERVICE_ACCOUNT`}}", "GCP_PROJECT_ID": "{{env `GCP_PROJECT_ID`}}", - "CIRRUS_BUILD_ID": "{{env `CIRRUS_BUILD_ID`}}", + "BUILT_IMAGE_SUFFIX": "{{env `BUILT_IMAGE_SUFFIX`}}", "GCE_SSH_USERNAME": "{{env `GCE_SSH_USERNAME`}}", "RHSM_COMMAND": "{{env `RHSM_COMMAND`}}" }, @@ -29,7 +29,7 @@ "project_id": "{{user `GCP_PROJECT_ID`}}", "zone": "us-central1-a", "source_image": "{{user `RHEL_BASE_IMAGE`}}", - "image_name": "{{user `RHEL_BASE_IMAGE`}}-libpod-{{user `CIRRUS_BUILD_ID`}}", + "image_name": "{{user `RHEL_BASE_IMAGE`}}{{user `BUILT_IMAGE_SUFFIX`}}", "image_family": "{{user `RHEL_BASE_IMAGE`}}-libpod", "service_account_email": "{{user `SERVICE_ACCOUNT`}}", "communicator": "ssh", @@ -41,7 +41,7 @@ "project_id": "{{user `GCP_PROJECT_ID`}}", "zone": "us-central1-a", "source_image": "{{user `CENTOS_BASE_IMAGE`}}", - "image_name": "{{user `CENTOS_BASE_IMAGE`}}-libpod-{{user `CIRRUS_BUILD_ID`}}", + "image_name": "{{user `CENTOS_BASE_IMAGE`}}{{user `BUILT_IMAGE_SUFFIX`}}", "image_family": "{{user `CENTOS_BASE_IMAGE`}}-libpod", "service_account_email": "{{user `SERVICE_ACCOUNT`}}", "communicator": "ssh", @@ -53,7 +53,7 @@ "project_id": "{{user `GCP_PROJECT_ID`}}", "zone": "us-central1-a", "source_image": "{{user `FEDORA_BASE_IMAGE`}}", - "image_name": "{{user `FEDORA_BASE_IMAGE`}}-libpod-{{user `CIRRUS_BUILD_ID`}}", + "image_name": "{{user `FEDORA_BASE_IMAGE`}}{{user `BUILT_IMAGE_SUFFIX`}}", "image_family": "{{user `FEDORA_BASE_IMAGE`}}-libpod", "service_account_email": "{{user `SERVICE_ACCOUNT`}}", "communicator": "ssh", @@ -65,7 +65,7 @@ "project_id": "{{user `GCP_PROJECT_ID`}}", "zone": "us-central1-a", "source_image": "{{user `UBUNTU_BASE_IMAGE`}}", - "image_name": "{{user `UBUNTU_BASE_IMAGE`}}-libpod-{{user `CIRRUS_BUILD_ID`}}", + "image_name": "{{user `UBUNTU_BASE_IMAGE`}}{{user `BUILT_IMAGE_SUFFIX`}}", "image_family": "{{user `UBUNTU_BASE_IMAGE`}}-libpod", "service_account_email": "{{user `SERVICE_ACCOUNT`}}", "communicator": "ssh", -- cgit v1.2.3-54-g00ecf From a18494a345920bfbfa0c4db50731fd21f6ef7cd2 Mon Sep 17 00:00:00 2001 From: Chris Evich Date: Tue, 16 Oct 2018 17:16:29 -0400 Subject: Cirrus: Use different CNI_COMMIT for Fedora Just noticed this in Dockerfile.Fedora. Updated all the right places to make this happen. Signed-off-by: Chris Evich --- .cirrus.yml | 1 + contrib/cirrus/packer/libpod_images.json | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) (limited to 'contrib/cirrus/packer/libpod_images.json') diff --git a/.cirrus.yml b/.cirrus.yml index f59dd9396..04640662d 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -16,6 +16,7 @@ gce_instance: # Main collection of env. varss to set for all scripts. All others # are cooked in by $SCRIPT_BASE/setup_environment.sh env: + FEDORA_CNI_COMMIT: "412b6d31280682bb4fab4446f113c22ff1886554" CNI_COMMIT: "7480240de9749f9a0a5c8614b17f1f03e0c06ab9" CRIO_COMMIT: "662dbb31b5d4f5ed54511a47cde7190c61c28677" RUNC_COMMIT: "ad0f5255060d36872be04de22f8731f38ef2d7b1" diff --git a/contrib/cirrus/packer/libpod_images.json b/contrib/cirrus/packer/libpod_images.json index ec2641282..2135adef5 100644 --- a/contrib/cirrus/packer/libpod_images.json +++ b/contrib/cirrus/packer/libpod_images.json @@ -1,5 +1,6 @@ { "variables": { + "FEDORA_CNI_COMMIT": "{{env `FEDORA_CNI_COMMIT`}}", "CNI_COMMIT": "{{env `CNI_COMMIT`}}", "CRIO_COMMIT": "{{env `CRIO_COMMIT`}}", "RUNC_COMMIT": "{{env `RUNC_COMMIT`}}", @@ -105,7 +106,7 @@ "script": "{{user `GOSRC`}}/{{user `PACKER_BASE`}}/fedora_setup.sh", "environment_vars": [ "SCRIPT_BASE={{user `SCRIPT_BASE`}}", - "CNI_COMMIT={{user `CNI_COMMIT`}}", + "CNI_COMMIT={{user `FEDORA_CNI_COMMIT`}}", "CRIO_COMMIT={{user `CRIO_COMMIT`}}", "RUNC_COMMIT={{user `RUNC_COMMIT`}}" ] -- cgit v1.2.3-54-g00ecf From 89b5484ad2969ad139986da1ffd6855d708824b1 Mon Sep 17 00:00:00 2001 From: Chris Evich Date: Tue, 16 Oct 2018 15:53:16 -0400 Subject: Cirrus: Install CRIU in test images Signed-off-by: Chris Evich --- .cirrus.yml | 9 ++-- contrib/cirrus/lib.sh | 15 +++++++ contrib/cirrus/packer/centos_setup.sh | 72 ++++++++++++++++++------------ contrib/cirrus/packer/fedora_setup.sh | 16 ++++++- contrib/cirrus/packer/libpod_images.json | 5 +++ contrib/cirrus/packer/rhel_setup.sh | 76 +++++++++++++++++++------------- contrib/cirrus/packer/ubuntu_setup.sh | 11 ++++- 7 files changed, 137 insertions(+), 67 deletions(-) (limited to 'contrib/cirrus/packer/libpod_images.json') diff --git a/.cirrus.yml b/.cirrus.yml index 04640662d..473184ad2 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -19,6 +19,7 @@ env: FEDORA_CNI_COMMIT: "412b6d31280682bb4fab4446f113c22ff1886554" CNI_COMMIT: "7480240de9749f9a0a5c8614b17f1f03e0c06ab9" CRIO_COMMIT: "662dbb31b5d4f5ed54511a47cde7190c61c28677" + CRIU_COMMIT: "584cbe4643c3fc7dc901ff08bf923ca0fe7326f9" RUNC_COMMIT: "ad0f5255060d36872be04de22f8731f38ef2d7b1" # Enable debugging delay on test-failure if non-empty. FLAKE_DEBUG_DELAY: 1 @@ -42,11 +43,11 @@ full_vm_testing_task: # 'matrix' combinations. All run in parallel. matrix: # Images are generated separetly, from build_images_task (below) - image_name: "ubuntu-1804-bionic-v20180911-libpod-5763563410948096" + image_name: "ubuntu-1804-bionic-v20180911-libpod-fce09afe" # TODO: Make these work (also build_images_task below) - #image_name: "rhel-server-ec2-7-5-165-1-libpod-5358668723781632" - #image_name: "centos-7-v20180911-libpod-5358668723781632" - #image_name: "fedora-cloud-base-28-1-1-7-libpod-5358668723781632" + #image_name: "rhel-server-ec2-7-5-165-1-libpod-fce09afe" + #image_name: "centos-7-v20180911-libpod-fce09afe" + #image_name: "fedora-cloud-base-28-1-1-7-libpod-fce09afe" timeout_in: 120m diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh index 1b61fd5ec..c6cac44ec 100644 --- a/contrib/cirrus/lib.sh +++ b/contrib/cirrus/lib.sh @@ -202,6 +202,21 @@ install_conmon(){ sudo install -D -m 755 bin/conmon /usr/libexec/podman/conmon } +install_criu(){ + echo "Installing CRIU from commit $CRIU_COMMIT" + req_env_var " + CRIU_COMMIT $CRIU_COMMIT + " + DEST="/tmp/criu" + rm -rf "$DEST" + ooe.sh git clone https://github.com/checkpoint-restore/criu.git "$DEST" + cd $DEST + ooe.sh git fetch origin --tags + ooe.sh git checkout -q "$CRIU_COMMIT" + ooe.sh make + sudo install -D -m 755 criu/criu /usr/sbin/ +} + # Runs in testing VM, not image building install_testing_dependencies() { echo "Installing ginkgo, gomega, and easyjson into \$GOPATH=$GOPATH" diff --git a/contrib/cirrus/packer/centos_setup.sh b/contrib/cirrus/packer/centos_setup.sh index 2253d7b35..7b2308739 100644 --- a/contrib/cirrus/packer/centos_setup.sh +++ b/contrib/cirrus/packer/centos_setup.sh @@ -12,6 +12,7 @@ req_env_var " SCRIPT_BASE $SCRIPT_BASE CNI_COMMIT $CNI_COMMIT CRIO_COMMIT $CRIO_COMMIT +CRIU_COMMIT $CRIU_COMMIT " install_ooe @@ -24,35 +25,46 @@ ooe.sh sudo yum -y update ooe.sh sudo yum -y install centos-release-scl epel-release ooe.sh sudo yum -y install \ - atomic-registries \ - btrfs-progs-devel \ - bzip2 \ - device-mapper-devel \ - findutils \ - glib2-devel \ - glibc-static \ - gnupg \ - golang \ - golang-github-cpuguy83-go-md2man \ - golang-github-cpuguy83-go-md2man \ - gpgme-devel \ - iptables \ - libassuan-devel \ - libseccomp-devel \ - libselinux-devel \ - lsof \ - make \ - nmap-ncat \ - ostree-devel \ - python \ - python3-dateutil \ - python3-psutil \ - python3-pytoml \ - runc \ - skopeo-containers \ - unzip \ - which \ - xz + atomic-registries \ + btrfs-progs-devel \ + bzip2 \ + device-mapper-devel \ + findutils \ + glib2-devel \ + glibc-static \ + gnupg \ + golang \ + golang-github-cpuguy83-go-md2man \ + golang-github-cpuguy83-go-md2man \ + gpgme-devel \ + iptables \ + libassuan-devel \ + libcap-devel \ + libnet \ + libnet-devel \ + libnl3-devel \ + libseccomp-devel \ + libselinux-devel \ + lsof \ + make \ + nmap-ncat \ + ostree-devel \ + protobuf \ + protobuf-c \ + protobuf-c-devel \ + protobuf-compiler \ + protobuf-devel \ + protobuf-python \ + python \ + python2-future \ + python3-dateutil \ + python3-psutil \ + python3-pytoml \ + runc \ + skopeo-containers \ + unzip \ + which \ + xz install_scl_git @@ -62,6 +74,8 @@ install_buildah install_conmon +install_criu + install_packer_copied_files rh_finalize diff --git a/contrib/cirrus/packer/fedora_setup.sh b/contrib/cirrus/packer/fedora_setup.sh index 53709fbdd..16b6e4e6b 100644 --- a/contrib/cirrus/packer/fedora_setup.sh +++ b/contrib/cirrus/packer/fedora_setup.sh @@ -12,6 +12,7 @@ req_env_var " SCRIPT_BASE $SCRIPT_BASE CNI_COMMIT $CNI_COMMIT CRIO_COMMIT $CRIO_COMMIT +CRIU_COMMIT $CRIU_COMMIT RUNC_COMMIT $RUNC_COMMIT " @@ -40,6 +41,10 @@ ooe.sh sudo dnf install -y \ gpgme-devel \ iptables \ libassuan-devel \ + libcap-devel \ + libnet \ + libnet-devel \ + libnl3-devel \ libseccomp-devel \ libselinux-devel \ lsof \ @@ -47,14 +52,21 @@ ooe.sh sudo dnf install -y \ nmap-ncat \ ostree-devel \ procps-ng \ + protobuf \ + protobuf-c \ + protobuf-c-devel \ + protobuf-compiler \ + protobuf-devel \ + protobuf-python \ python \ + python2-future \ python3-dateutil \ python3-psutil \ python3-pytoml \ runc \ skopeo-containers \ slirp4netns \ - which\ + which \ xz install_varlink @@ -65,6 +77,8 @@ install_buildah install_conmon +install_criu + install_packer_copied_files rh_finalize # N/B: Halts system! diff --git a/contrib/cirrus/packer/libpod_images.json b/contrib/cirrus/packer/libpod_images.json index 2135adef5..9dac3e8ea 100644 --- a/contrib/cirrus/packer/libpod_images.json +++ b/contrib/cirrus/packer/libpod_images.json @@ -3,6 +3,7 @@ "FEDORA_CNI_COMMIT": "{{env `FEDORA_CNI_COMMIT`}}", "CNI_COMMIT": "{{env `CNI_COMMIT`}}", "CRIO_COMMIT": "{{env `CRIO_COMMIT`}}", + "CRIU_COMMIT": "{{env `CRIU_COMMIT`}}", "RUNC_COMMIT": "{{env `RUNC_COMMIT`}}", "CENTOS_BASE_IMAGE": "{{env `CENTOS_BASE_IMAGE`}}" , @@ -87,6 +88,7 @@ "SCRIPT_BASE={{user `SCRIPT_BASE`}}", "CNI_COMMIT={{user `CNI_COMMIT`}}", "CRIO_COMMIT={{user `CRIO_COMMIT`}}", + "CRIU_COMMIT={{user `CRIU_COMMIT`}}", "RUNC_COMMIT={{user `RUNC_COMMIT`}}", "RHSM_COMMAND={{user `RHSM_COMMAND`}}" ] @@ -98,6 +100,7 @@ "SCRIPT_BASE={{user `SCRIPT_BASE`}}", "CNI_COMMIT={{user `CNI_COMMIT`}}", "CRIO_COMMIT={{user `CRIO_COMMIT`}}", + "CRIU_COMMIT={{user `CRIU_COMMIT`}}", "RUNC_COMMIT={{user `RUNC_COMMIT`}}" ] },{ @@ -108,6 +111,7 @@ "SCRIPT_BASE={{user `SCRIPT_BASE`}}", "CNI_COMMIT={{user `FEDORA_CNI_COMMIT`}}", "CRIO_COMMIT={{user `CRIO_COMMIT`}}", + "CRIU_COMMIT={{user `CRIU_COMMIT`}}", "RUNC_COMMIT={{user `RUNC_COMMIT`}}" ] },{ @@ -118,6 +122,7 @@ "SCRIPT_BASE={{user `SCRIPT_BASE`}}", "CNI_COMMIT={{user `CNI_COMMIT`}}", "CRIO_COMMIT={{user `CRIO_COMMIT`}}", + "CRIU_COMMIT={{user `CRIU_COMMIT`}}", "RUNC_COMMIT={{user `RUNC_COMMIT`}}" ] } diff --git a/contrib/cirrus/packer/rhel_setup.sh b/contrib/cirrus/packer/rhel_setup.sh index b776a0d97..d296713fc 100644 --- a/contrib/cirrus/packer/rhel_setup.sh +++ b/contrib/cirrus/packer/rhel_setup.sh @@ -12,6 +12,7 @@ req_env_var " SCRIPT_BASE $SCRIPT_BASE CNI_COMMIT $CNI_COMMIT CRIO_COMMIT $CRIO_COMMIT +CRIU_COMMIT $CRIU_COMMIT RHSM_COMMAND $RHSM_COMMAND " @@ -62,37 +63,48 @@ gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg EOM ooe.sh sudo yum -y install \ - atomic-registries \ - btrfs-progs-devel \ - bzip2 \ - device-mapper-devel \ - findutils \ - glib2-devel \ - glibc-static \ - gnupg \ - golang \ - golang-github-cpuguy83-go-md2man \ - golang-github-cpuguy83-go-md2man \ - google-compute-engine \ - google-compute-engine-oslogin \ - gpgme-devel \ - iptables \ - libassuan-devel \ - libseccomp-devel \ - libselinux-devel \ - lsof \ - make \ - nmap-ncat \ - ostree-devel \ - python \ - python34-dateutil \ - python34-psutil \ - python34-pytoml \ - runc \ - skopeo-containers \ - unzip \ - which \ - xz + atomic-registries \ + btrfs-progs-devel \ + bzip2 \ + device-mapper-devel \ + findutils \ + glib2-devel \ + glibc-static \ + gnupg \ + golang \ + golang-github-cpuguy83-go-md2man \ + golang-github-cpuguy83-go-md2man \ + google-compute-engine \ + google-compute-engine-oslogin \ + gpgme-devel \ + iptables \ + libassuan-devel \ + libcap-devel \ + libnet \ + libnet-devel \ + libnl3-devel \ + libseccomp-devel \ + libselinux-devel \ + lsof \ + make \ + nmap-ncat \ + ostree-devel \ + protobuf \ + protobuf-c \ + protobuf-c-devel \ + protobuf-compiler \ + protobuf-devel \ + protobuf-python \ + python \ + python2-future \ + python34-dateutil \ + python34-psutil \ + python34-pytoml \ + runc \ + skopeo-containers \ + unzip \ + which \ + xz install_scl_git @@ -102,6 +114,8 @@ install_buildah install_conmon +install_criu + install_packer_copied_files exit_handler # release subscription! diff --git a/contrib/cirrus/packer/ubuntu_setup.sh b/contrib/cirrus/packer/ubuntu_setup.sh index 96b3a573f..ff20944dc 100644 --- a/contrib/cirrus/packer/ubuntu_setup.sh +++ b/contrib/cirrus/packer/ubuntu_setup.sh @@ -12,6 +12,7 @@ req_env_var " SCRIPT_BASE $SCRIPT_BASE CNI_COMMIT $CNI_COMMIT CRIO_COMMIT $CRIO_COMMIT +CRIU_COMMIT $CRIU_COMMIT RUNC_COMMIT $RUNC_COMMIT " @@ -34,8 +35,8 @@ ooe.sh sudo apt-get -qq install --no-install-recommends \ e2fslibs-dev \ gawk \ gettext \ - golang \ go-md2man \ + golang \ iptables \ libaio-dev \ libapparmor-dev \ @@ -46,18 +47,22 @@ ooe.sh sudo apt-get -qq install --no-install-recommends \ libglib2.0-dev \ libgpgme11-dev \ liblzma-dev \ + libnet1 \ + libnet1-dev \ + libnl-3-dev \ libostree-dev \ libprotobuf-c0-dev \ libprotobuf-dev \ libtool \ - libtool \ libudev-dev \ lsof \ netcat \ pkg-config \ protobuf-c-compiler \ protobuf-compiler \ + python-future \ python-minimal \ + python-protobuf \ python3-dateutil \ python3-pip \ python3-psutil \ @@ -77,6 +82,8 @@ install_runc install_conmon +install_criu + install_cni_plugins install_buildah -- cgit v1.2.3-54-g00ecf