diff options
Diffstat (limited to 'contrib/cirrus')
-rwxr-xr-x | contrib/cirrus/build_vm_images.sh | 2 | ||||
-rw-r--r-- | contrib/cirrus/lib.sh | 33 | ||||
-rw-r--r-- | contrib/cirrus/packer/centos_setup.sh | 72 | ||||
-rw-r--r-- | contrib/cirrus/packer/fedora_setup.sh | 16 | ||||
-rw-r--r-- | contrib/cirrus/packer/libpod_images.json | 18 | ||||
-rw-r--r-- | contrib/cirrus/packer/rhel_setup.sh | 76 | ||||
-rw-r--r-- | contrib/cirrus/packer/ubuntu_setup.sh | 11 | ||||
-rwxr-xr-x | contrib/cirrus/podbot.py | 98 | ||||
-rwxr-xr-x | contrib/cirrus/setup_environment.sh | 1 | ||||
-rwxr-xr-x | contrib/cirrus/success.sh | 22 |
10 files changed, 278 insertions, 71 deletions
diff --git a/contrib/cirrus/build_vm_images.sh b/contrib/cirrus/build_vm_images.sh index 8538ee910..80c689a6c 100755 --- a/contrib/cirrus/build_vm_images.sh +++ b/contrib/cirrus/build_vm_images.sh @@ -13,7 +13,7 @@ UBUNTU_BASE_IMAGE $UBUNTU_BASE_IMAGE FEDORA_BASE_IMAGE $FEDORA_BASE_IMAGE RHEL_BASE_IMAGE $RHEL_BASE_IMAGE RHSM_COMMAND $RHSM_COMMAND -CIRRUS_BUILD_ID $CIRRUS_BUILD_ID +BUILT_IMAGE_SUFFIX $BUILT_IMAGE_SUFFIX SERVICE_ACCOUNT $SERVICE_ACCOUNT GCE_SSH_USERNAME $GCE_SSH_USERNAME GCP_PROJECT_ID $GCP_PROJECT_ID diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh index e69f1e040..1e0052a65 100644 --- a/contrib/cirrus/lib.sh +++ b/contrib/cirrus/lib.sh @@ -35,8 +35,11 @@ req_env_var() { # and useful variables. # ref: https://cirrus-ci.org/guide/writing-tasks/#environment-variables show_env_vars() { + # This is almost always multi-line, print it separately + echo "export CIRRUS_CHANGE_MESSAGE=$CIRRUS_CHANGE_MESSAGE" echo " BUILDTAGS $BUILDTAGS +BUILT_IMAGE_SUFFIX $BUILT_IMAGE_SUFFIX CI $CI CIRRUS_CI $CIRRUS_CI CI_NODE_INDEX $CI_NODE_INDEX @@ -47,7 +50,6 @@ CIRRUS_BASE_SHA $CIRRUS_BASE_SHA CIRRUS_BRANCH $CIRRUS_BRANCH CIRRUS_BUILD_ID $CIRRUS_BUILD_ID CIRRUS_CHANGE_IN_REPO $CIRRUS_CHANGE_IN_REPO -CIRRUS_CHANGE_MESSAGE $CIRRUS_CHANGE_MESSAGE CIRRUS_CLONE_DEPTH $CIRRUS_CLONE_DEPTH CIRRUS_DEFAULT_BRANCH $CIRRUS_DEFAULT_BRANCH CIRRUS_PR $CIRRUS_PR @@ -66,6 +68,7 @@ CIRRUS_USER_PERMISSION $CIRRUS_USER_PERMISSION CIRRUS_WORKING_DIR $CIRRUS_WORKING_DIR CIRRUS_HTTP_CACHE_HOST $CIRRUS_HTTP_CACHE_HOST $(go env) +PACKER_BUILDS $PACKER_BUILDS " | while read NAME VALUE do [[ -z "$NAME" ]] || echo "export $NAME=\"$VALUE\"" @@ -93,6 +96,19 @@ stub() { echo "STUB: Pretending to do $1" } +ircmsg() { + req_env_var " + SCRIPT_BASE $SCRIPT_BASE + GOSRC $GOSRC + CIRRUS_TASK_ID $CIRRUS_TASK_ID + 1 $1 + " + SCRIPT="$GOSRC/$SCRIPT_BASE/podbot.py" + NICK="podbot_$CIRRUS_TASK_ID" + NICK="${NICK:0:15}" # Any longer will break things + $SCRIPT $NICK $1 +} + # Run sudo in directory with GOPATH set cdsudo() { DIR="$1" @@ -199,6 +215,21 @@ install_conmon(){ sudo install -D -m 755 bin/conmon /usr/libexec/podman/conmon } +install_criu(){ + echo "Installing CRIU from commit $CRIU_COMMIT" + req_env_var " + CRIU_COMMIT $CRIU_COMMIT + " + DEST="/tmp/criu" + rm -rf "$DEST" + ooe.sh git clone https://github.com/checkpoint-restore/criu.git "$DEST" + cd $DEST + ooe.sh git fetch origin --tags + ooe.sh git checkout -q "$CRIU_COMMIT" + ooe.sh make + sudo install -D -m 755 criu/criu /usr/sbin/ +} + # Runs in testing VM, not image building install_testing_dependencies() { echo "Installing ginkgo, gomega, and easyjson into \$GOPATH=$GOPATH" diff --git a/contrib/cirrus/packer/centos_setup.sh b/contrib/cirrus/packer/centos_setup.sh index 2253d7b35..7b2308739 100644 --- a/contrib/cirrus/packer/centos_setup.sh +++ b/contrib/cirrus/packer/centos_setup.sh @@ -12,6 +12,7 @@ req_env_var " SCRIPT_BASE $SCRIPT_BASE CNI_COMMIT $CNI_COMMIT CRIO_COMMIT $CRIO_COMMIT +CRIU_COMMIT $CRIU_COMMIT " install_ooe @@ -24,35 +25,46 @@ ooe.sh sudo yum -y update ooe.sh sudo yum -y install centos-release-scl epel-release ooe.sh sudo yum -y install \ - atomic-registries \ - btrfs-progs-devel \ - bzip2 \ - device-mapper-devel \ - findutils \ - glib2-devel \ - glibc-static \ - gnupg \ - golang \ - golang-github-cpuguy83-go-md2man \ - golang-github-cpuguy83-go-md2man \ - gpgme-devel \ - iptables \ - libassuan-devel \ - libseccomp-devel \ - libselinux-devel \ - lsof \ - make \ - nmap-ncat \ - ostree-devel \ - python \ - python3-dateutil \ - python3-psutil \ - python3-pytoml \ - runc \ - skopeo-containers \ - unzip \ - which \ - xz + atomic-registries \ + btrfs-progs-devel \ + bzip2 \ + device-mapper-devel \ + findutils \ + glib2-devel \ + glibc-static \ + gnupg \ + golang \ + golang-github-cpuguy83-go-md2man \ + golang-github-cpuguy83-go-md2man \ + gpgme-devel \ + iptables \ + libassuan-devel \ + libcap-devel \ + libnet \ + libnet-devel \ + libnl3-devel \ + libseccomp-devel \ + libselinux-devel \ + lsof \ + make \ + nmap-ncat \ + ostree-devel \ + protobuf \ + protobuf-c \ + protobuf-c-devel \ + protobuf-compiler \ + protobuf-devel \ + protobuf-python \ + python \ + python2-future \ + python3-dateutil \ + python3-psutil \ + python3-pytoml \ + runc \ + skopeo-containers \ + unzip \ + which \ + xz install_scl_git @@ -62,6 +74,8 @@ install_buildah install_conmon +install_criu + install_packer_copied_files rh_finalize diff --git a/contrib/cirrus/packer/fedora_setup.sh b/contrib/cirrus/packer/fedora_setup.sh index 53709fbdd..16b6e4e6b 100644 --- a/contrib/cirrus/packer/fedora_setup.sh +++ b/contrib/cirrus/packer/fedora_setup.sh @@ -12,6 +12,7 @@ req_env_var " SCRIPT_BASE $SCRIPT_BASE CNI_COMMIT $CNI_COMMIT CRIO_COMMIT $CRIO_COMMIT +CRIU_COMMIT $CRIU_COMMIT RUNC_COMMIT $RUNC_COMMIT " @@ -40,6 +41,10 @@ ooe.sh sudo dnf install -y \ gpgme-devel \ iptables \ libassuan-devel \ + libcap-devel \ + libnet \ + libnet-devel \ + libnl3-devel \ libseccomp-devel \ libselinux-devel \ lsof \ @@ -47,14 +52,21 @@ ooe.sh sudo dnf install -y \ nmap-ncat \ ostree-devel \ procps-ng \ + protobuf \ + protobuf-c \ + protobuf-c-devel \ + protobuf-compiler \ + protobuf-devel \ + protobuf-python \ python \ + python2-future \ python3-dateutil \ python3-psutil \ python3-pytoml \ runc \ skopeo-containers \ slirp4netns \ - which\ + which \ xz install_varlink @@ -65,6 +77,8 @@ install_buildah install_conmon +install_criu + install_packer_copied_files rh_finalize # N/B: Halts system! diff --git a/contrib/cirrus/packer/libpod_images.json b/contrib/cirrus/packer/libpod_images.json index 82a41ca25..9dac3e8ea 100644 --- a/contrib/cirrus/packer/libpod_images.json +++ b/contrib/cirrus/packer/libpod_images.json @@ -1,7 +1,9 @@ { "variables": { + "FEDORA_CNI_COMMIT": "{{env `FEDORA_CNI_COMMIT`}}", "CNI_COMMIT": "{{env `CNI_COMMIT`}}", "CRIO_COMMIT": "{{env `CRIO_COMMIT`}}", + "CRIU_COMMIT": "{{env `CRIU_COMMIT`}}", "RUNC_COMMIT": "{{env `RUNC_COMMIT`}}", "CENTOS_BASE_IMAGE": "{{env `CENTOS_BASE_IMAGE`}}" , @@ -15,7 +17,7 @@ "SERVICE_ACCOUNT": "{{env `SERVICE_ACCOUNT`}}", "GCP_PROJECT_ID": "{{env `GCP_PROJECT_ID`}}", - "CIRRUS_BUILD_ID": "{{env `CIRRUS_BUILD_ID`}}", + "BUILT_IMAGE_SUFFIX": "{{env `BUILT_IMAGE_SUFFIX`}}", "GCE_SSH_USERNAME": "{{env `GCE_SSH_USERNAME`}}", "RHSM_COMMAND": "{{env `RHSM_COMMAND`}}" }, @@ -29,7 +31,7 @@ "project_id": "{{user `GCP_PROJECT_ID`}}", "zone": "us-central1-a", "source_image": "{{user `RHEL_BASE_IMAGE`}}", - "image_name": "{{user `RHEL_BASE_IMAGE`}}-libpod-{{user `CIRRUS_BUILD_ID`}}", + "image_name": "{{user `RHEL_BASE_IMAGE`}}{{user `BUILT_IMAGE_SUFFIX`}}", "image_family": "{{user `RHEL_BASE_IMAGE`}}-libpod", "service_account_email": "{{user `SERVICE_ACCOUNT`}}", "communicator": "ssh", @@ -41,7 +43,7 @@ "project_id": "{{user `GCP_PROJECT_ID`}}", "zone": "us-central1-a", "source_image": "{{user `CENTOS_BASE_IMAGE`}}", - "image_name": "{{user `CENTOS_BASE_IMAGE`}}-libpod-{{user `CIRRUS_BUILD_ID`}}", + "image_name": "{{user `CENTOS_BASE_IMAGE`}}{{user `BUILT_IMAGE_SUFFIX`}}", "image_family": "{{user `CENTOS_BASE_IMAGE`}}-libpod", "service_account_email": "{{user `SERVICE_ACCOUNT`}}", "communicator": "ssh", @@ -53,7 +55,7 @@ "project_id": "{{user `GCP_PROJECT_ID`}}", "zone": "us-central1-a", "source_image": "{{user `FEDORA_BASE_IMAGE`}}", - "image_name": "{{user `FEDORA_BASE_IMAGE`}}-libpod-{{user `CIRRUS_BUILD_ID`}}", + "image_name": "{{user `FEDORA_BASE_IMAGE`}}{{user `BUILT_IMAGE_SUFFIX`}}", "image_family": "{{user `FEDORA_BASE_IMAGE`}}-libpod", "service_account_email": "{{user `SERVICE_ACCOUNT`}}", "communicator": "ssh", @@ -65,7 +67,7 @@ "project_id": "{{user `GCP_PROJECT_ID`}}", "zone": "us-central1-a", "source_image": "{{user `UBUNTU_BASE_IMAGE`}}", - "image_name": "{{user `UBUNTU_BASE_IMAGE`}}-libpod-{{user `CIRRUS_BUILD_ID`}}", + "image_name": "{{user `UBUNTU_BASE_IMAGE`}}{{user `BUILT_IMAGE_SUFFIX`}}", "image_family": "{{user `UBUNTU_BASE_IMAGE`}}-libpod", "service_account_email": "{{user `SERVICE_ACCOUNT`}}", "communicator": "ssh", @@ -86,6 +88,7 @@ "SCRIPT_BASE={{user `SCRIPT_BASE`}}", "CNI_COMMIT={{user `CNI_COMMIT`}}", "CRIO_COMMIT={{user `CRIO_COMMIT`}}", + "CRIU_COMMIT={{user `CRIU_COMMIT`}}", "RUNC_COMMIT={{user `RUNC_COMMIT`}}", "RHSM_COMMAND={{user `RHSM_COMMAND`}}" ] @@ -97,6 +100,7 @@ "SCRIPT_BASE={{user `SCRIPT_BASE`}}", "CNI_COMMIT={{user `CNI_COMMIT`}}", "CRIO_COMMIT={{user `CRIO_COMMIT`}}", + "CRIU_COMMIT={{user `CRIU_COMMIT`}}", "RUNC_COMMIT={{user `RUNC_COMMIT`}}" ] },{ @@ -105,8 +109,9 @@ "script": "{{user `GOSRC`}}/{{user `PACKER_BASE`}}/fedora_setup.sh", "environment_vars": [ "SCRIPT_BASE={{user `SCRIPT_BASE`}}", - "CNI_COMMIT={{user `CNI_COMMIT`}}", + "CNI_COMMIT={{user `FEDORA_CNI_COMMIT`}}", "CRIO_COMMIT={{user `CRIO_COMMIT`}}", + "CRIU_COMMIT={{user `CRIU_COMMIT`}}", "RUNC_COMMIT={{user `RUNC_COMMIT`}}" ] },{ @@ -117,6 +122,7 @@ "SCRIPT_BASE={{user `SCRIPT_BASE`}}", "CNI_COMMIT={{user `CNI_COMMIT`}}", "CRIO_COMMIT={{user `CRIO_COMMIT`}}", + "CRIU_COMMIT={{user `CRIU_COMMIT`}}", "RUNC_COMMIT={{user `RUNC_COMMIT`}}" ] } diff --git a/contrib/cirrus/packer/rhel_setup.sh b/contrib/cirrus/packer/rhel_setup.sh index b776a0d97..d296713fc 100644 --- a/contrib/cirrus/packer/rhel_setup.sh +++ b/contrib/cirrus/packer/rhel_setup.sh @@ -12,6 +12,7 @@ req_env_var " SCRIPT_BASE $SCRIPT_BASE CNI_COMMIT $CNI_COMMIT CRIO_COMMIT $CRIO_COMMIT +CRIU_COMMIT $CRIU_COMMIT RHSM_COMMAND $RHSM_COMMAND " @@ -62,37 +63,48 @@ gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg EOM ooe.sh sudo yum -y install \ - atomic-registries \ - btrfs-progs-devel \ - bzip2 \ - device-mapper-devel \ - findutils \ - glib2-devel \ - glibc-static \ - gnupg \ - golang \ - golang-github-cpuguy83-go-md2man \ - golang-github-cpuguy83-go-md2man \ - google-compute-engine \ - google-compute-engine-oslogin \ - gpgme-devel \ - iptables \ - libassuan-devel \ - libseccomp-devel \ - libselinux-devel \ - lsof \ - make \ - nmap-ncat \ - ostree-devel \ - python \ - python34-dateutil \ - python34-psutil \ - python34-pytoml \ - runc \ - skopeo-containers \ - unzip \ - which \ - xz + atomic-registries \ + btrfs-progs-devel \ + bzip2 \ + device-mapper-devel \ + findutils \ + glib2-devel \ + glibc-static \ + gnupg \ + golang \ + golang-github-cpuguy83-go-md2man \ + golang-github-cpuguy83-go-md2man \ + google-compute-engine \ + google-compute-engine-oslogin \ + gpgme-devel \ + iptables \ + libassuan-devel \ + libcap-devel \ + libnet \ + libnet-devel \ + libnl3-devel \ + libseccomp-devel \ + libselinux-devel \ + lsof \ + make \ + nmap-ncat \ + ostree-devel \ + protobuf \ + protobuf-c \ + protobuf-c-devel \ + protobuf-compiler \ + protobuf-devel \ + protobuf-python \ + python \ + python2-future \ + python34-dateutil \ + python34-psutil \ + python34-pytoml \ + runc \ + skopeo-containers \ + unzip \ + which \ + xz install_scl_git @@ -102,6 +114,8 @@ install_buildah install_conmon +install_criu + install_packer_copied_files exit_handler # release subscription! diff --git a/contrib/cirrus/packer/ubuntu_setup.sh b/contrib/cirrus/packer/ubuntu_setup.sh index 96b3a573f..ff20944dc 100644 --- a/contrib/cirrus/packer/ubuntu_setup.sh +++ b/contrib/cirrus/packer/ubuntu_setup.sh @@ -12,6 +12,7 @@ req_env_var " SCRIPT_BASE $SCRIPT_BASE CNI_COMMIT $CNI_COMMIT CRIO_COMMIT $CRIO_COMMIT +CRIU_COMMIT $CRIU_COMMIT RUNC_COMMIT $RUNC_COMMIT " @@ -34,8 +35,8 @@ ooe.sh sudo apt-get -qq install --no-install-recommends \ e2fslibs-dev \ gawk \ gettext \ - golang \ go-md2man \ + golang \ iptables \ libaio-dev \ libapparmor-dev \ @@ -46,18 +47,22 @@ ooe.sh sudo apt-get -qq install --no-install-recommends \ libglib2.0-dev \ libgpgme11-dev \ liblzma-dev \ + libnet1 \ + libnet1-dev \ + libnl-3-dev \ libostree-dev \ libprotobuf-c0-dev \ libprotobuf-dev \ libtool \ - libtool \ libudev-dev \ lsof \ netcat \ pkg-config \ protobuf-c-compiler \ protobuf-compiler \ + python-future \ python-minimal \ + python-protobuf \ python3-dateutil \ python3-pip \ python3-psutil \ @@ -77,6 +82,8 @@ install_runc install_conmon +install_criu + install_cni_plugins install_buildah diff --git a/contrib/cirrus/podbot.py b/contrib/cirrus/podbot.py new file mode 100755 index 000000000..1be41a8ed --- /dev/null +++ b/contrib/cirrus/podbot.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 + +# Simple and dumb script to send a message to the #podman IRC channel on frenode +# Based on example from: https://pythonspot.com/building-an-irc-bot/ + +import os +import time +import random +import errno +import socket +import sys + +class IRC: + + response_timeout = 10 # seconds + irc = socket.socket() + + def __init__(self, server, nickname, channel): + self.server = server + self.nickname = nickname + self.channel = channel + self.irc = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + + def _send(self, cmdstr): + self.irc.send(bytes(cmdstr + '\r\n', 'utf-8')) + + def message(self, msg): + data = 'PRIVMSG {0} :{1}\r\n'.format(self.channel, msg) + print(data) + self._send(data) + + @staticmethod + def fix_newlines(bufr): + return bufr.replace('\\r\\n', '\n') + + def _required_response(self, needle, haystack): + start = time.time() + end = start + self.response_timeout + while time.time() < end: + if haystack.find(needle) != -1: + return (False, haystack) + time.sleep(0.1) + try: + haystack += str(self.irc.recv(4096, socket.MSG_DONTWAIT)) + except socket.error as serr: + if serr.errno == errno.EWOULDBLOCK: + continue + raise # can't handle this + return (True, haystack) # Error + + def connect(self, username, password): + # This is ugly as sin, but seems to be a working send/expect sequence + + print("connecting to: {0}".format(self.server)) + self.irc.connect((self.server, 6667)) #connects to the server + self._send("USER {0} {0} {0} :I am {0}".format(self.nickname)) + self._send("NICK {0}".format(self.nickname)) + + err, haystack = self._required_response('End of /MOTD command.' + ''.format(self.nickname), "") + if err: + print(self.fix_newlines(haystack)) + print("Error connecting to {0}".format(self.server)) + return True + + print("Logging in as {0}".format(username)) + self._send("PRIVMSG NickServ :IDENTIFY {0} {1}".format(username, password)) + err, _ = self._required_response("You are now identified for", "") + if err: + print("Error logging in to {0} as {1}".format(self.server, username)) + return True + + print("Joining {0}".format(self.channel)) + self._send("JOIN {0}".format(self.channel)) + err, haystack = self._required_response("{0} {1} :End of /NAMES list." + "".format(self.nickname, self.channel), + haystack) + print(self.fix_newlines(haystack)) + if err: + print("Error joining {0}".format(self.channel)) + return True + return False + + def quit(self): + print("Quitting") + self._send("QUIT :my work is done here") + self.irc.close() + + +if len(sys.argv) < 3: + print("Error: Must pass desired nick and message as parameters") +else: + irc = IRC("irc.freenode.net", sys.argv[1], "#podman") + err = irc.connect(*os.environ.get('IRCID', 'Big Bug').split(" ", 2)) + if not err: + irc.message(" ".join(sys.argv[2:])) + time.sleep(5.0) # avoid join/quit spam + irc.quit() diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh index 2302f0e15..167db127f 100755 --- a/contrib/cirrus/setup_environment.sh +++ b/contrib/cirrus/setup_environment.sh @@ -41,6 +41,7 @@ then "export OS_RELEASE_ID=\"$(os_release_id)\"" \ "export OS_RELEASE_VER=\"$(os_release_ver)\"" \ "export OS_REL_VER=\"${OS_RELEASE_ID}-${OS_RELEASE_VER}\"" \ + "export BUILT_IMAGE_SUFFIX=\"-$CIRRUS_REPO_NAME-${CIRRUS_CHANGE_IN_REPO:0:8}\"" \ "export GOPATH=\"/go\"" \ 'export PATH="$HOME/bin:$GOPATH/bin:/usr/local/bin:$PATH"' \ 'export LD_LIBRARY_PATH="/usr/local/lib${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}"' diff --git a/contrib/cirrus/success.sh b/contrib/cirrus/success.sh new file mode 100755 index 000000000..d1daf9043 --- /dev/null +++ b/contrib/cirrus/success.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +set -e +source $(dirname $0)/lib.sh + +req_env_var " + CIRRUS_TASK_NAME $CIRRUS_TASK_NAME + CIRRUS_BRANCH $CIRRUS_BRANCH + OS_RELEASE_ID $OS_RELEASE_ID + OS_RELEASE_VER $OS_RELEASE_VER + CIRRUS_REPO_CLONE_URL $CIRRUS_REPO_CLONE_URL +" + +REF_URL="$(echo $CIRRUS_REPO_CLONE_URL | sed 's/.git$//g')" +if [[ "$CIRRUS_BRANCH" =~ "pull" ]] +then + REF_URL="$REF_URL/$CIRRUS_BRANCH" # pull request URL +else + REF_URL="$REF_URL/commits/$CIRRUS_BRANCH" # branch merge +fi + +ircmsg "Cirrus-CI $CIRRUS_TASK_NAME on $OS_RELEASE_ID-$OS_RELEASE_VER successful for $REF_URL" |