Merge pull request #1660 from cevich/cirrus_criu

Cirrus: Install CRIU in test images

9 files changed, 162 insertions, 84 deletions

diff --git a/.cirrus.yml b/.cirrus.yml
index c00f2d095..473184ad2 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -16,8 +16,10 @@ gce_instance:
 # Main collection of env. varss to set for all scripts. All others
 # are cooked in by $SCRIPT_BASE/setup_environment.sh
 env:
+    FEDORA_CNI_COMMIT: "412b6d31280682bb4fab4446f113c22ff1886554"
     CNI_COMMIT: "7480240de9749f9a0a5c8614b17f1f03e0c06ab9"
     CRIO_COMMIT: "662dbb31b5d4f5ed54511a47cde7190c61c28677"
+    CRIU_COMMIT: "584cbe4643c3fc7dc901ff08bf923ca0fe7326f9"
     RUNC_COMMIT: "ad0f5255060d36872be04de22f8731f38ef2d7b1"
     # Enable debugging delay on test-failure if non-empty.
     FLAKE_DEBUG_DELAY: 1
@@ -41,11 +43,11 @@ full_vm_testing_task:
         # 'matrix' combinations.  All run in parallel.
         matrix:
             # Images are generated separetly, from build_images_task (below)
-            image_name: "ubuntu-1804-bionic-v20180911-libpod-5763563410948096"
+            image_name: "ubuntu-1804-bionic-v20180911-libpod-fce09afe"
             # TODO: Make these work (also build_images_task below)
-            #image_name: "rhel-server-ec2-7-5-165-1-libpod-5358668723781632"
-            #image_name: "centos-7-v20180911-libpod-5358668723781632"
-            #image_name: "fedora-cloud-base-28-1-1-7-libpod-5358668723781632"
+            #image_name: "rhel-server-ec2-7-5-165-1-libpod-fce09afe"
+            #image_name: "centos-7-v20180911-libpod-fce09afe"
+            #image_name: "fedora-cloud-base-28-1-1-7-libpod-fce09afe"
 
     timeout_in: 120m
 
@@ -82,19 +84,21 @@ build_vm_images_task:
         PACKER_BUILDS: "ubuntu-18"
         # TODO: Make these work (also full_vm_testing_task above)
         # PACKER_BUILDS: "rhel-7,centos-7,fedora-28,ubuntu-18"
-        # Command to register a RHEL VM
-        RHSM_COMMAND: ENCRYPTED[fec01433222af1ed0b8e40e89e7d18f6ee2fa9f49a1e721dc72f7eed3c740661215d1bd05cb54ac66a1a62116b92bdce]
-        # Additional environment variables needed to build GCE images, within a GCE VM
-        SERVICE_ACCOUNT: ENCRYPTED[02e03838b1156eb9516c7cc1e888e287910759842275f3c7bc2b4d56075cc6740e29ffa0ab71ebdbbd079673361dd8c9]
-        GCE_SSH_USERNAME: ENCRYPTED[a19a4ec62423e3e0fe4e7d1a5c9f11eda8fde321b9047ab5ed5590c2b1d7a2d12091c2be1531f949eae927059c2ae531]
-        GCP_PROJECT_ID: ENCRYPTED[77cb2d392bbc8d17412547d7d91f8d190089bf6e6b96eab9927994bbff6ab2c691ba0329ac7a650ba6182fbbab9fb68d]
-        # Existing base values to use, output images get epoc stamped names
-        PACKER_VER: "1.3.1"
-        # low-level base VM image name inputs to packer
         CENTOS_BASE_IMAGE: "centos-7-v20180911"
         RHEL_BASE_IMAGE: "rhel-server-ec2-7-5-165-1"
         FEDORA_BASE_IMAGE: "fedora-cloud-base-28-1-1-7"
         UBUNTU_BASE_IMAGE: "ubuntu-1804-bionic-v20180911"
+        # low-level base VM image name inputs to packer
+
+        # Command to register a RHEL VM
+        RHSM_COMMAND: ENCRYPTED[5caa5ff8c5370c3d25c7a1a28168501ab0fa2e5e3b627926f6eaba02b3fed965a7638a6151657809661f8c905c7dc187]
+        # Additional environment variables needed to build GCE images, within a GCE VM
+        SERVICE_ACCOUNT: ENCRYPTED[99e9a0b1c23f8dd29e83dfdf164f064cfd17afd9b895ca3b5e4c41170bd4290a8366fe2ad8e7a210b9f751711d1d002a]
+        GCE_SSH_USERNAME: ENCRYPTED[a7706b9e4b8bbb47f76358df7407f4fffa2e8552531190cc0b3315180c4b50588f560c4f85731e99cb5f43a396778277]
+        GCP_PROJECT_ID: ENCRYPTED[7c80e728e046b1c76147afd156a32c1c57d4a1ac1eab93b7e68e718c61ca8564fc61fef815952b8ae0a64e7034b8fe4f]
+        # Version of packer to use
+        PACKER_VER: "1.3.1"
+
 
     gce_instance:
         image_name: "image-builder-image"  # Simply CentOS 7 + packer dependencies
diff --git a/contrib/cirrus/build_vm_images.sh b/contrib/cirrus/build_vm_images.sh
index 8538ee910..80c689a6c 100755
--- a/contrib/cirrus/build_vm_images.sh
+++ b/contrib/cirrus/build_vm_images.sh
@@ -13,7 +13,7 @@ UBUNTU_BASE_IMAGE $UBUNTU_BASE_IMAGE
 FEDORA_BASE_IMAGE $FEDORA_BASE_IMAGE
 RHEL_BASE_IMAGE $RHEL_BASE_IMAGE
 RHSM_COMMAND $RHSM_COMMAND
-CIRRUS_BUILD_ID $CIRRUS_BUILD_ID
+BUILT_IMAGE_SUFFIX $BUILT_IMAGE_SUFFIX
 SERVICE_ACCOUNT $SERVICE_ACCOUNT
 GCE_SSH_USERNAME $GCE_SSH_USERNAME
 GCP_PROJECT_ID $GCP_PROJECT_ID
diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh
index e69f1e040..c6cac44ec 100644
--- a/contrib/cirrus/lib.sh
+++ b/contrib/cirrus/lib.sh
@@ -35,8 +35,11 @@ req_env_var() {
 # and useful variables.
 # ref: https://cirrus-ci.org/guide/writing-tasks/#environment-variables
 show_env_vars() {
+    # This is almost always multi-line, print it separately
+    echo "export CIRRUS_CHANGE_MESSAGE=$CIRRUS_CHANGE_MESSAGE"
     echo "
 BUILDTAGS $BUILDTAGS
+BUILT_IMAGE_SUFFIX $BUILT_IMAGE_SUFFIX
 CI $CI
 CIRRUS_CI $CIRRUS_CI
 CI_NODE_INDEX $CI_NODE_INDEX
@@ -47,7 +50,6 @@ CIRRUS_BASE_SHA $CIRRUS_BASE_SHA
 CIRRUS_BRANCH $CIRRUS_BRANCH
 CIRRUS_BUILD_ID $CIRRUS_BUILD_ID
 CIRRUS_CHANGE_IN_REPO $CIRRUS_CHANGE_IN_REPO
-CIRRUS_CHANGE_MESSAGE $CIRRUS_CHANGE_MESSAGE
 CIRRUS_CLONE_DEPTH $CIRRUS_CLONE_DEPTH
 CIRRUS_DEFAULT_BRANCH $CIRRUS_DEFAULT_BRANCH
 CIRRUS_PR $CIRRUS_PR
@@ -66,6 +68,7 @@ CIRRUS_USER_PERMISSION $CIRRUS_USER_PERMISSION
 CIRRUS_WORKING_DIR $CIRRUS_WORKING_DIR
 CIRRUS_HTTP_CACHE_HOST $CIRRUS_HTTP_CACHE_HOST
 $(go env)
+PACKER_BUILDS $PACKER_BUILDS
     " | while read NAME VALUE
     do
         [[ -z "$NAME" ]] || echo "export $NAME=\"$VALUE\""
@@ -199,6 +202,21 @@ install_conmon(){
     sudo install -D -m 755 bin/conmon /usr/libexec/podman/conmon
 }
 
+install_criu(){
+    echo "Installing CRIU from commit $CRIU_COMMIT"
+    req_env_var "
+    CRIU_COMMIT $CRIU_COMMIT
+    "
+    DEST="/tmp/criu"
+    rm -rf "$DEST"
+    ooe.sh git clone https://github.com/checkpoint-restore/criu.git "$DEST"
+    cd $DEST
+    ooe.sh git fetch origin --tags
+    ooe.sh git checkout -q "$CRIU_COMMIT"
+    ooe.sh make
+    sudo install -D -m 755  criu/criu /usr/sbin/
+}
+
 # Runs in testing VM, not image building
 install_testing_dependencies() {
     echo "Installing ginkgo, gomega, and easyjson into \$GOPATH=$GOPATH"
diff --git a/contrib/cirrus/packer/centos_setup.sh b/contrib/cirrus/packer/centos_setup.sh
index 2253d7b35..7b2308739 100644
--- a/contrib/cirrus/packer/centos_setup.sh
+++ b/contrib/cirrus/packer/centos_setup.sh
@@ -12,6 +12,7 @@ req_env_var "
 SCRIPT_BASE $SCRIPT_BASE
 CNI_COMMIT $CNI_COMMIT
 CRIO_COMMIT $CRIO_COMMIT
+CRIU_COMMIT $CRIU_COMMIT
 "
 
 install_ooe
@@ -24,35 +25,46 @@ ooe.sh sudo yum -y update
 ooe.sh sudo yum -y install centos-release-scl epel-release
 
 ooe.sh sudo yum -y install \
-          atomic-registries \
-          btrfs-progs-devel \
-          bzip2 \
-          device-mapper-devel \
-          findutils \
-          glib2-devel \
-          glibc-static \
-          gnupg \
-          golang \
-          golang-github-cpuguy83-go-md2man \
-          golang-github-cpuguy83-go-md2man \
-          gpgme-devel \
-          iptables \
-          libassuan-devel \
-          libseccomp-devel \
-          libselinux-devel \
-          lsof \
-          make \
-          nmap-ncat \
-          ostree-devel \
-          python \
-          python3-dateutil \
-          python3-psutil \
-          python3-pytoml \
-          runc \
-          skopeo-containers \
-          unzip \
-          which \
-          xz
+    atomic-registries \
+    btrfs-progs-devel \
+    bzip2 \
+    device-mapper-devel \
+    findutils \
+    glib2-devel \
+    glibc-static \
+    gnupg \
+    golang \
+    golang-github-cpuguy83-go-md2man \
+    golang-github-cpuguy83-go-md2man \
+    gpgme-devel \
+    iptables \
+    libassuan-devel \
+    libcap-devel \
+    libnet \
+    libnet-devel \
+    libnl3-devel \
+    libseccomp-devel \
+    libselinux-devel \
+    lsof \
+    make \
+    nmap-ncat \
+    ostree-devel \
+    protobuf \
+    protobuf-c \
+    protobuf-c-devel \
+    protobuf-compiler \
+    protobuf-devel \
+    protobuf-python \
+    python \
+    python2-future \
+    python3-dateutil \
+    python3-psutil \
+    python3-pytoml \
+    runc \
+    skopeo-containers \
+    unzip \
+    which \
+    xz
 
 install_scl_git
 
@@ -62,6 +74,8 @@ install_buildah
 
 install_conmon
 
+install_criu
+
 install_packer_copied_files
 
 rh_finalize
diff --git a/contrib/cirrus/packer/fedora_setup.sh b/contrib/cirrus/packer/fedora_setup.sh
index 53709fbdd..16b6e4e6b 100644
--- a/contrib/cirrus/packer/fedora_setup.sh
+++ b/contrib/cirrus/packer/fedora_setup.sh
@@ -12,6 +12,7 @@ req_env_var "
 SCRIPT_BASE $SCRIPT_BASE
 CNI_COMMIT $CNI_COMMIT
 CRIO_COMMIT $CRIO_COMMIT
+CRIU_COMMIT $CRIU_COMMIT
 RUNC_COMMIT $RUNC_COMMIT
 "
 
@@ -40,6 +41,10 @@ ooe.sh sudo dnf install -y \
     gpgme-devel \
     iptables \
     libassuan-devel \
+    libcap-devel \
+    libnet \
+    libnet-devel \
+    libnl3-devel \
     libseccomp-devel \
     libselinux-devel \
     lsof \
@@ -47,14 +52,21 @@ ooe.sh sudo dnf install -y \
     nmap-ncat \
     ostree-devel \
     procps-ng \
+    protobuf \
+    protobuf-c \
+    protobuf-c-devel \
+    protobuf-compiler \
+    protobuf-devel \
+    protobuf-python \
     python \
+    python2-future \
     python3-dateutil \
     python3-psutil \
     python3-pytoml \
     runc \
     skopeo-containers \
     slirp4netns \
-    which\
+    which \
     xz
 
 install_varlink
@@ -65,6 +77,8 @@ install_buildah
 
 install_conmon
 
+install_criu
+
 install_packer_copied_files
 
 rh_finalize # N/B: Halts system!
diff --git a/contrib/cirrus/packer/libpod_images.json b/contrib/cirrus/packer/libpod_images.json
index 82a41ca25..9dac3e8ea 100644
--- a/contrib/cirrus/packer/libpod_images.json
+++ b/contrib/cirrus/packer/libpod_images.json
@@ -1,7 +1,9 @@
 {
   "variables": {
+    "FEDORA_CNI_COMMIT": "{{env `FEDORA_CNI_COMMIT`}}",
     "CNI_COMMIT": "{{env `CNI_COMMIT`}}",
     "CRIO_COMMIT": "{{env `CRIO_COMMIT`}}",
+    "CRIU_COMMIT": "{{env `CRIU_COMMIT`}}",
     "RUNC_COMMIT": "{{env `RUNC_COMMIT`}}",
 
     "CENTOS_BASE_IMAGE": "{{env `CENTOS_BASE_IMAGE`}}" ,
@@ -15,7 +17,7 @@
 
     "SERVICE_ACCOUNT": "{{env `SERVICE_ACCOUNT`}}",
     "GCP_PROJECT_ID": "{{env `GCP_PROJECT_ID`}}",
-    "CIRRUS_BUILD_ID": "{{env `CIRRUS_BUILD_ID`}}",
+    "BUILT_IMAGE_SUFFIX": "{{env `BUILT_IMAGE_SUFFIX`}}",
     "GCE_SSH_USERNAME": "{{env `GCE_SSH_USERNAME`}}",
     "RHSM_COMMAND": "{{env `RHSM_COMMAND`}}"
   },
@@ -29,7 +31,7 @@
       "project_id": "{{user `GCP_PROJECT_ID`}}",
       "zone": "us-central1-a",
       "source_image": "{{user `RHEL_BASE_IMAGE`}}",
-      "image_name": "{{user `RHEL_BASE_IMAGE`}}-libpod-{{user `CIRRUS_BUILD_ID`}}",
+      "image_name": "{{user `RHEL_BASE_IMAGE`}}{{user `BUILT_IMAGE_SUFFIX`}}",
       "image_family": "{{user `RHEL_BASE_IMAGE`}}-libpod",
       "service_account_email": "{{user `SERVICE_ACCOUNT`}}",
       "communicator": "ssh",
@@ -41,7 +43,7 @@
       "project_id": "{{user `GCP_PROJECT_ID`}}",
       "zone": "us-central1-a",
       "source_image": "{{user `CENTOS_BASE_IMAGE`}}",
-      "image_name": "{{user `CENTOS_BASE_IMAGE`}}-libpod-{{user `CIRRUS_BUILD_ID`}}",
+      "image_name": "{{user `CENTOS_BASE_IMAGE`}}{{user `BUILT_IMAGE_SUFFIX`}}",
       "image_family": "{{user `CENTOS_BASE_IMAGE`}}-libpod",
       "service_account_email": "{{user `SERVICE_ACCOUNT`}}",
       "communicator": "ssh",
@@ -53,7 +55,7 @@
       "project_id": "{{user `GCP_PROJECT_ID`}}",
       "zone": "us-central1-a",
       "source_image": "{{user `FEDORA_BASE_IMAGE`}}",
-      "image_name": "{{user `FEDORA_BASE_IMAGE`}}-libpod-{{user `CIRRUS_BUILD_ID`}}",
+      "image_name": "{{user `FEDORA_BASE_IMAGE`}}{{user `BUILT_IMAGE_SUFFIX`}}",
       "image_family": "{{user `FEDORA_BASE_IMAGE`}}-libpod",
       "service_account_email": "{{user `SERVICE_ACCOUNT`}}",
       "communicator": "ssh",
@@ -65,7 +67,7 @@
       "project_id": "{{user `GCP_PROJECT_ID`}}",
       "zone": "us-central1-a",
       "source_image": "{{user `UBUNTU_BASE_IMAGE`}}",
-      "image_name": "{{user `UBUNTU_BASE_IMAGE`}}-libpod-{{user `CIRRUS_BUILD_ID`}}",
+      "image_name": "{{user `UBUNTU_BASE_IMAGE`}}{{user `BUILT_IMAGE_SUFFIX`}}",
       "image_family": "{{user `UBUNTU_BASE_IMAGE`}}-libpod",
       "service_account_email": "{{user `SERVICE_ACCOUNT`}}",
       "communicator": "ssh",
@@ -86,6 +88,7 @@
         "SCRIPT_BASE={{user `SCRIPT_BASE`}}",
         "CNI_COMMIT={{user `CNI_COMMIT`}}",
         "CRIO_COMMIT={{user `CRIO_COMMIT`}}",
+        "CRIU_COMMIT={{user `CRIU_COMMIT`}}",
         "RUNC_COMMIT={{user `RUNC_COMMIT`}}",
         "RHSM_COMMAND={{user `RHSM_COMMAND`}}"
       ]
@@ -97,6 +100,7 @@
         "SCRIPT_BASE={{user `SCRIPT_BASE`}}",
         "CNI_COMMIT={{user `CNI_COMMIT`}}",
         "CRIO_COMMIT={{user `CRIO_COMMIT`}}",
+        "CRIU_COMMIT={{user `CRIU_COMMIT`}}",
         "RUNC_COMMIT={{user `RUNC_COMMIT`}}"
       ]
     },{
@@ -105,8 +109,9 @@
       "script": "{{user `GOSRC`}}/{{user `PACKER_BASE`}}/fedora_setup.sh",
       "environment_vars": [
         "SCRIPT_BASE={{user `SCRIPT_BASE`}}",
-        "CNI_COMMIT={{user `CNI_COMMIT`}}",
+        "CNI_COMMIT={{user `FEDORA_CNI_COMMIT`}}",
         "CRIO_COMMIT={{user `CRIO_COMMIT`}}",
+        "CRIU_COMMIT={{user `CRIU_COMMIT`}}",
         "RUNC_COMMIT={{user `RUNC_COMMIT`}}"
       ]
     },{
@@ -117,6 +122,7 @@
         "SCRIPT_BASE={{user `SCRIPT_BASE`}}",
         "CNI_COMMIT={{user `CNI_COMMIT`}}",
         "CRIO_COMMIT={{user `CRIO_COMMIT`}}",
+        "CRIU_COMMIT={{user `CRIU_COMMIT`}}",
         "RUNC_COMMIT={{user `RUNC_COMMIT`}}"
       ]
     }
diff --git a/contrib/cirrus/packer/rhel_setup.sh b/contrib/cirrus/packer/rhel_setup.sh
index b776a0d97..d296713fc 100644
--- a/contrib/cirrus/packer/rhel_setup.sh
+++ b/contrib/cirrus/packer/rhel_setup.sh
@@ -12,6 +12,7 @@ req_env_var "
 SCRIPT_BASE $SCRIPT_BASE
 CNI_COMMIT $CNI_COMMIT
 CRIO_COMMIT $CRIO_COMMIT
+CRIU_COMMIT $CRIU_COMMIT
 RHSM_COMMAND $RHSM_COMMAND
 "
 
@@ -62,37 +63,48 @@ gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg
 EOM
 
 ooe.sh sudo yum -y install \
-          atomic-registries \
-          btrfs-progs-devel \
-          bzip2 \
-          device-mapper-devel \
-          findutils \
-          glib2-devel \
-          glibc-static \
-          gnupg \
-          golang \
-          golang-github-cpuguy83-go-md2man \
-          golang-github-cpuguy83-go-md2man \
-          google-compute-engine \
-          google-compute-engine-oslogin \
-          gpgme-devel \
-          iptables \
-          libassuan-devel \
-          libseccomp-devel \
-          libselinux-devel \
-          lsof \
-          make \
-          nmap-ncat \
-          ostree-devel \
-          python \
-          python34-dateutil \
-          python34-psutil \
-          python34-pytoml \
-          runc \
-          skopeo-containers \
-          unzip \
-          which \
-          xz
+    atomic-registries \
+    btrfs-progs-devel \
+    bzip2 \
+    device-mapper-devel \
+    findutils \
+    glib2-devel \
+    glibc-static \
+    gnupg \
+    golang \
+    golang-github-cpuguy83-go-md2man \
+    golang-github-cpuguy83-go-md2man \
+    google-compute-engine \
+    google-compute-engine-oslogin \
+    gpgme-devel \
+    iptables \
+    libassuan-devel \
+    libcap-devel \
+    libnet \
+    libnet-devel \
+    libnl3-devel \
+    libseccomp-devel \
+    libselinux-devel \
+    lsof \
+    make \
+    nmap-ncat \
+    ostree-devel \
+    protobuf \
+    protobuf-c \
+    protobuf-c-devel \
+    protobuf-compiler \
+    protobuf-devel \
+    protobuf-python \
+    python \
+    python2-future \
+    python34-dateutil \
+    python34-psutil \
+    python34-pytoml \
+    runc \
+    skopeo-containers \
+    unzip \
+    which \
+    xz
 
 install_scl_git
 
@@ -102,6 +114,8 @@ install_buildah
 
 install_conmon
 
+install_criu
+
 install_packer_copied_files
 
 exit_handler  # release subscription!
diff --git a/contrib/cirrus/packer/ubuntu_setup.sh b/contrib/cirrus/packer/ubuntu_setup.sh
index 96b3a573f..ff20944dc 100644
--- a/contrib/cirrus/packer/ubuntu_setup.sh
+++ b/contrib/cirrus/packer/ubuntu_setup.sh
@@ -12,6 +12,7 @@ req_env_var "
 SCRIPT_BASE $SCRIPT_BASE
 CNI_COMMIT $CNI_COMMIT
 CRIO_COMMIT $CRIO_COMMIT
+CRIU_COMMIT $CRIU_COMMIT
 RUNC_COMMIT $RUNC_COMMIT
 "
 
@@ -34,8 +35,8 @@ ooe.sh sudo apt-get -qq install --no-install-recommends \
     e2fslibs-dev \
     gawk \
     gettext \
-    golang \
     go-md2man \
+    golang \
     iptables \
     libaio-dev \
     libapparmor-dev \
@@ -46,18 +47,22 @@ ooe.sh sudo apt-get -qq install --no-install-recommends \
     libglib2.0-dev \
     libgpgme11-dev \
     liblzma-dev \
+    libnet1 \
+    libnet1-dev \
+    libnl-3-dev \
     libostree-dev \
     libprotobuf-c0-dev \
     libprotobuf-dev \
     libtool \
-    libtool \
     libudev-dev \
     lsof \
     netcat \
     pkg-config \
     protobuf-c-compiler \
     protobuf-compiler \
+    python-future \
     python-minimal \
+    python-protobuf \
     python3-dateutil \
     python3-pip \
     python3-psutil \
@@ -77,6 +82,8 @@ install_runc
 
 install_conmon
 
+install_criu
+
 install_cni_plugins
 
 install_buildah
diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh
index 2302f0e15..167db127f 100755
--- a/contrib/cirrus/setup_environment.sh
+++ b/contrib/cirrus/setup_environment.sh
@@ -41,6 +41,7 @@ then
         "export OS_RELEASE_ID=\"$(os_release_id)\"" \
         "export OS_RELEASE_VER=\"$(os_release_ver)\"" \
         "export OS_REL_VER=\"${OS_RELEASE_ID}-${OS_RELEASE_VER}\"" \
+        "export BUILT_IMAGE_SUFFIX=\"-$CIRRUS_REPO_NAME-${CIRRUS_CHANGE_IN_REPO:0:8}\"" \
         "export GOPATH=\"/go\"" \
         'export PATH="$HOME/bin:$GOPATH/bin:/usr/local/bin:$PATH"' \
         'export LD_LIBRARY_PATH="/usr/local/lib${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}"'