summaryrefslogtreecommitdiff
path: root/contrib/cirrus/lib.sh
blob: d50c1e1a35f179f62674365b260d02fd9e93a098 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
# Library of common, shared utility functions.  This file is intended
# to be sourced by other scripts, not called directly.

# Under some contexts these values are not set, make sure they are.
export USER="$(whoami)"
export HOME="$(getent passwd $USER | cut -d : -f 6)"

# These are normally set by cirrus, but can't be for VMs setup by hack/get_ci_vm.sh
# Pick some reasonable defaults
ENVLIB=${ENVLIB:-.bash_profile}
CIRRUS_WORKING_DIR="${CIRRUS_WORKING_DIR:-/var/tmp/go/src/github.com/containers/libpod}"
GOSRC="${GOSRC:-$CIRRUS_WORKING_DIR}"
SCRIPT_BASE=${SCRIPT_BASE:-./contrib/cirrus}
PACKER_BASE=${PACKER_BASE:-./contrib/cirrus/packer}
CIRRUS_BUILD_ID=${CIRRUS_BUILD_ID:-DEADBEEF}  # a human
CIRRUS_BASE_SHA=${CIRRUS_BASE_SHA:-HEAD}
CIRRUS_CHANGE_IN_REPO=${CIRRUS_CHANGE_IN_REPO:-FETCH_HEAD}
START_STAMP_FILEPATH="${START_STAMP_FILEPATH:-/var/tmp/start.timestamp}"

if ! [[ "$PATH" =~ "/usr/local/bin" ]]
then
    export PATH="$PATH:/usr/local/bin"
fi

# In ci/testing environment, ensure variables are always loaded
if [[ -r "$HOME/$ENVLIB" ]] && [[ -n "$CI" ]]
then
    # Make sure this is always loaded
    source "$HOME/$ENVLIB"
fi

# Pass in a line delimited list of, space delimited name/value pairs
# exit non-zero with helpful error message if any value is empty
req_env_var() {
    echo "$1" | while read NAME VALUE
    do
        if [[ -n "$NAME" ]] && [[ -z "$VALUE" ]]
        then
            echo "Required env. var. \$$NAME is not set"
            exit 9
        fi
    done
}

# Some env. vars may contain secrets.  Display values for known "safe"
# and useful variables.
# ref: https://cirrus-ci.org/guide/writing-tasks/#environment-variables
show_env_vars() {
    # This is almost always multi-line, print it separately
    echo "export CIRRUS_CHANGE_MESSAGE=$CIRRUS_CHANGE_MESSAGE"
    echo "
BUILDTAGS $BUILDTAGS
BUILT_IMAGE_SUFFIX $BUILT_IMAGE_SUFFIX
CI $CI
CIRRUS_CI $CIRRUS_CI
CI_NODE_INDEX $CI_NODE_INDEX
CI_NODE_TOTAL $CI_NODE_TOTAL
CONTINUOUS_INTEGRATION $CONTINUOUS_INTEGRATION
CIRRUS_BASE_BRANCH $CIRRUS_BASE_BRANCH
CIRRUS_BASE_SHA $CIRRUS_BASE_SHA
CIRRUS_BRANCH $CIRRUS_BRANCH
CIRRUS_BUILD_ID $CIRRUS_BUILD_ID
CIRRUS_CHANGE_IN_REPO $CIRRUS_CHANGE_IN_REPO
CIRRUS_CLONE_DEPTH $CIRRUS_CLONE_DEPTH
CIRRUS_DEFAULT_BRANCH $CIRRUS_DEFAULT_BRANCH
CIRRUS_PR $CIRRUS_PR
CIRRUS_TAG $CIRRUS_TAG
CIRRUS_OS $CIRRUS_OS
OS $OS
CIRRUS_TASK_NAME $CIRRUS_TASK_NAME
CIRRUS_TASK_ID $CIRRUS_TASK_ID
CIRRUS_REPO_NAME $CIRRUS_REPO_NAME
CIRRUS_REPO_OWNER $CIRRUS_REPO_OWNER
CIRRUS_REPO_FULL_NAME $CIRRUS_REPO_FULL_NAME
CIRRUS_REPO_CLONE_URL $CIRRUS_REPO_CLONE_URL
CIRRUS_SHELL $CIRRUS_SHELL
CIRRUS_USER_COLLABORATOR $CIRRUS_USER_COLLABORATOR
CIRRUS_USER_PERMISSION $CIRRUS_USER_PERMISSION
CIRRUS_WORKING_DIR $CIRRUS_WORKING_DIR
CIRRUS_HTTP_CACHE_HOST $CIRRUS_HTTP_CACHE_HOST
$(go env)
PACKER_BUILDS $PACKER_BUILDS
    " | while read NAME VALUE
    do
        [[ -z "$NAME" ]] || echo "export $NAME=\"$VALUE\""
    done
    echo ""
    echo "##### $(go version) #####"
    echo ""
}

# Unset environment variables not needed for testing purposes
clean_env() {
    req_env_var "
        UNSET_ENV_VARS $UNSET_ENV_VARS
    "
    echo "Unsetting $(echo $UNSET_ENV_VARS | wc -w) environment variables"
    unset -v UNSET_ENV_VARS $UNSET_ENV_VARS || true  # don't fail on read-only
}

# Return a GCE image-name compatible string representation of distribution name
os_release_id() {
    eval "$(egrep -m 1 '^ID=' /etc/os-release | tr -d \' | tr -d \")"
    echo "$ID"
}

# Return a GCE image-name compatible string representation of distribution major version
os_release_ver() {
    eval "$(egrep -m 1 '^VERSION_ID=' /etc/os-release | tr -d \' | tr -d \")"
    echo "$VERSION_ID" | cut -d '.' -f 1
}

bad_os_id_ver() {
    echo "Unknown/Unsupported distro. $OS_RELEASE_ID and/or version $OS_RELEASE_VER for $ARGS"
    exit 42
}

stub() {
    echo "STUB: Pretending to do $1"
}

ircmsg() {
    req_env_var "
        CIRRUS_TASK_ID $CIRRUS_TASK_ID
        1 $1
    "
    # Sometimes setup_environment.sh didn't run
    SCRIPT="$(dirname $0)/podbot.py"
    NICK="podbot_$CIRRUS_TASK_ID"
    NICK="${NICK:0:15}"  # Any longer will break things
    set +e
    $SCRIPT $NICK $1
    echo "Ignoring exit($?)"
    set -e
}

start_timestamp() {
    req_env_var "START_STAMP_FILEPATH $START_STAMP_FILEPATH"
    [[ -r "$START_STAMP_FILEPATH" ]] || \
        echo -e ".\nThe time at the tone will be:\n$(date --iso-8601=seconds | \
            tee $START_STAMP_FILEPATH)\nBLEEEEEEEEEEP!\n.\n"  # Cirrus strips blank lines from output
}

# Run sudo in directory with GOPATH set
cdsudo() {
    DIR="$1"
    shift
    CMD="cd $DIR && $@"
    sudo --preserve-env=GOPATH --non-interactive bash -c "$CMD"
}

# Helper/wrapper script to only show stderr/stdout on non-zero exit
install_ooe() {
    req_env_var "SCRIPT_BASE $SCRIPT_BASE"
    echo "Installing script to mask stdout/stderr unless non-zero exit."
    sudo install -D -m 755 "/tmp/libpod/$SCRIPT_BASE/ooe.sh" /usr/local/bin/ooe.sh
}

# Grab a newer version of git from software collections
# https://www.softwarecollections.org/en/
# and use it with a wrapper
install_scl_git() {
    echo "Installing SoftwareCollections updated 'git' version."
    ooe.sh sudo yum -y install rh-git29
    cat << "EOF" | sudo tee /usr/bin/git
#!/bin/bash

scl enable rh-git29 -- git $@
EOF
    sudo chmod 755 /usr/bin/git
}

install_cni_plugins() {
    echo "Installing CNI Plugins from commit $CNI_COMMIT"
    req_env_var "
        GOPATH $GOPATH
        CNI_COMMIT $CNI_COMMIT
    "
    DEST="$GOPATH/src/github.com/containernetworking/plugins"
    rm -rf "$DEST"
    ooe.sh git clone "https://github.com/containernetworking/plugins.git" "$DEST"
    cd "$DEST"
    ooe.sh git checkout -q "$CNI_COMMIT"
    ooe.sh ./build.sh
    sudo mkdir -p /usr/libexec/cni
    sudo cp bin/* /usr/libexec/cni
}

install_runc_from_git(){
    wd=$(pwd)
    DEST="$GOPATH/src/github.com/opencontainers/runc"
    rm -rf "$DEST"
    ooe.sh git clone https://github.com/opencontainers/runc.git "$DEST"
    cd "$DEST"
    ooe.sh git fetch origin --tags
    ooe.sh git checkout -q "$RUNC_COMMIT"
    ooe.sh make static BUILDTAGS="seccomp selinux"
    sudo install -m 755 runc /usr/bin/runc
    cd $wd
}

install_runc(){
    OS_RELEASE_ID=$(os_release_id)
    echo "Installing RunC from commit $RUNC_COMMIT"
    echo "Platform is $OS_RELEASE_ID"
    req_env_var "
        GOPATH $GOPATH
        RUNC_COMMIT $RUNC_COMMIT
        OS_RELEASE_ID $OS_RELEASE_ID
    "
    if [[ "$OS_RELEASE_ID" =~ "ubuntu" ]]; then
        echo "Running make install.libseccomp.sudo for ubuntu"
        if ! [[ -d "/tmp/libpod" ]]
        then
            echo "Expecting a copy of libpod repository in /tmp/libpod"
            exit 5
        fi
        mkdir -p "$GOPATH/src/github.com/containers/"
        # Symlinks don't work with Go
        cp -a /tmp/libpod "$GOPATH/src/github.com/containers/"
        cd "$GOPATH/src/github.com/containers/libpod"
        ooe.sh sudo make install.libseccomp.sudo
    fi
    install_runc_from_git
}

install_buildah() {
    echo "Installing buildah from latest upstream master"
    req_env_var "GOPATH $GOPATH"
    DEST="$GOPATH/src/github.com/containers/buildah"
    rm -rf "$DEST"
    ooe.sh git clone https://github.com/containers/buildah "$DEST"
    cd "$DEST"
    ooe.sh make
    ooe.sh sudo make install
}

# Requires $GOPATH and $CRIO_COMMIT to be set
install_conmon(){
    echo "Installing conmon from commit $CRIO_COMMIT"
    req_env_var "
        GOPATH $GOPATH
        CRIO_COMMIT $CRIO_COMMIT
    "
    DEST="$GOPATH/src/github.com/kubernetes-sigs/cri-o.git"
    rm -rf "$DEST"
    ooe.sh git clone https://github.com/kubernetes-sigs/cri-o.git "$DEST"
    cd "$DEST"
    ooe.sh git fetch origin --tags
    ooe.sh git checkout -q "$CRIO_COMMIT"
    ooe.sh make
    sudo install -D -m 755 bin/conmon /usr/libexec/podman/conmon
}

install_criu(){
    OS_RELEASE_ID=$(os_release_id)
    OS_RELEASE_VER=$(os_release_ver)
    echo "Installing CRIU"
    echo "Installing CRIU from commit $CRIU_COMMIT"
    echo "Platform is $OS_RELEASE_ID"
    req_env_var "
    CRIU_COMMIT $CRIU_COMMIT
    "

    if [[ "$OS_RELEASE_ID" =~ "ubuntu" ]]; then
        ooe.sh sudo -E add-apt-repository -y ppa:criu/ppa
        ooe.sh sudo -E apt-get -qq -y update
        ooe.sh sudo -E apt-get -qq -y install criu
    elif [[ ( "$OS_RELEASE_ID" =~ "centos" || "$OS_RELEASE_ID" =~ "rhel" ) && "$OS_RELEASE_VER" =~ "7"* ]]; then
        echo "Configuring Repositories for latest CRIU"
        ooe.sh sudo tee /etc/yum.repos.d/adrian-criu-el7.repo <<EOF
[adrian-criu-el7]
name=Copr repo for criu-el7 owned by adrian
baseurl=https://copr-be.cloud.fedoraproject.org/results/adrian/criu-el7/epel-7-$basearch/
type=rpm-md
skip_if_unavailable=True
gpgcheck=1
gpgkey=https://copr-be.cloud.fedoraproject.org/results/adrian/criu-el7/pubkey.gpg
repo_gpgcheck=0
enabled=1
enabled_metadata=1
EOF
        ooe.sh sudo yum -y install criu
    elif [[ "$OS_RELEASE_ID" =~ "fedora" ]]; then
        echo "Using CRIU from distribution"
    else
        DEST="/tmp/criu"
        rm -rf "$DEST"
        ooe.sh git clone https://github.com/checkpoint-restore/criu.git "$DEST"
        cd $DEST
        ooe.sh git fetch origin --tags
        ooe.sh git checkout -q "$CRIU_COMMIT"
        ooe.sh make
        sudo install -D -m 755  criu/criu /usr/sbin/
    fi
}

# Runs in testing VM, not image building
install_testing_dependencies() {
    echo "Installing ginkgo, gomega, and easyjson into \$GOPATH=$GOPATH"
    req_env_var "
        GOPATH $GOPATH
        GOSRC $GOSRC
    "
    cd "$GOSRC"
    ooe.sh go get -u github.com/onsi/ginkgo/ginkgo
    ooe.sh install -D -m 755 "$GOPATH"/bin/ginkgo /usr/bin/
    ooe.sh go get github.com/onsi/gomega/...
    ooe.sh go get -u github.com/mailru/easyjson/...
    sudo install -D -m 755 "$GOPATH"/bin/easyjson /usr/bin/
}

install_packer_copied_files(){
    # Install cni config, policy and registry config
    sudo install -D -m 755 /tmp/libpod/cni/87-podman-bridge.conflist \
                           /etc/cni/net.d/87-podman-bridge.conflist
    sudo install -D -m 755 /tmp/libpod/test/policy.json \
                           /etc/containers/policy.json
    sudo install -D -m 755 /tmp/libpod/test/redhat_sigstore.yaml \
                           /etc/containers/registries.d/registry.access.redhat.com.yaml
}

install_varlink(){
    echo "Installing varlink from the cheese-factory"
    ooe.sh sudo -H pip3 install varlink
}

_finalize(){
    set +e  # Don't fail at the very end
    set +e  # make errors non-fatal
    echo "Removing leftover giblets from cloud-init"
    cd /
    sudo rm -rf /var/lib/cloud/instance?
    sudo rm -rf /root/.ssh/*
    sudo rm -rf /home/*
    sudo rm -rf /tmp/*
    sudo rm -rf /tmp/.??*
    sync
    sudo fstrim -av
}

rh_finalize(){
    set +e  # Don't fail at the very end
    # Allow root ssh-logins
    if [[ -r /etc/cloud/cloud.cfg ]]
    then
        sudo sed -re 's/^disable_root:.*/disable_root: 0/g' -i /etc/cloud/cloud.cfg
    fi
    echo "Resetting to fresh-state for usage as cloud-image."
    PKG=$(type -P dnf || type -P yum || echo "")
    [[ -z "$PKG" ]] || sudo $PKG clean all  # not on atomic
    sudo rm -rf /var/cache/{yum,dnf}
    sudo rm -f /etc/udev/rules.d/*-persistent-*.rules
    sudo touch /.unconfigured  # force firstboot to run
    _finalize
}

ubuntu_finalize(){
    set +e  # Don't fail at the very end
    echo "Resetting to fresh-state for usage as cloud-image."
    sudo rm -rf /var/cache/apt
    _finalize
}

rhel_exit_handler() {
    set +ex
    req_env_var "
        GOPATH $GOPATH
        RHSMCMD $RHSMCMD
    "
    cd /
    sudo rm -rf "$RHSMCMD"
    sudo rm -rf "$GOPATH"
    sudo subscription-manager remove --all
    sudo subscription-manager unregister
    sudo subscription-manager clean
}

rhsm_enable() {
    req_env_var "
        RHSM_COMMAND $RHSM_COMMAND
    "
    export GOPATH="$(mktemp -d)"
    export RHSMCMD="$(mktemp)"
    trap "rhel_exit_handler" EXIT
    # Avoid logging sensitive details
    echo "$RHSM_COMMAND" > "$RHSMCMD"
    ooe.sh sudo bash "$RHSMCMD"
    sudo rm -rf "$RHSMCMD"
}