diff options
Diffstat (limited to 'hack')
| -rwxr-xr-x | hack/get_ci_vm.sh | 316 | ||||
| -rwxr-xr-x | hack/swagger-check | 101 |
2 files changed, 128 insertions, 289 deletions
diff --git a/hack/get_ci_vm.sh b/hack/get_ci_vm.sh index 4f6c42a06..1a4804857 100755 --- a/hack/get_ci_vm.sh +++ b/hack/get_ci_vm.sh @@ -3,270 +3,68 @@ # # For help and usage information, simply execute the script w/o any arguments. # -# This script is intended to be run by podman developers who need to debug -# problems specifically related to Cirrus-CI automated testing. However, -# because it's only loosely coupled to the `.cirrus.yml` configuration, it must -# orchestrate VMs in GCP directly. This means users need to have -# pre-authorization (access) to manipulate google-cloud resources. Additionally, -# there are no guarantees it will remain in-sync with other automation-related -# scripts. Therefore it may not always function for everybody in every -# future scenario without updates/modifications/tweaks. -# -# When successful, you will end up connected to a GCP VM with with a clone of -# the upstream podman repository 'master' branch, using a remote named 'origin'. -# If you want to customize this behavior, you will want to use a "hook" script. -# Please use this example carefully, since git setups vary by person, you -# will probably need to make local edits. -# -# https://gist.github.com/cevich/626a0790c0b476d5cd2a5a76fbdae0a1 +# This script is intended to be run by Red Hat podman developers who need +# to debug problems specifically related to Cirrus-CI automated testing. +# It requires that you have been granted prior access to create VMs in +# google-cloud. For non-Red Hat contributors, VMs are available as-needed, +# with supervision upon request. set -e -RED="\e[1;31m" -YEL="\e[1;32m" -NOR="\e[0m" -USAGE_WARNING=" -${YEL}WARNING: This will not work without podman,${NOR} - ${YEL}and prior authorization to use the libpod GCP project.${NOR} -" -# These values come from .cirrus.yml gce_instance clause -ZONE="${ZONE:-us-central1-a}" -CPUS="2" -MEMORY="4Gb" -DISK="200" -PROJECT="libpod-218412" -GOSRC="/var/tmp/go/src/github.com/containers/podman" -GIT_REPO="https://github.com/containers/podman.git" - -# Container image with necessary runtime elements -GCLOUD_IMAGE="${GCLOUD_IMAGE:-docker.io/google/cloud-sdk:alpine}" -GCLOUD_CFGDIR=".config/gcloud" - -SCRIPT_FILENAME=$(basename ${BASH_SOURCE[0]}) -HOOK_FILENAME="hook_${SCRIPT_FILENAME}" +SCRIPT_FILEPATH=$(realpath "${BASH_SOURCE[0]}") +SCRIPT_DIRPATH=$(dirname "$SCRIPT_FILEPATH") +REPO_DIRPATH=$(realpath "$SCRIPT_DIRPATH/../") -# Shared tmp directory between container and us -TMPDIR=$(mktemp -d --tmpdir ${SCRIPT_FILENAME}_tmpdir_XXXXXX) - -show_usage() { - echo -e "\n${RED}ERROR: $1${NOR}" - echo -e "${YEL}Usage: $SCRIPT_FILENAME <image_name>${NOR}" - echo "" - if [[ -r ".cirrus.yml" ]] - then - echo -e "${YEL}Some possible image_name values (from .cirrus.yml):${NOR}" - image_hints - echo "" - echo -e "${YEL}Optional:${NOR} If a $HOME/$GCLOUD_CFGDIR/$HOOK_FILENAME executable exists during" - echo "VM creation, it will be executed remotely after cloning" - echo "$GIT_REPO. The" - echo "current local working branch name and commit ID, will be provided as" - echo "it's arguments." +# Help detect if we were called by get_ci_vm container +GET_CI_VM="${GET_CI_VM:-0}" +in_get_ci_vm() { + if ((GET_CI_VM==0)); then + echo "Error: $1 is not intended for use in this context" + exit 2 fi - exit 1 -} - -LIBPODROOT=$(realpath "$(dirname ${BASH_SOURCE[0]})/../") -# else: Assume $PWD is the root of the libpod repository -[[ "$LIBPODROOT" != "/" ]] || \ - show_usage "Must execute script from within clone of containers/podman repo." - -[[ "$UID" -ne 0 ]] || \ - show_usage "Must execute script as a regular (non-root) user." - -[[ "${LIBPODROOT#$HOME}" != "$LIBPODROOT" ]] || \ - show_usage "Clone of containers/podman must be a subdirectory of \$HOME ($HOME)" - -# Disable SELinux labeling to allow read-only mounting of repository files -PGCLOUD="podman run -it --rm --security-opt label=disable -v $TMPDIR:$TMPDIR -v $HOME/.config/gcloud:/root/.config/gcloud -v $HOME/.config/gcloud/ssh:/root/.ssh -v $LIBPODROOT:$LIBPODROOT:ro $GCLOUD_IMAGE gcloud --configuration=libpod --project=$PROJECT" -SCP_CMD="$PGCLOUD compute scp" - -showrun() { - echo '+ '$(printf " %q" "$@") > /dev/stderr - echo "" - "$@" -} - -cleanup() { - RET=$? - set +e - wait - - # set GCLOUD_DEBUG to leave tmpdir behind for postmortem - # shellcheck disable=SC2154 - test -z "$GCLOUD_DEBUG" && rm -rf $TMPDIR - - # Not always called from an exit handler, but should always exit when called - exit $RET -} -trap cleanup EXIT - -delvm() { - echo -e "\n" - echo -e "\n${YEL}Offering to Delete $VMNAME${NOR}" - echo -e "${RED}(Deletion might take a minute or two)${NOR}" - echo -e "${YEL}Note: It's safe to answer N, then re-run script again later.${NOR}" - showrun $CLEANUP_CMD # prompts for Yes/No - cleanup -} - -get_env_vars() { - # Deal with both YAML and embedded shell-like substitutions in values - # if substitution fails, fall back to printing naked env. var as-is. - python3 -c ' -import sys,yaml,re -env=yaml.load(open(".cirrus.yml"), Loader=yaml.SafeLoader)["env"] -dollar_env_var=re.compile(r"\$(\w+)") -dollarcurly_env_var=re.compile(r"\$\{(\w+)\}") -class ReIterKey(dict): - def __missing__(self, key): - # Cirrus-CI provides some runtime-only env. vars. Avoid - # breaking this hack-script if/when any are present in YAML - return "${0}".format(key) -rep=r"{\1}" # Convert env vars markup to -> str.format_map(re_iter_key) markup -out=ReIterKey() -for k,v in env.items(): - if "ENCRYPTED" not in str(v) and bool(v): - out[k]=dollar_env_var.sub(rep, dollarcurly_env_var.sub(rep, str(v))) -for k,v in out.items(): - sys.stdout.write("{0}=\"{1}\"\n".format(k, str(v).format_map(out))) - ' -} - -image_hints() { - get_env_vars | fgrep '_CACHE_IMAGE_NAME' | awk -F "=" '{print $2}' } -unset VM_IMAGE_NAME -unset VMNAME -unset CREATE_CMD -unset SSH_CMD -unset CLEANUP_CMD -declare -xa ENVS -parse_args(){ - local arg - echo -e "$USAGE_WARNING" - - if [[ "$USER" =~ "root" ]] - then - show_usage "This script must be run as a regular user." - fi - - [[ "$#" -eq 1 ]] || \ - show_usage "Must specify a VM Image name to use, and the test flavor." - - VM_IMAGE_NAME="$1" - - # Word-splitting is desirable in this case. - # Values are used literally (with '=') as args to future `env` command. - # get_env_vars() will take care of properly quoting it's output. - # shellcheck disable=SC2207,SC2191 - ENVS=( - $(get_env_vars) - VM_IMAGE_NAME="$VM_IMAGE_NAME" - UPSTREAM_REMOTE="upstream" - ) - - VMNAME="${VMNAME:-${USER}-${VM_IMAGE_NAME}}" - - CREATE_CMD="$PGCLOUD compute instances create --zone=$ZONE --image=${VM_IMAGE_NAME} --custom-cpu=$CPUS --custom-memory=$MEMORY --boot-disk-size=$DISK --labels=in-use-by=$USER $VMNAME" - - SSH_CMD="$PGCLOUD compute ssh root@$VMNAME" - - CLEANUP_CMD="$PGCLOUD compute instances delete --zone $ZONE --delete-disks=all $VMNAME" -} - -# Returns true if user has run an 'init' and has a valid token for -# the specific project-id and named-configuration arguments in $PGCLOUD. -function has_valid_credentials() { - if $PGCLOUD info |& grep -Eq 'Account:.*None'; then - return 1 - fi - - # It's possible for 'gcloud info' to list expired credentials, - # e.g. 'ERROR: ... invalid grant: Bad Request' - if $PGCLOUD auth print-access-token |& grep -q 'ERROR'; then - return 1 - fi - - return 0 -} - -##### main - -[[ "${LIBPODROOT%%${LIBPODROOT##$HOME}}" == "$HOME" ]] || \ - show_usage "Repo clone must be sub-dir of $HOME" - -cd "$LIBPODROOT" - -parse_args "$@" -mkdir -p $TMPDIR/.ssh -mkdir -p {$HOME,$TMPDIR}/.config/gcloud/ssh -chmod 700 {$HOME,$TMPDIR}/.config/gcloud/ssh $TMPDIR/.ssh - -echo -e "\n${YEL}Pulling gcloud image...${NOR}" -podman pull $GCLOUD_IMAGE - -if ! has_valid_credentials -then - echo -e "\n${YEL}WARNING: Can't find gcloud configuration for libpod, running init.${NOR}" - echo -e " ${RED}Please choose \"#1: Re-initialize\" and \"login\" if asked.${NOR}" - showrun $PGCLOUD init --project=$PROJECT --console-only --skip-diagnostics - - # Verify it worked (account name == someone@example.com) - $PGCLOUD info > $TMPDIR/gcloud-info-after-init - if egrep -q "Account:.*None" $TMPDIR/gcloud-info-after-init - then - echo -e "${RED}ERROR: Could not initialize libpod configuration in gcloud.${NOR}" - exit 5 - fi - - # If this is the only config, make it the default to avoid - # persistent warnings from gcloud about there being no default. - [[ -r "$HOME/.config/gcloud/configurations/config_default" ]] || \ - ln "$HOME/.config/gcloud/configurations/config_libpod" \ - "$HOME/.config/gcloud/configurations/config_default" +# get_ci_vm APIv1 container entrypoint calls into this script +# to obtain required repo. specific configuration options. +if [[ "$1" == "--config" ]]; then + in_get_ci_vm "$1" + cat <<EOF +DESTDIR="/var/tmp/go/src/github.com/containers/podman" +UPSTREAM_REPO="https://github.com/containers/podman.git" +CI_ENVFILE="/etc/ci_environment" +GCLOUD_PROJECT="libpod-218412" +GCLOUD_IMGPROJECT="libpod-218412" +GCLOUD_CFG="libpod" +GCLOUD_ZONE="${GCLOUD_ZONE:-us-central1-a}" +GCLOUD_CPUS="2" +GCLOUD_MEMORY="4Gb" +GCLOUD_DISK="200" +EOF +elif [[ "$1" == "--setup" ]]; then + in_get_ci_vm "$1" + # get_ci_vm container entrypoint calls us with this option on the + # Cirrus-CI environment instance, to perform repo.-specific setup. + cd $REPO_DIRPATH + echo "+ Loading ./contrib/cirrus/lib.sh" > /dev/stderr + source ./contrib/cirrus/lib.sh + echo "+ Mimicking .cirrus.yml clone_script and build_task" > /dev/stderr + make install.tools + make vendor + make podman + make podman-remote + echo "+ Running environment setup" > /dev/stderr + ./contrib/cirrus/setup_environment.sh +else + # Create and access VM for specified Cirrus-CI task + mkdir -p $HOME/.config/gcloud/ssh + podman run -it --rm \ + --tz=local \ + -e NAME="$USER" \ + -e SRCDIR=/src \ + -e GCLOUD_ZONE="$GCLOUD_ZONE" \ + -e DEBUG="${DEBUG:-0}" \ + -v $REPO_DIRPATH:/src:O \ + -v $HOME/.config/gcloud:/root/.config/gcloud:z \ + -v $HOME/.config/gcloud/ssh:/root/.ssh:z \ + quay.io/libpod/get_ci_vm:latest "$@" fi - -trap delvm EXIT # Allow deleting VM if CTRL-C during create -echo -e "\n${YEL}Trying to creating a VM named $VMNAME${NOR}\n${YEL}in GCE region/zone $ZONE${NOR}" -echo -e "For faster terminal access, export ZONE='<something-closer>'" -echo -e 'Zone-list at: https://cloud.google.com/compute/docs/regions-zones/\n' -if showrun $CREATE_CMD; then # Freshly created VM needs initial setup - - echo -e "\n${YEL}Waiting up to 30s for ssh port to open${NOR}" - ATTEMPTS=10 - trap "exit 1" INT - while ((ATTEMPTS)) && ! $SSH_CMD --command "true"; do - let "ATTEMPTS--" - echo -e "${RED}Nope, not yet.${NOR}" - sleep 3s - done - trap - INT - if ! ((ATTEMPTS)); then - echo -e "\n${RED}Failed${NOR}" - exit 7 - fi - echo -e "${YEL}Got it. Cloning upstream repository as a starting point.${NOR}" - - showrun $SSH_CMD -- "mkdir -p $GOSRC" - showrun $SSH_CMD -- "git clone --progress $GIT_REPO $GOSRC" - - if [[ -x "$HOME/$GCLOUD_CFGDIR/$HOOK_FILENAME" ]]; then - echo -e "\n${YEL}Copying hook to VM and executing (ignoring errors).${NOR}" - $PGCLOUD compute scp "/root/$GCLOUD_CFGDIR/$HOOK_FILENAME" root@$VMNAME:. - if ! showrun $SSH_CMD -- "cd $GOSRC && bash /root/$HOOK_FILENAME $(git branch --show-current) $(git rev-parse HEAD)"; then - echo "-e ${RED}Hook exited: $?${NOR}" - fi - fi -fi - -echo -e "\n${YEL}Generating connection script for $VMNAME.${NOR}" -echo -e "Note: Script can be re-used in another terminal if needed." -echo -e "${RED}(option to delete VM presented upon exiting).${NOR}" -# TODO: This is fairly fragile, specifically the quoting for the remote command. -echo '#!/bin/bash' > $TMPDIR/ssh -echo "$SSH_CMD -- -t 'cd $GOSRC && exec env ${ENVS[*]} bash -il'" >> $TMPDIR/ssh -chmod +x $TMPDIR/ssh - -showrun $TMPDIR/ssh diff --git a/hack/swagger-check b/hack/swagger-check index f2e30cc33..646cbcb84 100755 --- a/hack/swagger-check +++ b/hack/swagger-check @@ -59,11 +59,6 @@ says 'libpod'. OPTIONS: - --pedantic Compare operation names (the last part of swagger comment). - There are far too many of these inconsistencies to allow us - to enable this by default, but it still might be a useful - check in some circumstances. - -v, --verbose show verbose progress indicators -n, --dry-run make no actual changes @@ -75,7 +70,6 @@ END_USAGE } # Command-line options. Note that this operates directly on @ARGV ! -our $pedantic; our $debug = 0; our $force = 0; our $verbose = 0; @@ -83,8 +77,6 @@ our $NOT = ''; # print "blahing the blah$NOT\n" if $debug sub handle_opts { use Getopt::Long; GetOptions( - 'pedantic' => \$pedantic, - 'debug!' => \$debug, 'dry-run|n!' => sub { $NOT = ' [NOT]' }, 'force' => \$force, @@ -225,26 +217,14 @@ sub handle_handle { my $tag = ($endpoint =~ /(libpod)/ ? $1 : 'compat'); # - # Determine the OPERATION. *** NOTE: This is mostly useless! *** - # In an ideal world the swagger comment would match actual function call; - # in reality there are over thirty mismatches. Use --pedantic to see. + # Determine the OPERATION. Done in a helper function because there + # are a lot of complicated special cases. # - my $operation = ''; - if ($line =~ /(generic|handlers|compat)\.(\w+)/) { - $operation = lcfirst $2; - if ($endpoint =~ m!/libpod/! && $operation !~ /^libpod/) { - $operation = 'libpod' . ucfirst $operation; - } - } - elsif ($line =~ /(libpod)\.(\w+)/) { - $operation = "$1$2"; - } + my $operation = operation_name($method, $endpoint); # Special case: the following endpoints all get a custom tag if ($endpoint =~ m!/(pods|manifests)/!) { $tag = $1; - $operation =~ s/^libpod//; - $operation = lcfirst $operation; } # Special case: anything related to 'events' gets a system tag @@ -264,13 +244,6 @@ sub handle_handle { my $actual = $actual[0]; - # By default, don't compare the operation: there are far too many - # mismatches here. - if (! $pedantic) { - $actual =~ s/\s+\S+\s*$//; - $expect =~ s/\s+\S+\s*$//; - } - # (Ignore whitespace discrepancies) (my $a_trimmed = $actual) =~ s/\s+/ /g; @@ -309,4 +282,72 @@ sub handle_handle { return; } + +#################### +# operation_name # Given a method + endpoint, return the swagger operation +#################### +sub operation_name { + my ($method, $endpoint) = @_; + + # /libpod/foo/bar -> (libpod, foo, bar) + my @endpoints = grep { /\S/ } split '/', $endpoint; + + # /libpod endpoints -> add 'Libpod' to end, e.g. PodStatsLibpod + my $Libpod = ''; + my $main = shift(@endpoints); + if ($main eq 'libpod') { + $Libpod = ucfirst($main); + $main = shift(@endpoints); + } + $main =~ s/s$//; # e.g. Volumes -> Volume + + # Next path component is an optional action: + # GET /containers/json -> ContainerList + # DELETE /libpod/containers/{name} -> ContainerDelete + # GET /libpod/containers/{name}/logs -> ContainerLogsLibpod + my $action = shift(@endpoints) || 'list'; + $action = 'list' if $action eq 'json'; + $action = 'delete' if $method eq 'DELETE'; + + # Anything with {id}, {name}, {name:..} may have a following component + if ($action =~ m!\{.*\}!) { + $action = shift(@endpoints) || 'inspect'; + $action = 'inspect' if $action eq 'json'; + } + + # All sorts of special cases + if ($action eq 'df') { + $action = 'dataUsage'; + } + # Grrrrrr, this one is annoying: some operations get an extra 'All' + elsif ($action =~ /^(delete|get|stats)$/ && $endpoint !~ /\{/) { + $action .= "All"; + $main .= 's' if $main eq 'container'; + } + # No real way to used MixedCase in an endpoint, so we have to hack it here + elsif ($action eq 'showmounted') { + $action = 'showMounted'; + } + # Ping is a special endpoint, and even if /libpod/_ping, no 'Libpod' + elsif ($main eq '_ping') { + $main = 'system'; + $action = 'ping'; + $Libpod = ''; + } + # Top-level compat endpoints + elsif ($main =~ /^(build|commit)$/) { + $main = 'image'; + $action = $1; + } + # Top-level system endpoints + elsif ($main =~ /^(auth|event|info|version)$/) { + $main = 'system'; + $action = $1; + $action .= 's' if $action eq 'event'; + } + + return "\u${main}\u${action}$Libpod"; +} + + 1; |