diff options
Diffstat (limited to 'hack')
| -rwxr-xr-x | hack/bats | 4 | ||||
| -rwxr-xr-x | hack/buildah-vendor-treadmill | 172 | ||||
| -rwxr-xr-x | hack/check_root.sh | 2 | ||||
| -rwxr-xr-x | hack/get_ci_vm.sh | 22 | ||||
| -rwxr-xr-x | hack/man-page-checker | 4 | ||||
| -rwxr-xr-x | hack/markdown-preprocess | 113 | ||||
| -rwxr-xr-x | hack/markdown-preprocess-review | 184 | ||||
| -rwxr-xr-x | hack/markdown-preprocess.t | 78 | ||||
| -rwxr-xr-x | hack/podman-registry | 12 | ||||
| -rw-r--r-- | hack/podman-registry-go/registry.go | 4 | ||||
| -rwxr-xr-x | hack/swagger-check | 4 |
11 files changed, 535 insertions, 64 deletions
@@ -106,7 +106,7 @@ export PODMAN_ROOTLESS_USER=$(id -un) # Root if [ -z "$ROOTLESS_ONLY" ]; then - echo "# bats ${bats_filter[@]} $TESTS" + echo "# bats ${bats_filter[*]} $TESTS" sudo --preserve-env=PODMAN \ --preserve-env=PODMAN_TEST_DEBUG \ --preserve-env=OCI_RUNTIME \ @@ -119,7 +119,7 @@ fi # Rootless. (Only if we're not already root) if [[ -z "$ROOT_ONLY" && "$(id -u)" != 0 ]]; then echo "--------------------------------------------------" - echo "\$ bats ${bats_filter[@]} $TESTS" + echo "\$ bats ${bats_filter[*]} $TESTS" bats "${bats_opts[@]}" "${bats_filter[@]}" $TESTS rc=$((rc | $?)) fi diff --git a/hack/buildah-vendor-treadmill b/hack/buildah-vendor-treadmill index d579a180a..b95290841 100755 --- a/hack/buildah-vendor-treadmill +++ b/hack/buildah-vendor-treadmill @@ -38,10 +38,6 @@ our $Treadmill_PR_Title = 'DO NOT MERGE: buildah vendor treadmill'; # Github API; this is where we query to find out the active treadmill PR our $API_URL = 'https://api.github.com/graphql'; -# Temporary file used to preserve current treadmill patches. This file -# should only exist very briefly while we perform branch operations. -our $Patch_File = "0000-$ME.patch"; - # Use colors if available and if stdout is a tty our $C_Highlight = ''; our $C_Warning = ''; @@ -66,14 +62,14 @@ eval ' sub usage { print <<"END_USAGE"; -Usage: $ME [OPTIONS] [--sync | --pick | --reset ] +Usage: $ME [OPTIONS] [--sync | --pick [PR] | --reset ] $ME is (2022-04-20) **EXPERIMENTAL** $ME is intended to solve the problem of vendoring buildah into podman. -Call me with one of two options: +Call me with one of three options: --sync The usual case. Mostly used by Ed. Called from a development branch, this just updates everything so @@ -81,7 +77,8 @@ Call me with one of two options: latest-podman (main). With a few sanity checks. --pick Used for really-truly vendoring in a new buildah; will - cherry-pick a commit on your buildah-vendor working branch + cherry-pick a commit on your buildah-vendor working branch. + Optional PR arg is the ID of the treadmill PR on github. --reset Used after vendoring buildah into main, when there really aren't any buildah patches to keep rolling. @@ -103,6 +100,7 @@ END_USAGE our %action; our $debug = 0; our $force_old_main = 0; # in --pick, proceeds even if main is old +our $force_retry = 0; # in --sync, continue despite saved checkpoint our $force_testing = 0; # in --sync, test even no podman/buildah changes our $verbose = 0; our $NOT = ''; # print "blahing the blah$NOT\n" if $debug @@ -114,6 +112,7 @@ sub handle_opts { 'reset' => sub { $action{reset}++ }, 'force-old-main' => \$force_old_main, + 'force-retry' => \$force_retry, 'force-testing' => \$force_testing, 'debug!' => \$debug, @@ -140,11 +139,6 @@ sub main { # and there's no clean way to make it use @_. handle_opts(); # will set package globals - # Fetch command-line arguments. Barf if too many. - # FIXME: if called with arg, that's the --sync branch? - # FIXME: if called with --pick + arg, that's the PR? - die "$ME: Too many arguments; try $ME --help\n" if @ARGV; - my @action = keys(%action); die "$ME: Please invoke me with one of --sync or --pick\n" if ! @action; @@ -158,13 +152,15 @@ sub main { # that repo is clean. None of our actions can be run on a dirty repo. assert_clean_repo(); - $handler->(); + $handler->(@ARGV); } ############################################################################### # BEGIN sync and its helpers sub do_sync { + die "$ME: --sync takes no arguments; try $ME --help\n" if @_; + # Preserve current branch name, so we can come back after switching to main my $current_branch = git_current_branch(); @@ -188,11 +184,13 @@ sub do_sync { pull_main(); git('checkout', '-q', $current_branch); - # Preserve local patches. --always will generate empty patches (e.g., - # after a buildah vendor when everything is copacetic); --no-signature - # prevents a buildup of "-- 2.35" (git version) lines at the end. - git('format-patch', '--always', '--no-signature', "--output=$Patch_File", 'HEAD^'); - progress("Treadmill patches saved to $Patch_File"); + # Make a temporary copy of this branch + my $temp_branch = strftime("__buildah-treadmill-checkpoint/%Y%m%d-%H%M%S", localtime); + git('branch', $temp_branch, $current_branch); + progress("Current branch preserved as $temp_branch"); + + # Get the hash of the top (treadmill) commit, to cherry-pick later + my $treadmill_commit = git('rev-parse', 'HEAD'); # # Danger Will Robinson! This is where it gets scary: a failure here @@ -207,7 +205,11 @@ This is not something I can recover from. Your human judgment is needed. You will need to recover from this manually. Your best option is to look at the source code for this script. -Your treadmill patches are here: $Patch_File +Treadmill branch copy is preserved in $temp_branch + +To restore state to where you were before this sync: + \$ git checkout main + \$ git branch -f $current_branch $treadmill_commit END_FAIL_INSTRUCTIONS exit 1; @@ -260,12 +262,34 @@ END_FAIL_INSTRUCTIONS git_commit_buildah($buildah_new); # And, finally, this has the highest possibility of failing - progress('Reapplying preserved patches'); - git('am', '--empty=keep', $Patch_File); + local $SIG{__DIE__} = sub { + print STDERR $C_Warning, "@_", <<"END_FAIL_INSTRUCTIONS"; + +This is not something I can recover from. Your human judgment is needed. + +Chances are, you might be able to run 'git status', look for +merge conflicts, manually resolve those, 'git add', then +'git cherry-pick --continue'. If that works, run this script +again (you will probably need the --force-retry option). - # It worked! Clean up: remove our local die() handler and the patch file +If that DOES NOT work, your only option is to look at the source code +for this script. Sorry. There's only so much that can be done automatically. + +Treadmill branch copy is preserved in $temp_branch + +To restore state to where you were before this sync: + \$ git checkout main + \$ git branch -f $current_branch $treadmill_commit +END_FAIL_INSTRUCTIONS + + exit 1; + }; + progress('Reapplying treadmill patches'); + git('cherry-pick', '--allow-empty', $treadmill_commit); + + # It worked! Clean up: remove our local die() handler and the saved branch undef $SIG{__DIE__}; - unlink $Patch_File; + git('branch', '-D', $temp_branch); # if buildah is unchanged, and we did not pull main, exit cleanly my $change_message = ''; @@ -295,6 +319,13 @@ END_FAIL_INSTRUCTIONS progress("All OK. It's now up to you to 'git push --force'"); progress(" --- Reminder: $change_message"); + + # Kind of kludgy. If user had to retry a prior failed attempt, and + # things are now successful, remind them to delete old checkpoints. + # ($force_retry is a 'git branch -D' command string at this point.) + if ($force_retry) { + progress(" --- Retry worked! You may now $force_retry"); + } } ############### @@ -429,8 +460,9 @@ sub do_pick { assert_buildah_vendor_commit('HEAD'); progress("HEAD is a buildah vendor commit. Good."); - # Identify and pull the treadmill PR - my $treadmill_pr = treadmill_pr(); + # Identify and pull the treadmill PR. + my $treadmill_pr = shift || treadmill_pr(); + my $treadmill_branch = "$ME/pr$treadmill_pr/tmp$$"; progress("Fetching treadmill PR $treadmill_pr into $treadmill_branch"); git('fetch', '-q', git_upstream(), "pull/$treadmill_pr/head:$treadmill_branch"); @@ -465,6 +497,26 @@ sub do_pick { # treadmill_pr # Returns ID of open podman PR with the desired subject ################## sub treadmill_pr { + # Github API (or maybe just the search endpoint???) is restricted. + my $token = $ENV{GITHUB_TOKEN} + or do { + warn <<"END_NEED_PR"; +$ME: Cannot proceed without PR ID. + +If you have a github API token, please: export GITHUB_TOKEN=....... +and re-run me. + +If you do not have a github API token, please go here: + + https://github.com/containers/podman/pulls?q=is%3Apr+is%3Aopen+%22buildah+vendor+treadmill%22 + +...then reinvoke me, adding that PR ID to the command line args. + +As of 2022-09-12 the treadmill PR is 13808, but that may change over time. +END_NEED_PR + exit 1; + }; + my $query = <<'END_QUERY'; { search( @@ -481,16 +533,10 @@ END_QUERY $ua->agent("$ME " . $ua->agent); # Identify ourself my %headers = ( + 'Authorization' => "bearer $token", 'Accept' => "application/vnd.github.antiope-preview+json", 'Content-Type' => "application/json", ); - - # Use github token if available, but don't require it. (All it does is - # bump up our throttling limit, which shouldn't be an issue) (unless - # someone invokes this script hundreds of times per minute). - if (my $token = $ENV{GITHUB_TOKEN}) { - $headers{Authorization} = "bearer $token"; - } $ua->default_header($_ => $headers{$_}) for keys %headers; # Massage the query: escape quotes, put it all in one line, collapse spaces @@ -503,7 +549,9 @@ END_QUERY print $postquery, "\n" if $debug; my $res = $ua->post($API_URL, Content => $postquery); if ((my $code = $res->code) != 200) { - print $code, " ", $res->message, "\n"; + warn "$ME: GraphQL request failed on $API_URL:\n"; + print STDERR " ", $code, " ", $res->message, "\n"; + warn "Cannot continue.\n"; exit 1; } @@ -621,8 +669,8 @@ from the buildah vendor treadmill PR, #%s EOF # Strip the "DO NOT MERGE" header from the treadmill PR, print only -# the "Changes as of YYYY-MM-DD" and subsequent lines -sed -ne '/^Changes as of/,$ p' <$msgfile >>$tmpfile +# the "Changes since YYYY-MM-DD" and subsequent lines +sed -ne '/^Changes since /,$ p' <$msgfile >>$tmpfile mv $tmpfile $msgfile END_EDIT_SCRIPT @@ -639,6 +687,8 @@ END_EDIT_SCRIPT # BEGIN reset and its helpers sub do_reset { + die "$ME: --sync takes no arguments; try $ME --help\n" if @_; + my $current_branch = git_current_branch(); # Make sure side branch == main (i.e., there are no commits on the branch) @@ -681,20 +731,46 @@ sub progress { # assert_clean_repo # Don't even think of running with local changes ####################### sub assert_clean_repo { - # Our patch file should only exist for brief moments during a sync run. - # If it exists at any other time, something has gone very wrong. - if (-e $Patch_File) { - warn <<"END_WARN"; -$ME: File exists: $Patch_File + # During --sync we create a temporary copy of the treadmill branch, + # in case something goes wrong. The branch is deleted on success. + # If one exists, it means we may have lost work. + my @relics = grep { + m!^__buildah-treadmill-checkpoint/\d+-\d+$! + } git('branch', '--list', '--format=%(refname:lstrip=2)'); + if (@relics) { + if ($force_retry) { + warn <<"END_WARN"; +$ME: WARNING: leftover checkpoint(s): @relics + + ...continuing due to --force-retry. + + If things work out, you can 'git branch -D @relics' +END_WARN + + # OK, ugly override of a binary flag, but it's OK because + # it helps with user-friendliness: offer a reminder upon + # successful completion of the script. + $force_retry = "git branch -D @relics"; + } + else { + warn <<"END_WARN"; +$ME: FATAL: leftover checkpoint: @relics This means that something went very wrong during an earlier sync run. Your git branch may be in an inconsistent state. Your work to date - may be lost. This file may be your only hope of recovering it. + may be lost. This branch may be your only hope of recovering it. This is not something a script can resolve. You need to look at this - file, compare to your git HEAD, and manually reconcile any differences. + branch, compare to your git HEAD, and manually reconcile any differences. + + If you really know what you're doing, i.e., if you've reconciled + merge conflicts and have a pretty secure branch structure, try + rerunning me with --force-retry. Or, if that checkpoint is a + remnant from a past run, and you're ultra-certain that you don't + need it, you can git branch -D @relics END_WARN - exit 1; + exit 1; + } } # OK so far. Now check for modified files. @@ -727,7 +803,15 @@ sub git_current_branch() { # git_forkpoint # Hash at which branch (default: cur) branched from main ################### sub git_forkpoint { - return git('merge-base', '--fork-point', 'main', @_); + # '--fork-point vendor-branch' fails silently on Paul's git tree, + # but plain merge-base works fine. My head hurts from trying to + # understand the docs, so I give up. Just try fork-point first, + # and if it fails, try without. #cargocult #gitishard + my $forkpoint = eval { git('merge-base', '--fork-point', 'main', @_) }; + if ($@) { + $forkpoint = git('merge-base', 'main', @_); + } + return $forkpoint; } ##################### diff --git a/hack/check_root.sh b/hack/check_root.sh index 1f53887ff..2489a4edd 100755 --- a/hack/check_root.sh +++ b/hack/check_root.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash if ! [ $(id -u) = 0 ]; then - echo "Please run as root! '$@' requires root privileges." + echo "Please run as root! '$*' requires root privileges." exit 1 fi diff --git a/hack/get_ci_vm.sh b/hack/get_ci_vm.sh index 6632a0178..bdd947bba 100755 --- a/hack/get_ci_vm.sh +++ b/hack/get_ci_vm.sh @@ -15,7 +15,7 @@ SCRIPT_FILEPATH=$(realpath "${BASH_SOURCE[0]}") SCRIPT_DIRPATH=$(dirname "$SCRIPT_FILEPATH") REPO_DIRPATH=$(realpath "$SCRIPT_DIRPATH/../") -# Help detect if we were called by get_ci_vm container +# Help detect what get_ci_vm container called this script GET_CI_VM="${GET_CI_VM:-0}" in_get_ci_vm() { if ((GET_CI_VM==0)); then @@ -27,8 +27,10 @@ in_get_ci_vm() { # get_ci_vm APIv1 container entrypoint calls into this script # to obtain required repo. specific configuration options. if [[ "$1" == "--config" ]]; then - in_get_ci_vm "$1" - cat <<EOF + in_get_ci_vm "$1" # handles GET_CI_VM==0 case + case "$GET_CI_VM" in + 1) + cat <<EOF DESTDIR="/var/tmp/go/src/github.com/containers/podman" UPSTREAM_REPO="https://github.com/containers/podman.git" CI_ENVFILE="/etc/ci_environment" @@ -40,8 +42,18 @@ GCLOUD_CPUS="2" GCLOUD_MEMORY="4Gb" GCLOUD_DISK="200" EOF + ;; + 2) + # get_ci_vm APIv2 configuration details + echo "AWS_PROFILE=containers" + ;; + *) + echo "Error: Your get_ci_vm container image is too old." + ;; + esac elif [[ "$1" == "--setup" ]]; then in_get_ci_vm "$1" + unset GET_CI_VM # get_ci_vm container entrypoint calls us with this option on the # Cirrus-CI environment instance, to perform repo.-specific setup. cd $REPO_DIRPATH @@ -54,8 +66,9 @@ elif [[ "$1" == "--setup" ]]; then echo "+ Running environment setup" > /dev/stderr ./contrib/cirrus/setup_environment.sh else - # Create and access VM for specified Cirrus-CI task + # Pass this repo and CLI args into container for VM creation/management mkdir -p $HOME/.config/gcloud/ssh + mkdir -p $HOME/.aws podman run -it --rm \ --tz=local \ -e NAME="$USER" \ @@ -65,5 +78,6 @@ else -v $REPO_DIRPATH:/src:O \ -v $HOME/.config/gcloud:/root/.config/gcloud:z \ -v $HOME/.config/gcloud/ssh:/root/.ssh:z \ + -v $HOME/.aws:/root/.aws:z \ quay.io/libpod/get_ci_vm:latest "$@" fi diff --git a/hack/man-page-checker b/hack/man-page-checker index 8ee0aaf6d..83e0b8b1d 100755 --- a/hack/man-page-checker +++ b/hack/man-page-checker @@ -87,7 +87,7 @@ function compare_usage() { # strip off command name from both from_man=$(sed -e "s/\*\*$cmd\*\*[[:space:]]*//" <<<"$from_man") - from_help=$(sed -e "s/^[[:space:]]*$cmd[[:space:]]*//" <<<"$from_help") + from_help=$(sed -e "s/^[[:space:]]*${cmd}[[:space:]]*//" <<<"$from_help") # man page lists 'foo [*options*]', help msg shows 'foo [flags]'. # Make sure if one has it, the other does too. @@ -153,7 +153,7 @@ for md in *.1.md;do # special case: the command is "auto-update", with a hyphen md_nodash='podman auto-update' fi - if [ "$cmd" != "$md_nodash" -a "$cmd" != "podman-remote" ]; then + if [[ "$cmd" != "$md_nodash" ]] && [[ "$cmd" != "podman-remote" ]]; then echo printf "Inconsistent program name in SYNOPSIS in %s:\n" $md printf " SYNOPSIS = %s (expected: '%s')\n" "$cmd" "$md_nodash" diff --git a/hack/markdown-preprocess b/hack/markdown-preprocess new file mode 100755 index 000000000..9cd1e9605 --- /dev/null +++ b/hack/markdown-preprocess @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +# +# markdown-preprocess - filter *.md.in files, convert to .md +# + +import glob +import os +import re +import sys + +def main(): + script_dir = os.path.abspath(os.path.dirname(__file__)) + man_dir = os.path.join(script_dir,"../docs/source/markdown") + + try: + os.chdir(man_dir) + except FileNotFoundError: + raise Exception("Please invoke me from the base repo dir") + + # If called with args, process only those files + infiles = [ os.path.basename(x) for x in sys.argv[1:] ] + if len(infiles) == 0: + # Called without args: process all *.md.in files + infiles = glob.glob('*.md.in') + for infile in infiles: + process(infile) + +def process(infile): + # Some options are the same between containers and pods; determine + # which description to use from the name of the source man page. + pod_or_container = 'container' + if '-pod-' in infile or '-kube-' in infile: + pod_or_container = 'pod' + + # foo.md.in -> foo.md -- but always write to a tmpfile + outfile = os.path.splitext(infile)[0] + outfile_tmp = outfile + '.tmp.' + str(os.getpid()) + +# print("got here: ",infile, " -> ", outfile) + + with open(infile, 'r') as fh_in, open(outfile_tmp, 'w') as fh_out: + for line in fh_in: + # '@@option foo' -> include file options/foo.md + if line.startswith('@@option '): + _, optionname = line.strip().split(" ") + optionfile = os.path.join("options", optionname + '.md') + + # Comment intended to help someone viewing the .md file. + # Leading newline is important because if two lines are + # consecutive without a break, sphinx (but not go-md2man) + # treats them as one line and will unwantedly render the + # comment in its output. + fh_out.write("\n[//]: # (BEGIN included file " + optionfile + ")\n") + with open(optionfile, 'r') as fh_optfile: + for opt_line in fh_optfile: + opt_line = replace_type(opt_line, pod_or_container) + opt_line = opt_line.replace('<<subcommand>>', podman_subcommand(infile)) + opt_line = opt_line.replace('<<fullsubcommand>>', podman_subcommand(infile, 'full')) + fh_out.write(opt_line) + fh_out.write("\n[//]: # (END included file " + optionfile + ")\n") + else: + fh_out.write(line) + + os.chmod(outfile_tmp, 0o444) + os.rename(outfile_tmp, outfile) + +# Given a file path of the form podman-foo-bar.1.md.in, return "foo bar" +def podman_subcommand(string: str, full=None) -> str: + # Special case: 'podman-pod-start' becomes just 'start' + if not full: + if string.startswith("podman-pod-"): + string = string[len("podman-pod-"):] + if string.startswith("podman-"): + string = string[len("podman-"):] + if string.endswith(".1.md.in"): + string = string[:-len(".1.md.in")] + return string.replace("-", " ") + +# Replace instances of '<<pod|container>>' with the desired one (based on +# 'type' which is 'pod' or 'container'). +def replace_type(line: str, type: str) -> str: + # Internal helper function: determines the desired half of the <a|b> string + def replwith(matchobj): + lhs, rhs = matchobj[0].split('|') + # Strip off '<<' and '>>' + lhs = lhs[2:] + rhs = rhs[:len(rhs)-2] + + # Check both sides for 'pod' followed by (non-"m" or end-of-string). + # The non-m prevents us from triggering on 'podman', which could + # conceivably be present in both sides. And we check for 'pod', + # not 'container', because it's possible to have something like + # <<container in pod|container>>. + if re.match('.*pod([^m]|$)', lhs, re.IGNORECASE): + if re.match('.*pod([^m]|$)', rhs, re.IGNORECASE): + raise Exception("'%s' matches 'pod' in both left and right sides" % matchobj[0]) + # Only left-hand side has "pod" + if type == 'pod': + return lhs + else: + return rhs + else: + if not re.match('.*pod([^m]|$)', rhs, re.IGNORECASE): + raise Exception("'%s' does not match 'pod' in either side" % matchobj[0]) + if type == 'pod': + return rhs + else: + return lhs + + return re.sub('<<[^\|>]*\|[^\|>]*>>', replwith, line) + +if __name__ == "__main__": + main() diff --git a/hack/markdown-preprocess-review b/hack/markdown-preprocess-review new file mode 100755 index 000000000..a3e237fb6 --- /dev/null +++ b/hack/markdown-preprocess-review @@ -0,0 +1,184 @@ +#!/usr/bin/perl + +(our $ME = $0) =~ s|^.*/||; + +use v5.20; + +our $DSM = 'docs/source/markdown'; + +my ($oldname, $newname); +my %oldname; +my %changed; +open my $git_diff, '-|', 'git', 'log', '-1', '-p' + or die "$ME: Cannot fork: $!\n"; +while (my $line = <$git_diff>) { + chomp $line; + + if ($line =~ m!^\-\-\-\s+a/$DSM/(podman-\S+\.md(\.in)?)!) { + $oldname = $1; + $newname = undef; + } + elsif ($line =~ m!^\+\+\+\s+b/$DSM/(podman-\S+\.md(\.in)?)!) { + $newname = $1; + $oldname{$newname} = $oldname; + } + elsif ($newname) { + if ($line =~ s/^-####\s+//) { + $line =~ /^\*\*--(\S+?)\*\*/ + or die "$ME: in $newname: weird '$line'"; + $changed{$newname}{$1}{name} //= $1; + } + # Usually the same, but not for host.container and host.pod.md + elsif ($line =~ /^\+\@\@option\s+(\S+)/) { + my $optfile = $1; + if ($optfile =~ /^(.*)\.\S+$/) { + $changed{$newname}{$1}{name} = $optfile; + } + } + } +} +close $git_diff; + +# Pass 2: read each oldfile, parse changed options +for my $f (sort keys %changed) { + my $oldfile = $oldname{$f}; + open my $git_fh, '-|', 'git', 'show', "HEAD^:$DSM/$oldfile" + or die "$ME: Cannot fork: $!\n"; + my $opt; + while (my $line = <$git_fh>) { + if ($line =~ /^####\s+\*\*--(\S+?)\*\*/) { + $opt = $1; + if ($changed{$f}{$opt}) { + $changed{$f}{$opt}{text} = $line; + } + else { + undef $opt; + } + } + elsif ($line =~ /^#/ || $line =~ /^\@\@option\s/) { + undef $opt; + } + elsif ($opt) { + $changed{$f}{$opt}{text} .= $line; + } + } + close $git_fh + or die "$ME: Error running git on $oldfile\n"; +} + +# Pass 3: write out files +my $tempdir = "/tmp/$ME.diffs"; +system('rm', '-rf', $tempdir); +mkdir $tempdir, 0755; + +for my $md_file (sort keys %changed) { + for my $opt (sort keys %{$changed{$md_file}}) { + my $d = "$tempdir/$changed{$md_file}{$opt}{name}"; + mkdir $d, 0755; + + my $outfile = "$d/$md_file"; + open my $fh, '>', $outfile + or die "$ME: Cannot create $outfile: $!\n"; + # strip all trailing newlines + (my $text = $changed{$md_file}{$opt}{text}) =~ s/\n+$/\n/s; + print { $fh } $text; + close $fh + or die "$ME: Error writing $outfile: $!\n"; + + my $new_text = "$DSM/options/$changed{$md_file}{$opt}{name}.md"; + die "$ME: $md_file: File does not exist: $new_text\n" if ! -e $new_text; + system('cp', $new_text, "$d/zzz-chosen.md"); + } +} + +# Now run diffuse +chdir $tempdir or die; +my @all_opts = glob("*"); +for my $i (0..$#all_opts) { + my $opt = $all_opts[$i]; + chdir "$tempdir/$opt" + or die "??? Internal error, cannot cd $tempdir/$opt: $!"; + + $| = 1; printf "--%s (%d/%d) ", $opt, $i+1, scalar(@all_opts); + + my @all_files = glob("*"); + if (all_files_identical(@all_files)) { + pop @all_files; + print "[identical between @all_files]\n"; + next; + } + + # Prompt + print "[Y/n/q] "; + my $ans = <STDIN>; + next if $ans =~ /^n/i; + exit 0 if $ans =~ /^q/i; + + # Try to cull the files (remove identical ones) + my @files = glob("*"); + my $winner = pop @files; + + for my $f (@files) { + system('cmp', '-s', $f, $winner); + if ($? == 0) { + print "[ $f is the one we went with; removing from list ]\n"; + unlink $f; + next; + } + + system('wdiff', '-1', '-2', '-3', $f, $winner); + if ($? == 0) { + print "[ $f is whitespace-identical with what we went with ]\n"; + unlink $f; + next; + } + } + + # Recompute @files, in case some were deleted above + @files = glob("*"); pop @files; + + for (my $i=0; $i < $#files; $i++) { + my $f1 = $files[$i]; + next unless -e $f1; + + for (my $j=$i+1; $j <= $#files; $j++) { + my $f2 = $files[$j]; + next unless -e $f2; + + system('wdiff', '-1', '-2', '-3', $f1, $f2); + if ($? == 0) { + print "[ $f2 : removing, it =~ $f1 ]\n"; + unlink $f2; + } + } + } + + # Recompute @files, in case some were deleted above + @files = glob("*"); + + # diffuse works great for 3-4 files, passable for 5, not at all for >5 + if (@files <= 5) { + system("diffuse", "-w", @files) == 0 + or die "Diffuse failed\n"; + } + else { + # Too many files. Go by threes. + my $winner = pop @files; + for (my $i=0; $i < @files; $i += 3) { + system("diffuse", "-w", @files[$i..$i+2], $winner); + } + } +} + + +sub all_files_identical { + my %sha; + for my $f (@_) { + my $result = qx{sha256sum $f}; + $result =~ /^([0-9a-f]+)\s/ + or die "Internal error: unexpected result from sha256sum $f: $result"; + $sha{$1}++; + } + + return (keys(%sha) == 1); +} diff --git a/hack/markdown-preprocess.t b/hack/markdown-preprocess.t new file mode 100755 index 000000000..152da087b --- /dev/null +++ b/hack/markdown-preprocess.t @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 + +""" +Tests for markdown-preprocess +""" + +import unittest + +# https://stackoverflow.com/questions/66665217/how-to-import-a-python-script-without-a-py-extension +from importlib.util import spec_from_loader, module_from_spec +from importlib.machinery import SourceFileLoader + +spec = spec_from_loader("mp", SourceFileLoader("mp", "hack/markdown-preprocess")) +mp = module_from_spec(spec) +spec.loader.exec_module(mp) + +class TestPodReplacer(unittest.TestCase): + def test_basic(self): + """basic pod|container and vice-versa""" + s = '<<container|pod>>' + self.assertEqual(mp.replace_type(s, 'pod'), 'pod') + self.assertEqual(mp.replace_type(s, 'container'), 'container') + s = '<<container|pod>>' + self.assertEqual(mp.replace_type(s, 'pod'), 'pod') + self.assertEqual(mp.replace_type(s, 'container'), 'container') + + def test_case_insensitive(self): + """test case-insensitive replacement of Pod, Container""" + s = '<<Pod|Container>>' + self.assertEqual(mp.replace_type(s, 'pod'), 'Pod') + self.assertEqual(mp.replace_type(s, 'container'), 'Container') + s = '<<Container|Pod>>' + self.assertEqual(mp.replace_type(s, 'pod'), 'Pod') + self.assertEqual(mp.replace_type(s, 'container'), 'Container') + + def test_dont_care_about_podman(self): + """we ignore 'podman'""" + self.assertEqual(mp.replace_type('<<podman container|pod in podman>>', 'container'), 'podman container') + + def test_not_at_beginning(self): + """oops - test for 'pod' other than at beginning of string""" + s = '<<container|container or pod>>' + self.assertEqual(mp.replace_type(s, 'container'), 'container') + self.assertEqual(mp.replace_type(s, 'pod'), 'container or pod') + s = '<<container or pod|container>>' + self.assertEqual(mp.replace_type(s, 'container'), 'container') + self.assertEqual(mp.replace_type(s, 'pod'), 'container or pod') + + def test_blank(self): + """test that either side of '|' can be empty""" + s = 'abc container<<| or pod>> def' + self.assertEqual(mp.replace_type(s, 'container'), 'abc container def') + self.assertEqual(mp.replace_type(s, 'pod'), 'abc container or pod def') + s = 'abc container<< or pod|>> def' + self.assertEqual(mp.replace_type(s, 'container'), 'abc container def') + self.assertEqual(mp.replace_type(s, 'pod'), 'abc container or pod def') + + def test_exception_both(self): + """test that 'pod' on both sides raises exception""" + with self.assertRaisesRegex(Exception, "in both left and right sides"): + mp.replace_type('<<pod 123|pod 321>>', 'pod') + + def test_exception_neither(self): + """test that 'pod' on neither side raises exception""" + with self.assertRaisesRegex(Exception, "in either side"): + mp.replace_type('<<container 123|container 321>>', 'pod') + +class TestPodmanSubcommand(unittest.TestCase): + def test_basic(self): + """podman subcommand basic test""" + self.assertEqual(mp.podman_subcommand("podman-foo.1.md.in"), "foo") + self.assertEqual(mp.podman_subcommand("podman-foo-bar.1.md.in"), "foo bar") + self.assertEqual(mp.podman_subcommand("podman-pod-rm.1.md.in"), "rm") + self.assertEqual(mp.podman_subcommand("podman-pod-rm.1.md.in", "full"), "pod rm") + + +if __name__ == '__main__': + unittest.main() diff --git a/hack/podman-registry b/hack/podman-registry index f6a266883..f1b68deaa 100755 --- a/hack/podman-registry +++ b/hack/podman-registry @@ -7,7 +7,7 @@ ME=$(basename $0) ############################################################################### # BEGIN defaults -PODMAN_REGISTRY_IMAGE=quay.io/libpod/registry:2.6 +PODMAN_REGISTRY_IMAGE=quay.io/libpod/registry:2.8 PODMAN_REGISTRY_USER= PODMAN_REGISTRY_PASS= @@ -30,7 +30,7 @@ into a local temporary directory, create an htpasswd, start the registry, and dump a series of environment variables to stdout: \$ $ME start - PODMAN_REGISTRY_IMAGE=\"docker.io/library/registry:2.6\" + PODMAN_REGISTRY_IMAGE=\"docker.io/library/registry:2.8\" PODMAN_REGISTRY_PORT=\"5050\" PODMAN_REGISTRY_USER=\"userZ3RZ\" PODMAN_REGISTRY_PASS=\"T8JVJzKrcl4p6uT\" @@ -197,13 +197,11 @@ function do_start() { # Store credentials where container will see them. We can't run # this one via must_pass because we need its stdout. - podman run --rm \ - --entrypoint htpasswd ${PODMAN_REGISTRY_IMAGE} \ - -Bbn ${PODMAN_REGISTRY_USER} ${PODMAN_REGISTRY_PASS} \ + htpasswd -Bbn ${PODMAN_REGISTRY_USER} ${PODMAN_REGISTRY_PASS} \ > $AUTHDIR/htpasswd if [ $? -ne 0 ]; then rm -rf ${PODMAN_REGISTRY_WORKDIR} - die "Command failed: podman run [htpasswd]" + die "Command failed: htpasswd" fi # In case someone needs to debug @@ -220,7 +218,7 @@ function do_start() { -e "REGISTRY_AUTH_HTPASSWD_PATH=/auth/htpasswd" \ -e "REGISTRY_HTTP_TLS_CERTIFICATE=/auth/domain.crt" \ -e "REGISTRY_HTTP_TLS_KEY=/auth/domain.key" \ - registry:2.6 + ${PODMAN_REGISTRY_IMAGE} # Confirm that registry started and port is active wait_for_port $PODMAN_REGISTRY_PORT diff --git a/hack/podman-registry-go/registry.go b/hack/podman-registry-go/registry.go index d66d092b6..30da3b6da 100644 --- a/hack/podman-registry-go/registry.go +++ b/hack/podman-registry-go/registry.go @@ -57,7 +57,7 @@ func StartWithOptions(options *Options) (*Registry, error) { // Start a registry. out, err := utils.ExecCmd(binary, args...) if err != nil { - return nil, fmt.Errorf("error running %q: %s: %w", binary, out, err) + return nil, fmt.Errorf("running %q: %s: %w", binary, out, err) } // Parse the output. @@ -112,7 +112,7 @@ func (r *Registry) Stop() error { return nil } if _, err := utils.ExecCmd(binary, "-P", r.Port, "stop"); err != nil { - return fmt.Errorf("error stopping registry (%v) with %q: %w", *r, binary, err) + return fmt.Errorf("stopping registry (%v) with %q: %w", *r, binary, err) } r.running = false return nil diff --git a/hack/swagger-check b/hack/swagger-check index 1e5b95c3a..b4481f5bb 100755 --- a/hack/swagger-check +++ b/hack/swagger-check @@ -320,8 +320,8 @@ sub operation_name { if ($action eq 'df') { $action = 'dataUsage'; } - elsif ($action eq "delete" && $endpoint eq "/libpod/kube/play") { - $action = "PlayDown" + elsif ($action eq "delete" && $endpoint eq "/libpod/play/kube") { + $action = "KubeDown" } # Grrrrrr, this one is annoying: some operations get an extra 'All' elsif ($action =~ /^(delete|get|stats)$/ && $endpoint !~ /\{/) { |