diff options
Diffstat (limited to 'hack')
| -rwxr-xr-x | hack/buildah-vendor-treadmill | 401 | ||||
| -rw-r--r-- | hack/podman-registry-go/registry.go | 23 |
2 files changed, 339 insertions, 85 deletions
diff --git a/hack/buildah-vendor-treadmill b/hack/buildah-vendor-treadmill index 1feffaa60..0aa4245c5 100755 --- a/hack/buildah-vendor-treadmill +++ b/hack/buildah-vendor-treadmill @@ -16,7 +16,7 @@ use JSON; use LWP::UserAgent; (our $ME = $0) =~ s|.*/||; -our $VERSION = '0.1'; +our $VERSION = '0.2'; # For debugging, show data structures using DumpTree($var) #use Data::TreeDumper; $Data::TreeDumper::Displayaddress = 0; @@ -24,24 +24,36 @@ our $VERSION = '0.1'; ############################################################################### # BEGIN user-customizable section +# Page describing this process in much more detail +our $Docs_URL = + 'https://github.com/containers/podman/wiki/Buildah-Vendor-Treadmill'; + # github path to buildah our $Buildah = 'github.com/containers/buildah'; # FIXME FIXME FIXME: add 'main'? I hope we never need this script for branches. our $Treadmill_PR_Title = 'DO NOT MERGE: buildah vendor treadmill'; +# Github API; this is where we query to find out the active treadmill PR our $API_URL = 'https://api.github.com/graphql'; +# Temporary file used to preserve current treadmill patches. This file +# should only exist very briefly while we perform branch operations. +our $Patch_File = "0000-$ME.patch"; + # Use colors if available and if stdout is a tty -our $Highlight = ''; -our $Reset = ''; +our $C_Highlight = ''; +our $C_Warning = ''; +our $C_Reset = ''; eval ' use Term::ANSIColor; if (-t 1) { - $Highlight = color("green"); - $Reset = color("reset"); + $C_Highlight = color("green"); + $C_Warning = color("bold red"); + $C_Reset = color("reset"); + } - $SIG{__WARN__} = sub { print STDERR color("bold red"), "@_", $Reset; }; + $SIG{__WARN__} = sub { print STDERR $C_Warning, "@_", $C_Reset; }; '; @@ -72,7 +84,7 @@ Call me with one of two options: For latest documentation and best practices, please see: - https://github.com/containers/podman/wiki/Buildah-Vendor-Treadmill + $Docs_URL OPTIONS: @@ -86,7 +98,8 @@ END_USAGE # Command-line options. Note that this operates directly on @ARGV ! our %action; our $debug = 0; -our $force = 0; +our $force_old_main = 0; # in --pick, proceeds even if main is old +our $force_testing = 0; # in --sync, test even no podman/buildah changes our $verbose = 0; our $NOT = ''; # print "blahing the blah$NOT\n" if $debug sub handle_opts { @@ -95,9 +108,11 @@ sub handle_opts { 'sync' => sub { $action{sync}++ }, 'pick' => sub { $action{pick}++ }, + 'force-old-main' => \$force_old_main, + 'force-testing' => \$force_testing, + 'debug!' => \$debug, 'dry-run|n!' => sub { $NOT = ' [NOT]' }, - 'force' => \$force, 'verbose|v' => \$verbose, help => \&usage, @@ -148,70 +163,129 @@ sub do_sync { # Preserve current branch name, so we can come back after switching to main my $current_branch = git_current_branch(); + # Branch HEAD must be the treadmill commit. + my $commit_message = git('log', '-1', '--format=%s', 'HEAD'); + print "[$commit_message]\n" if $verbose; + $commit_message =~ /buildah.*treadmill/ + or die "$ME: HEAD must be a 'buildah treadmill' commit.\n"; + + # ...and previous commit must be a scratch buildah vendor + $commit_message = git('log', '-1', '--format=%B', 'HEAD^'); + $commit_message =~ /DO NOT MERGE.* vendor in buildah.*JUNK COMMIT/s + or die "$ME: HEAD^ must be a DO NOT MERGE / JUNK COMMIT commit\n"; + assert_buildah_vendor_commit('HEAD^'); + + # Looks good so far. my $buildah_old = vendored_buildah(); print "-> buildah old = $buildah_old\n"; - # If HEAD is a buildah-vendor commit (usual case), drop it now. - if (head_is_buildah_vendor_commit()) { - if (is_treadmill_commit('HEAD^')) { - progress("HEAD is buildah vendor (as expected); dropping it..."); - git('reset', '--hard', 'HEAD^'); - } - else { - die "$ME: HEAD is a buildah commit, but HEAD^ is not a treadmill commit! Cannot continue.\n"; - } - } - # HEAD must now be a treadmill commit - is_treadmill_commit('HEAD') - or die "$ME: HEAD is not a treadmill commit!\n"; - - # HEAD is now a change to buildah-tests. Now update main and rebase. + # Pull main, and pivot back to this branch pull_main(); git('checkout', '-q', $current_branch); - my $forkpoint = git('merge-base', '--fork-point', 'main'); - my $main_commit = git('rev-parse', 'main'); + + # Preserve local patches + git('format-patch', "--output=$Patch_File", 'HEAD^'); + progress("Treadmill patches saved to $Patch_File"); + + # + # Danger Will Robinson! This is where it gets scary: a failure here + # can leave us in a state where we could lose the treadmill patches. + # Proceed with extreme caution. + # + local $SIG{__DIE__} = sub { + print STDERR $C_Warning, "@_", <<"END_FAIL_INSTRUCTIONS"; + +This is not something I can recover from. Your human judgment is needed. + +You will need to recover from this manually. Your best option is to +look at the source code for this script. + +Your treadmill patches are here: $Patch_File +END_FAIL_INSTRUCTIONS + + exit 1; + }; + + my $forkpoint = git_forkpoint(); my $rebased; + + # Unlikely to fail + git('reset', '--hard', 'HEAD^^'); + + # Rebase branch. Also unlikely to fail + my $main_commit = git('rev-parse', 'main'); if ($forkpoint eq $main_commit) { progress("[Already rebased on podman main]"); } else { - # --empty=keep may be needed after a --pick commit, when we've - # vendored a new buildah into podman and incorporated the treadmill - # commit. Since this is a perpetual-motion workflow, in which we - # keep an in-progress PR open at all times, we need a baseline - # commit even if it's empty. progress("Rebasing on podman main..."); git('rebase', '--empty=keep', 'main'); - # FIXME: rebase can fail after --pick. If it does, offer instructions. $rebased = 1; } - # We're now back on our treadmill branch, with one commit on top of main. - # Now vendor in latest buildah. + # This does have a high possibility of failing. progress("Vendoring in buildah..."); system('go', 'mod', 'edit', '--require' => "${Buildah}\@main") == 0 - or die "$ME: go mod edit failed\n"; + or die "$ME: go mod edit failed"; system('make', 'vendor') == 0 - or die "$ME: make vendor failed\n"; + or die "$ME: make vendor failed"; my $buildah_new = vendored_buildah(); print "-> buildah new = $buildah_new\n"; + + # Tweak .cirrus.yml so we run bud tests first in CI (to fail fast). + tweak_cirrus_test_order(); + + # 'make vendor' seems to git-add files under buildah itself, but not + # under other changed modules. Add those now, otherwise we fail + # the dirty-tree test in CI. + if (my @v = git('status', '--porcelain', '--untracked=all', 'vendor')) { + if (my @untracked = grep { /^\?\?\s/ } @v) { + my %repos = map { + s!^.*?vendor/[^/]+/([^/]+/[^/]+)/.*$!$1!; $_ => 1; + } @untracked; + my $repos = join(', ', sort keys %repos); + progress("Adding untracked files under $repos"); + git('add', 'vendor'); + } + } + + # Commit everything. git('commit', '-as', '-m', <<"END_COMMIT_MESSAGE"); [DO NOT MERGE] vendor in buildah \@ $buildah_new This is a JUNK COMMIT from $ME v$VERSION. DO NOT MERGE. This is just a way to keep the buildah-podman -vendoring in sync. See script --help for details. +vendoring in sync. Refer to: + + $Docs_URL END_COMMIT_MESSAGE + # And, finally, this has the highest possibility of failing + progress('Reapplying preserved patches'); + git('am', $Patch_File); + + # It worked! Clean up: remove our local die() handler and the patch file + undef $SIG{__DIE__}; + unlink $Patch_File; + # if buildah is unchanged, and we did not pull main, exit cleanly my $change_message = ''; if ($buildah_new eq $buildah_old) { if (! $rebased) { - progress("Nothing has changed (same buildah, same podman). Bye!"); - exit 0; + $change_message = "Nothing has changed (same buildah, same podman)."; + if ($force_testing) { + $change_message .= " Testing anyway due to --force-testing."; + } + else { + progress($change_message); + progress("Not much point to testing this, but use --force-testing to continue."); + exit 0; + } + } + else { + $change_message = "Podman has bumped, but Buildah is unchanged. There's probably not much point to testing this."; } - $change_message = "Podman has bumped, but Buildah is unchanged. There's probably not much point to testing this."; } else { my $samenew = ($rebased ? 'new' : 'same'); @@ -225,15 +299,6 @@ END_COMMIT_MESSAGE progress(" --- Reminder: $change_message"); } -######################### -# is_treadmill_commit # ARG (HEAD or HEAD^) commit message =~ treadmill -######################### -sub is_treadmill_commit { - my $commit_message = git('log', '-1', '--format=%s', @_); - print "[$commit_message]\n" if $verbose; - $commit_message =~ /buildah.*treadmill/; -} - ############### # pull_main # Switch to main, and pull latest from github ############### @@ -243,6 +308,58 @@ sub pull_main { git('pull', '-r', git_upstream(), 'main'); } +############################# +# tweak_cirrus_test_order # Run bud tests first, to fail fast & early +############################# +sub tweak_cirrus_test_order { + my $cirrus_yml = '.cirrus.yml'; + my $tmpfile = "$cirrus_yml.tmp.$$"; + unlink $tmpfile; + + progress("Tweaking test order in $cirrus_yml to run bud tests early"); + open my $in, '<', $cirrus_yml + or do { + warn "$ME: Cannot read $cirrus_yml: $!\n"; + warn "$ME: Will continue anyway\n"; + return; + }; + open my $out, '>'. $tmpfile + or die "$ME: Cannot create $tmpfile: $!\n"; + my $current_task = ''; + my $in_depend; + while (my $line = <$in>) { + chomp $line; + if ($line =~ /^(\S+)_task:$/) { + $current_task = $1; + undef $in_depend; + } + elsif ($line =~ /^(\s+)depends_on:$/) { + $in_depend = $1; + } + elsif ($in_depend && $line =~ /^($in_depend\s+-\s+)(\S+)/) { + if ($current_task eq 'buildah_bud_test') { + # Buildah bud test now depends on validate, so it runs early + $line = "${1}validate"; + } + elsif ($2 eq 'validate' && $current_task ne 'success') { + # Other tests that relied on validate, now rely on bud instead + $line = "${1}buildah_bud_test"; + } + } + else { + undef $in_depend; + } + + print { $out } $line, "\n"; + } + close $in; + close $out + or die "$ME: Error writing $tmpfile: $!\n"; + chmod 0644 => $tmpfile; + rename $tmpfile => $cirrus_yml + or die "$ME: Could not rename $tmpfile: $!\n"; +} + ############################ # build_and_check_podman # Run quick (local) sanity checks before pushing ############################ @@ -254,21 +371,51 @@ sub build_and_check_podman { system('make') == 0 or die "$ME: 'make' failed with new buildah. Cannot continue.\n"; - # See if any new options need man pages + # See if any new options need man pages. (C_Warning will highlight errs) progress('Cross-checking man pages...'); + print $C_Warning; $errs += system('hack/xref-helpmsgs-manpages'); + print $C_Reset; # Confirm that buildah-bud patches still apply. This requires knowing # the name of the directory created by the bud-tests script. progress("Confirming that buildah-bud-tests patches still apply..."); system('rm -rf test-buildah-*'); - $errs += system('test/buildah-bud/run-buildah-bud-tests', '--no-test'); - # Clean up - system('rm -rf test-buildah-*'); + if (system('test/buildah-bud/run-buildah-bud-tests', '--no-test')) { + # Error + ++$errs; + warn "$ME: Leaving test-buildah- directory for you to investigate\n"; + } + else { + # Patches apply cleanly. Clean up + system('rm -rf test-buildah-*'); + } return if !$errs; - warn "$ME: Errors found. Please address, then add to HEAD^ commit\n"; - die " ...see $ME --help for more information.\n"; + warn <<"END_WARN"; +$ME: Errors found. I have to stop now for you to fix them. + Your best bet now is: + 1) Find and fix whatever needs to be fixed; then + 2) git commit -am'fixme-fixme'; then + 3) git rebase -i main: + a) you are now in an editor window + b) move the new fixme-fixme commit up a line, to between the + 'buildah vendor treadmill' and 'vendor in buildah @ ...' lines + c) change 'pick' to 'squash' (or just 's') + d) save & quit to continue the rebase + e) back to a new editor window + f) change the commit message: remove fixme-fixme, add a description + of what you actually fixed. If possible, reference the PR (buildah + or podman) that introduced the failure + g) save & quit to continue the rebase + + Now, for good measure, rerun this script. + + For full documentation, refer to + + $Docs_URL +END_WARN + exit 1; } # END sync and its helpers @@ -281,14 +428,18 @@ sub do_pick { my $current_branch = git_current_branch(); # Confirm that current branch is a buildah-vendor one - head_is_buildah_vendor_commit(1); + assert_buildah_vendor_commit('HEAD'); progress("HEAD is a buildah vendor commit. Good."); # Identify and pull the treadmill PR my $treadmill_pr = treadmill_pr(); my $treadmill_branch = "$ME/pr$treadmill_pr/tmp$$"; progress("Fetching treadmill PR $treadmill_pr into $treadmill_branch"); - git('fetch', git_upstream(), "pull/$treadmill_pr/head:$treadmill_branch"); + git('fetch', '-q', git_upstream(), "pull/$treadmill_pr/head:$treadmill_branch"); + + # Compare merge bases of our branch and the treadmill one + progress("Checking merge bases"); + check_merge_bases($treadmill_pr, $treadmill_branch); # read buildah go.mod from it, and from current tree, and compare my $buildah_on_treadmill = vendored_buildah($treadmill_branch); @@ -309,7 +460,7 @@ sub do_pick { build_and_check_podman(); - progress("Looks good! Please 'git commit --amend' before pushing."); + progress("Looks good! Please 'git commit --amend' and edit commit message before pushing."); } ################## @@ -383,6 +534,51 @@ END_QUERY return $prs[0]{node}{number}; } +####################### +# check_merge_bases # It's OK if our branch is newer than treadmill +####################### +sub check_merge_bases { + my $treadmill_pr = shift; # e.g., 12345 + my $treadmill_branch = shift; # e.g., b-v-p/pr12345/tmpNNN + + # Fetch latest main, for accurate comparison + git('fetch', '-q', git_upstream(), 'main'); + + my $forkpoint_cur = git_forkpoint(); + my $forkpoint_treadmill = git_forkpoint($treadmill_branch); + + print "fork cur: $forkpoint_cur\nfork tm: $forkpoint_treadmill\n" + if $debug; + if ($forkpoint_cur eq $forkpoint_treadmill) { + progress("Nice. This branch is up-to-date wrt treadmill PR $treadmill_pr"); + return; + } + + # They differ. + if (git_is_ancestor($forkpoint_cur, $forkpoint_treadmill)) { + warn <<"END_WARN"; +$ME: treadmill PR $treadmill_pr is based on + a newer main than this branch. This means it might have + more up-to-date patches. + +END_WARN + + if ($force_old_main) { + warn "$ME: Proceeding due to --force-old-main\n"; + return; + } + + # Cannot continue. Clean up side branch, and bail. + git('branch', '-D', $treadmill_branch); + warn "$ME: You might want to consider rebasing on latest main.\n"; + warn "$ME: Aborting. Use --force-old-main to continue without rebasing.\n"; + exit 1; + } + else { + progress("Your branch is based on a newer main than treadmill PR $treadmill_pr. This is usually OK."); + } +} + ################# # cherry_pick # cherry-pick a commit, updating its commit message ################# @@ -390,7 +586,7 @@ sub cherry_pick { my $treadmill_pr = shift; # e.g., 12345 my $treadmill_branch = shift; # e.g., b-v-p/pr12345/tmpNNN - progress("Cherry-picking from $treadmill_pr^"); + progress("Cherry-picking from $treadmill_pr"); # Create a temp script. Do so in /var/tmp because sometimes $TMPDIR # (e.g. /tmp) has noexec. @@ -436,7 +632,7 @@ END_EDIT_SCRIPT or die "$ME: Error writing $editor: $!\n"; chmod 0755 => $editor; local $ENV{EDITOR} = $editor; - git('cherry-pick', '--allow-empty', '--edit', "$treadmill_branch^"); + git('cherry-pick', '--allow-empty', '--edit', $treadmill_branch); unlink $editor; } @@ -448,19 +644,42 @@ END_EDIT_SCRIPT # progress # Progris riport Dr Strauss says I shud rite down what I think ############## sub progress { - print $Highlight, "|\n+---> @_\n", $Reset; + print $C_Highlight, "|\n+---> @_\n", $C_Reset; } ####################### # assert_clean_repo # Don't even think of running with local changes ####################### sub assert_clean_repo { - my @changed = git('status', '--porcelain', '--untracked=no') - or return; + # Our patch file should only exist for brief moments during a sync run. + # If it exists at any other time, something has gone very wrong. + if (-e $Patch_File) { + warn <<"END_WARN"; +$ME: File exists: $Patch_File + + This means that something went very wrong during an earlier sync run. + Your git branch may be in an inconsistent state. Your work to date + may be lost. This file may be your only hope of recovering it. + + This is not something a script can resolve. You need to look at this + file, compare to your git HEAD, and manually reconcile any differences. +END_WARN + exit 1; + } - warn "$ME: Modified files in repo:\n"; - warn " $_\n" for @changed; - exit 1; + # OK so far. Now check for modified files. + if (my @changed = git('status', '--porcelain', '--untracked=no')) { + warn "$ME: Modified files in repo:\n"; + warn " $_\n" for @changed; + exit 1; + } + + # ...and for untracked files under vendor/ + if (my @v = git('status', '--porcelain', '--untracked=all', 'vendor')) { + warn "$ME: Untracked vendor files:\n"; + warn " $_\n" for @v; + exit 1; + } } ######################## @@ -474,6 +693,25 @@ sub git_current_branch() { return $b; } +################### +# git_forkpoint # Hash at which branch (default: cur) branched from main +################### +sub git_forkpoint { + return git('merge-base', '--fork-point', 'main', @_); +} + +##################### +# git_is_ancestor # Is hash1 an ancestor of hash2? +##################### +sub git_is_ancestor { + # Use system(), not git(), because we don't want to abort on exit status + my $rc = system('git', 'merge-base', '--is-ancestor', @_); + die "$ME: Cannot continue\n" if $? > 256; # e.g., Not a valid object + + # Translate shell 0/256 status to logical 1/0 + return !$rc; +} + ################## # git_upstream # Name of true github upstream ################## @@ -510,13 +748,16 @@ sub git { return wantarray ? @results : join("\n", @results); } -################################### -# head_is_buildah_vendor_commit # Returns 1 if HEAD is buildah vendor -################################### -sub head_is_buildah_vendor_commit { - my $fatal = shift; # in: if true, die upon anything missing +################################## +# assert_buildah_vendor_commit # Fails if input arg is not a buildah vendor +################################## +sub assert_buildah_vendor_commit { + my $ref = shift; # in: probably HEAD or HEAD^ - my @deltas = git('diff', '--name-only', 'HEAD^', 'HEAD'); + my @deltas = git('diff', '--name-only', "$ref^", $ref); + + # It's OK if there are no deltas, e.g. immediately after a buildah vendor PR + return if !@deltas; # It's OK if there are more modified files than just these. # It's not OK if any of these are missing. @@ -532,19 +773,11 @@ sub head_is_buildah_vendor_commit { push @missing, "no changes under $Buildah"; } - if (@missing) { - if ($fatal || $verbose) { - warn "$ME: HEAD does not look like a buildah vendor commit:\n"; - warn "$ME: - $_\n" for @missing; - if ($fatal) { - die "$ME: Cannot continue\n"; - } - warn "$ME: ...this might be okay, continuing anyway...\n"; - } - return; - } + return if !@missing; - return 1; + warn "$ME: $ref does not look like a buildah vendor commit:\n"; + warn "$ME: - $_\n" for @missing; + die "$ME: Cannot continue\n"; } ###################### diff --git a/hack/podman-registry-go/registry.go b/hack/podman-registry-go/registry.go index 095f6fb18..af8f3117c 100644 --- a/hack/podman-registry-go/registry.go +++ b/hack/podman-registry-go/registry.go @@ -31,10 +31,31 @@ type Registry struct { running bool } +// Options allows for customizing a registry. +type Options struct { + // Image - custom registry image. + Image string +} + // Start a new registry and return it along with it's image, user, password, and port. func Start() (*Registry, error) { + return StartWithOptions(nil) +} + +// StartWithOptions a new registry and return it along with it's image, user, password, and port. +func StartWithOptions(options *Options) (*Registry, error) { + if options == nil { + options = &Options{} + } + + var args []string + if options.Image != "" { + args = append(args, "-i", options.Image) + } + args = append(args, "start") + // Start a registry. - out, err := utils.ExecCmd(binary, "start") + out, err := utils.ExecCmd(binary, args...) if err != nil { return nil, errors.Wrapf(err, "error running %q: %s", binary, out) } |