Merge pull request #13937 from edsantiago/buildah_vendor_treadmill_script

Buildah Vendor Treadmill: the script

1 files changed, 582 insertions, 0 deletions

diff --git a/hack/buildah-vendor-treadmill b/hack/buildah-vendor-treadmill
new file mode 100755
index 000000000..1feffaa60
--- /dev/null
+++ b/hack/buildah-vendor-treadmill
@@ -0,0 +1,582 @@
+#!/usr/bin/perl
+#
+# buildah-vendor-treadmill - daily vendor of latest-buildah onto latest-podman
+#
+package Podman::BuildahVendorTreadmill;
+
+use v5.14;
+use utf8;
+use open qw( :encoding(UTF-8) :std );
+
+use strict;
+use warnings;
+
+use File::Temp                  qw(tempfile);
+use JSON;
+use LWP::UserAgent;
+
+(our $ME = $0) =~ s|.*/||;
+our $VERSION = '0.1';
+
+# For debugging, show data structures using DumpTree($var)
+#use Data::TreeDumper; $Data::TreeDumper::Displayaddress = 0;
+
+###############################################################################
+# BEGIN user-customizable section
+
+# github path to buildah
+our $Buildah = 'github.com/containers/buildah';
+
+# FIXME FIXME FIXME: add 'main'? I hope we never need this script for branches.
+our $Treadmill_PR_Title = 'DO NOT MERGE: buildah vendor treadmill';
+
+our $API_URL = 'https://api.github.com/graphql';
+
+# Use colors if available and if stdout is a tty
+our $Highlight = '';
+our $Reset = '';
+eval '
+    use Term::ANSIColor;
+    if (-t 1) {
+        $Highlight = color("green");
+        $Reset     = color("reset");
+    }
+    $SIG{__WARN__} = sub { print STDERR color("bold red"), "@_", $Reset; };
+
+';
+
+# END   user-customizable section
+###############################################################################
+
+###############################################################################
+# BEGIN boilerplate args checking, usage messages
+
+sub usage {
+    print  <<"END_USAGE";
+Usage: $ME [OPTIONS] [--sync | --pick ]
+
+$ME is (2022-04-20) **EXPERIMENTAL**
+
+$ME is intended to solve the problem of vendoring
+buildah into podman.
+
+Call me with one of two options:
+
+    --sync  The usual case. Mostly used by Ed. Called from a
+            development branch, this just updates everything so
+            we vendor in latest-buildah (main) on top of
+            latest-podman (main). With a few sanity checks.
+
+    --pick  Used for really-truly vendoring in a new buildah; will
+            cherry-pick a commit on your buildah-vendor working branch
+
+For latest documentation and best practices, please see:
+
+    https://github.com/containers/podman/wiki/Buildah-Vendor-Treadmill
+
+OPTIONS:
+
+  --help         display this message
+  --version      display program name and version
+END_USAGE
+
+    exit;
+}
+
+# Command-line options.  Note that this operates directly on @ARGV !
+our %action;
+our $debug   = 0;
+our $force   = 0;
+our $verbose = 0;
+our $NOT     = '';              # print "blahing the blah$NOT\n" if $debug
+sub handle_opts {
+    use Getopt::Long;
+    GetOptions(
+        'sync'       => sub { $action{sync}++ },
+        'pick'       => sub { $action{pick}++ },
+
+        'debug!'     => \$debug,
+        'dry-run|n!' => sub { $NOT = ' [NOT]' },
+        'force'      => \$force,
+        'verbose|v'  => \$verbose,
+
+        help         => \&usage,
+        version      => sub { print "$ME version $VERSION\n"; exit 0 },
+    ) or die "Try `$ME --help' for help\n";
+}
+
+# END   boilerplate args checking, usage messages
+###############################################################################
+
+############################## CODE BEGINS HERE ###############################
+
+# The term is "modulino".
+__PACKAGE__->main()                                     unless caller();
+
+# Main code.
+sub main {
+    # Note that we operate directly on @ARGV, not on function parameters.
+    # This is deliberate: it's because Getopt::Long only operates on @ARGV
+    # and there's no clean way to make it use @_.
+    handle_opts();                      # will set package globals
+
+    # Fetch command-line arguments.  Barf if too many.
+    # FIXME: if called with arg, that's the --sync branch?
+    # FIXME: if called with --pick + arg, that's the PR?
+    die "$ME: Too many arguments; try $ME --help\n"                 if @ARGV;
+
+    my @action = keys(%action);
+    die "$ME: Please invoke me with one of --sync or --pick\n"
+        if ! @action;
+    die "$ME: Please invoke me with ONLY one of --sync or --pick\n"
+        if @action > 1;
+
+    my $handler = __PACKAGE__->can("do_@action")
+        or die "$ME: No handler available for --@action\n";
+
+    # We've validated the command-line args. Before running action, check
+    # that repo is clean. None of our actions can be run on a dirty repo.
+    assert_clean_repo();
+
+    $handler->();
+}
+
+###############################################################################
+# BEGIN sync and its helpers
+
+sub do_sync {
+    # Preserve current branch name, so we can come back after switching to main
+    my $current_branch = git_current_branch();
+
+    my $buildah_old = vendored_buildah();
+    print "-> buildah old = $buildah_old\n";
+
+    # If HEAD is a buildah-vendor commit (usual case), drop it now.
+    if (head_is_buildah_vendor_commit()) {
+        if (is_treadmill_commit('HEAD^')) {
+            progress("HEAD is buildah vendor (as expected); dropping it...");
+            git('reset', '--hard', 'HEAD^');
+        }
+        else {
+            die "$ME: HEAD is a buildah commit, but HEAD^ is not a treadmill commit! Cannot continue.\n";
+        }
+    }
+    # HEAD must now be a treadmill commit
+    is_treadmill_commit('HEAD')
+        or die "$ME: HEAD is not a treadmill commit!\n";
+
+    # HEAD is now a change to buildah-tests. Now update main and rebase.
+    pull_main();
+    git('checkout', '-q', $current_branch);
+    my $forkpoint = git('merge-base', '--fork-point', 'main');
+    my $main_commit = git('rev-parse', 'main');
+    my $rebased;
+    if ($forkpoint eq $main_commit) {
+        progress("[Already rebased on podman main]");
+    }
+    else {
+        # --empty=keep may be needed after a --pick commit, when we've
+        # vendored a new buildah into podman and incorporated the treadmill
+        # commit. Since this is a perpetual-motion workflow, in which we
+        # keep an in-progress PR open at all times, we need a baseline
+        # commit even if it's empty.
+        progress("Rebasing on podman main...");
+        git('rebase', '--empty=keep', 'main');
+        # FIXME: rebase can fail after --pick. If it does, offer instructions.
+        $rebased = 1;
+    }
+
+    # We're now back on our treadmill branch, with one commit on top of main.
+    # Now vendor in latest buildah.
+    progress("Vendoring in buildah...");
+    system('go', 'mod', 'edit', '--require' => "${Buildah}\@main") == 0
+        or die "$ME: go mod edit failed\n";
+    system('make', 'vendor') == 0
+        or die "$ME: make vendor failed\n";
+    my $buildah_new = vendored_buildah();
+    print "-> buildah new = $buildah_new\n";
+    git('commit', '-as', '-m', <<"END_COMMIT_MESSAGE");
+[DO NOT MERGE] vendor in buildah \@ $buildah_new
+
+This is a JUNK COMMIT from $ME v$VERSION.
+
+DO NOT MERGE. This is just a way to keep the buildah-podman
+vendoring in sync. See script --help for details.
+END_COMMIT_MESSAGE
+
+    # if buildah is unchanged, and we did not pull main, exit cleanly
+    my $change_message = '';
+    if ($buildah_new eq $buildah_old) {
+        if (! $rebased) {
+            progress("Nothing has changed (same buildah, same podman). Bye!");
+            exit 0;
+        }
+        $change_message = "Podman has bumped, but Buildah is unchanged. There's probably not much point to testing this.";
+    }
+    else {
+        my $samenew = ($rebased ? 'new' : 'same');
+        $change_message = "New buildah, $samenew podman. Good candidate for pushing.";
+    }
+    progress($change_message);
+
+    build_and_check_podman();
+
+    progress("All OK. It's now up to you to 'git push --force'");
+    progress(" --- Reminder: $change_message");
+}
+
+#########################
+#  is_treadmill_commit  #  ARG (HEAD or HEAD^) commit message =~ treadmill
+#########################
+sub is_treadmill_commit {
+    my $commit_message = git('log', '-1', '--format=%s', @_);
+    print "[$commit_message]\n"         if $verbose;
+    $commit_message =~ /buildah.*treadmill/;
+}
+
+###############
+#  pull_main  #  Switch to main, and pull latest from github
+###############
+sub pull_main {
+    progress("Pulling podman main...");
+    git('checkout', '-q', 'main');
+    git('pull', '-r', git_upstream(), 'main');
+}
+
+############################
+#  build_and_check_podman  #  Run quick (local) sanity checks before pushing
+############################
+sub build_and_check_podman {
+    my $errs = 0;
+
+    # Confirm that we can still build podman
+    progress("Running 'make' to confirm that podman builds cleanly...");
+    system('make') == 0
+        or die "$ME: 'make' failed with new buildah. Cannot continue.\n";
+
+    # See if any new options need man pages
+    progress('Cross-checking man pages...');
+    $errs += system('hack/xref-helpmsgs-manpages');
+
+    # Confirm that buildah-bud patches still apply. This requires knowing
+    # the name of the directory created by the bud-tests script.
+    progress("Confirming that buildah-bud-tests patches still apply...");
+    system('rm -rf test-buildah-*');
+    $errs += system('test/buildah-bud/run-buildah-bud-tests', '--no-test');
+    # Clean up
+    system('rm -rf test-buildah-*');
+
+    return if !$errs;
+    warn "$ME: Errors found. Please address, then add to HEAD^ commit\n";
+    die  "    ...see $ME --help for more information.\n";
+}
+
+# END   sync and its helpers
+###############################################################################
+# BEGIN pick and its helpers
+#
+# This is what gets used on a real vendor-new-buildah PR
+
+sub do_pick {
+    my $current_branch = git_current_branch();
+
+    # Confirm that current branch is a buildah-vendor one
+    head_is_buildah_vendor_commit(1);
+    progress("HEAD is a buildah vendor commit. Good.");
+
+    # Identify and pull the treadmill PR
+    my $treadmill_pr = treadmill_pr();
+    my $treadmill_branch = "$ME/pr$treadmill_pr/tmp$$";
+    progress("Fetching treadmill PR $treadmill_pr into $treadmill_branch");
+    git('fetch', git_upstream(), "pull/$treadmill_pr/head:$treadmill_branch");
+
+    # read buildah go.mod from it, and from current tree, and compare
+    my $buildah_on_treadmill = vendored_buildah($treadmill_branch);
+    my $buildah_here         = vendored_buildah();
+    if ($buildah_on_treadmill ne $buildah_here) {
+        warn "$ME: Warning: buildah version mismatch:\n";
+        warn "$ME: on treadmill:   $buildah_on_treadmill\n";
+        warn "$ME: on this branch: $buildah_here\n";
+        # FIXME: should this require --force? A yes/no prompt?
+        # FIXME: I think not, because usual case will be a true tagged version
+        warn "$ME: Continuing anyway\n";
+    }
+
+    cherry_pick($treadmill_pr, $treadmill_branch);
+
+    # Clean up
+    git('branch', '-D', $treadmill_branch);
+
+    build_and_check_podman();
+
+    progress("Looks good! Please 'git commit --amend' before pushing.");
+}
+
+##################
+#  treadmill_pr  #  Returns ID of open podman PR with the desired subject
+##################
+sub treadmill_pr {
+    my $query = <<'END_QUERY';
+{
+  search(
+    query: "buildah vendor treadmill repo:containers/podman",
+    type: ISSUE,
+    first: 10
+  ) {
+    edges { node { ... on PullRequest { number state title } } }
+  }
+}
+END_QUERY
+
+    my $ua = LWP::UserAgent->new;
+    $ua->agent("$ME " . $ua->agent);              # Identify ourself
+
+    my %headers = (
+        'Accept'        => "application/vnd.github.antiope-preview+json",
+        'Content-Type'  => "application/json",
+    );
+
+    # Use github token if available, but don't require it. (All it does is
+    # bump up our throttling limit, which shouldn't be an issue) (unless
+    # someone invokes this script hundreds of times per minute).
+    if (my $token = $ENV{GITHUB_TOKEN}) {
+        $headers{Authorization} = "bearer $token";
+    }
+    $ua->default_header($_ => $headers{$_}) for keys %headers;
+
+    # Massage the query: escape quotes, put it all in one line, collapse spaces
+    $query =~ s/\"/\\"/g;
+    $query =~ s/\n/\\n/g;
+    $query =~ s/\s+/ /g;
+    # ...and now one more massage
+    my $postquery = qq/{ "query": \"$query\" }/;
+
+    print $postquery, "\n"            if $debug;
+    my $res = $ua->post($API_URL, Content => $postquery);
+    if ((my $code = $res->code) != 200) {
+        print $code, " ", $res->message, "\n";
+        exit 1;
+    }
+
+    # Got something. Confirm that it has all our required fields
+    my $content = decode_json($res->content);
+    use Data::Dump; dd $content         if $debug;
+    exists $content->{data}
+        or die "$ME: No '{data}' section in response\n";
+    exists $content->{data}{search}
+        or die "$ME: No '{data}{search}' section in response\n";
+    exists $content->{data}{search}{edges}
+        or die "$ME: No '{data}{search}{edges}' section in response\n";
+
+    # Confirm that there is exactly one such PR
+    my @prs = @{ $content->{data}{search}{edges} };
+    @prs > 0
+        or die "$ME: WEIRD! No 'buildah vendor treadmill' PRs found!\n";
+    @prs = grep { $_->{node}{title} eq $Treadmill_PR_Title } @prs
+        or die "$ME: No PRs found with title '$Treadmill_PR_Title'\n";
+    @prs = grep { $_->{node}{state} eq 'OPEN' } @prs
+        or die "$ME: Found '$Treadmill_PR_Title' PRs, but none are OPEN\n";
+    @prs == 1
+        or die "$ME: Multiple OPEN '$Treadmill_PR_Title' PRs found!\n";
+
+    # Yay. Found exactly one.
+    return $prs[0]{node}{number};
+}
+
+#################
+#  cherry_pick  #  cherry-pick a commit, updating its commit message
+#################
+sub cherry_pick {
+    my $treadmill_pr     = shift;       # e.g., 12345
+    my $treadmill_branch = shift;       # e.g., b-v-p/pr12345/tmpNNN
+
+    progress("Cherry-picking from $treadmill_pr^");
+
+    # Create a temp script. Do so in /var/tmp because sometimes $TMPDIR
+    # (e.g. /tmp) has noexec.
+    my ($fh, $editor) = tempfile( "$ME.edit-commit-message.XXXXXXXX", DIR => "/var/tmp" );
+    printf { $fh } <<'END_EDIT_SCRIPT', $ME, $VERSION, $treadmill_pr;
+#!/bin/bash
+
+if [[ -z "$1" ]]; then
+    echo "FATAL: Did not get called with an arg" >&2
+    exit 1
+fi
+
+msgfile=$1
+if [[ ! -e $msgfile ]]; then
+    echo "FATAL: git-commit file does not exist: $msgfile" >&2
+    exit 1
+fi
+
+tmpfile=$msgfile.tmp
+rm -f $tmpfile
+
+cat >$tmpfile <<EOF
+WIP: Fixes for vendoring Buildah
+
+This commit was automatically cherry-picked
+by %s v%s
+from the buildah vendor treadmill PR, #%s
+
+/vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv
+> The git commit message from that PR is below. Please review it,
+> edit as necessary, then remove this comment block.
+\^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+EOF
+
+# Strip the "DO NOT MERGE" header from the treadmill PR, print only
+# the "Changes as of YYYY-MM-DD" and subsequent lines
+sed -ne '/^Changes as of/,$ p' <$msgfile >>$tmpfile
+mv $tmpfile $msgfile
+
+END_EDIT_SCRIPT
+    close $fh
+        or die "$ME: Error writing $editor: $!\n";
+    chmod 0755 => $editor;
+    local $ENV{EDITOR} = $editor;
+    git('cherry-pick', '--allow-empty', '--edit', "$treadmill_branch^");
+    unlink $editor;
+}
+
+# END   pick and its helpers
+###############################################################################
+# BEGIN general-purpose helpers
+
+##############
+#  progress  #  Progris riport Dr Strauss says I shud rite down what I think
+##############
+sub progress {
+    print $Highlight, "|\n+---> @_\n", $Reset;
+}
+
+#######################
+#  assert_clean_repo  #  Don't even think of running with local changes
+#######################
+sub assert_clean_repo {
+    my @changed = git('status', '--porcelain', '--untracked=no')
+        or return;
+
+    warn "$ME: Modified files in repo:\n";
+    warn "    $_\n" for @changed;
+    exit 1;
+}
+
+########################
+#  git_current_branch  #  e.g., 'vendor_buildah'
+########################
+sub git_current_branch() {
+    my $b = git('rev-parse', '--abbrev-ref=strict', 'HEAD');
+
+    # There is no circumstance in which we can ever be called from main
+    die "$ME: must run from side branch, not main\n" if $b eq 'main';
+    return $b;
+}
+
+##################
+#  git_upstream  #  Name of true github upstream
+##################
+sub git_upstream {
+    for my $line (git('remote', '-v')) {
+        my ($remote, $url, $type) = split(' ', $line);
+        if ($url =~ m!github\.com.*containers/(podman|libpod)!) {
+            if ($type =~ /fetch/) {
+                return $remote;
+            }
+        }
+    }
+
+    die "$ME: did not find a remote with 'github.com/containers/podman'\n";
+}
+
+
+#########
+#  git  #  Run a git command
+#########
+sub git {
+    my @cmd = ('git', @_);
+    print "\$ @cmd\n"                   if $verbose || $debug;
+    open my $fh, '-|', @cmd
+        or die "$ME: Cannot fork: $!\n";
+    my @results;
+    while (my $line = <$fh>) {
+        chomp $line;
+        push @results, $line;
+    }
+    close $fh
+        or die "$ME: command failed: @cmd\n";
+
+    return wantarray ? @results : join("\n", @results);
+}
+
+###################################
+#  head_is_buildah_vendor_commit  #  Returns 1 if HEAD is buildah vendor
+###################################
+sub head_is_buildah_vendor_commit {
+    my $fatal = shift;                  # in: if true, die upon anything missing
+
+    my @deltas = git('diff', '--name-only', 'HEAD^', 'HEAD');
+
+    # It's OK if there are more modified files than just these.
+    # It's not OK if any of these are missing.
+    my @expect = qw(go.mod go.sum vendor/modules.txt);
+    my @missing;
+    for my $expect (@expect) {
+        if (! grep { $_ eq $expect } @deltas) {
+            push @missing, "$expect is unchanged";
+        }
+    }
+
+    if (! grep { m!^vendor/\Q$Buildah\E/! } @deltas) {
+        push @missing, "no changes under $Buildah";
+    }
+
+    if (@missing) {
+        if ($fatal || $verbose) {
+            warn "$ME: HEAD does not look like a buildah vendor commit:\n";
+            warn "$ME:  - $_\n" for @missing;
+            if ($fatal) {
+                die "$ME: Cannot continue\n";
+            }
+            warn "$ME: ...this might be okay, continuing anyway...\n";
+        }
+        return;
+    }
+
+    return 1;
+}
+
+######################
+#  vendored_buildah  #  Returns currently-vendored buildah
+######################
+sub vendored_buildah {
+    my $gomod_file = 'go.mod';
+    my @gomod;
+    if (@_) {
+        # Called with a branch argument; fetch that version of go.mod
+        $gomod_file = "@_:$gomod_file";
+        @gomod = git('show', $gomod_file);
+    }
+    else {
+        # No branch argument, read file
+        open my $fh, '<', $gomod_file
+          or die "$ME: Cannot read $gomod_file: $!\n";
+        while (my $line = <$fh>) {
+            chomp $line;
+            push @gomod, $line;
+        }
+        close $fh;
+    }
+
+    for my $line (@gomod) {
+        if ($line =~ m!^\s+\Q$Buildah\E\s+(\S+)!) {
+            return $1;
+        }
+    }
+
+    die "$ME: Could not find buildah in $gomod_file!\n";
+}
+
+# END   general-purpose helpers
+###############################################################################