Buildah Vendor Treadmill: the script

This is the script I've been using (and tweaking) for the past
two weeks. It's ready for general review and use, with the
proviso that there are still corner cases I haven't tested.

See https://github.com/containers/podman/wiki/Buildah-Vendor-Treadmill
for an overview and instructions.

Signed-off-by: Ed Santiago <santiago@redhat.com>

1 files changed, 582 insertions, 0 deletions

diff --git a/hack/buildah-vendor-treadmill b/hack/buildah-vendor-treadmill
new file mode 100755
index 000000000..1feffaa60
--- /dev/null
+++ b/hack/buildah-vendor-treadmill
@@ -0,0 +1,582 @@
+#!/usr/bin/perl
+#
+# buildah-vendor-treadmill - daily vendor of latest-buildah onto latest-podman
+#
+package Podman::BuildahVendorTreadmill;
+
+use v5.14;
+use utf8;
+use open qw( :encoding(UTF-8) :std );
+
+use strict;
+use warnings;
+
+use File::Temp                  qw(tempfile);
+use JSON;
+use LWP::UserAgent;
+
+(our $ME = $0) =~ s|.*/||;
+our $VERSION = '0.1';
+
+# For debugging, show data structures using DumpTree($var)
+#use Data::TreeDumper; $Data::TreeDumper::Displayaddress = 0;
+
+###############################################################################
+# BEGIN user-customizable section
+
+# github path to buildah
+our $Buildah = 'github.com/containers/buildah';
+
+# FIXME FIXME FIXME: add 'main'? I hope we never need this script for branches.
+our $Treadmill_PR_Title = 'DO NOT MERGE: buildah vendor treadmill';
+
+our $API_URL = 'https://api.github.com/graphql';
+
+# Use colors if available and if stdout is a tty
+our $Highlight = '';
+our $Reset = '';
+eval '
+    use Term::ANSIColor;
+    if (-t 1) {
+        $Highlight = color("green");
+        $Reset     = color("reset");
+    }
+    $SIG{__WARN__} = sub { print STDERR color("bold red"), "@_", $Reset; };
+
+';
+
+# END   user-customizable section
+###############################################################################
+
+###############################################################################
+# BEGIN boilerplate args checking, usage messages
+
+sub usage {
+    print  <<"END_USAGE";
+Usage: $ME [OPTIONS] [--sync | --pick ]
+
+$ME is (2022-04-20) **EXPERIMENTAL**
+
+$ME is intended to solve the problem of vendoring
+buildah into podman.
+
+Call me with one of two options:
+
+    --sync  The usual case. Mostly used by Ed. Called from a
+            development branch, this just updates everything so
+            we vendor in latest-buildah (main) on top of
+            latest-podman (main). With a few sanity checks.
+
+    --pick  Used for really-truly vendoring in a new buildah; will
+            cherry-pick a commit on your buildah-vendor working branch
+
+For latest documentation and best practices, please see:
+
+    https://github.com/containers/podman/wiki/Buildah-Vendor-Treadmill
+
+OPTIONS:
+
+  --help         display this message
+  --version      display program name and version
+END_USAGE
+
+    exit;
+}
+
+# Command-line options.  Note that this operates directly on @ARGV !
+our %action;
+our $debug   = 0;
+our $force   = 0;
+our $verbose = 0;
+our $NOT     = '';              # print "blahing the blah$NOT\n" if $debug
+sub handle_opts {
+    use Getopt::Long;
+    GetOptions(
+        'sync'       => sub { $action{sync}++ },
+        'pick'       => sub { $action{pick}++ },
+
+        'debug!'     => \$debug,
+        'dry-run|n!' => sub { $NOT = ' [NOT]' },
+        'force'      => \$force,
+        'verbose|v'  => \$verbose,
+
+        help         => \&usage,
+        version      => sub { print "$ME version $VERSION\n"; exit 0 },
+    ) or die "Try `$ME --help' for help\n";
+}
+
+# END   boilerplate args checking, usage messages
+###############################################################################
+
+############################## CODE BEGINS HERE ###############################
+
+# The term is "modulino".
+__PACKAGE__->main()                                     unless caller();
+
+# Main code.
+sub main {
+    # Note that we operate directly on @ARGV, not on function parameters.
+    # This is deliberate: it's because Getopt::Long only operates on @ARGV
+    # and there's no clean way to make it use @_.
+    handle_opts();                      # will set package globals
+
+    # Fetch command-line arguments.  Barf if too many.
+    # FIXME: if called with arg, that's the --sync branch?
+    # FIXME: if called with --pick + arg, that's the PR?
+    die "$ME: Too many arguments; try $ME --help\n"                 if @ARGV;
+
+    my @action = keys(%action);
+    die "$ME: Please invoke me with one of --sync or --pick\n"
+        if ! @action;
+    die "$ME: Please invoke me with ONLY one of --sync or --pick\n"
+        if @action > 1;
+
+    my $handler = __PACKAGE__->can("do_@action")
+        or die "$ME: No handler available for --@action\n";
+
+    # We've validated the command-line args. Before running action, check
+    # that repo is clean. None of our actions can be run on a dirty repo.
+    assert_clean_repo();
+
+    $handler->();
+}
+
+###############################################################################
+# BEGIN sync and its helpers
+
+sub do_sync {
+    # Preserve current branch name, so we can come back after switching to main
+    my $current_branch = git_current_branch();
+
+    my $buildah_old = vendored_buildah();
+    print "-> buildah old = $buildah_old\n";
+
+    # If HEAD is a buildah-vendor commit (usual case), drop it now.
+    if (head_is_buildah_vendor_commit()) {
+        if (is_treadmill_commit('HEAD^')) {
+            progress("HEAD is buildah vendor (as expected); dropping it...");
+            git('reset', '--hard', 'HEAD^');
+        }
+        else {
+            die "$ME: HEAD is a buildah commit, but HEAD^ is not a treadmill commit! Cannot continue.\n";
+        }
+    }
+    # HEAD must now be a treadmill commit
+    is_treadmill_commit('HEAD')
+        or die "$ME: HEAD is not a treadmill commit!\n";
+
+    # HEAD is now a change to buildah-tests. Now update main and rebase.
+    pull_main();
+    git('checkout', '-q', $current_branch);
+    my $forkpoint = git('merge-base', '--fork-point', 'main');
+    my $main_commit = git('rev-parse', 'main');
+    my $rebased;
+    if ($forkpoint eq $main_commit) {
+        progress("[Already rebased on podman main]");
+    }
+    else {
+        # --empty=keep may be needed after a --pick commit, when we've
+        # vendored a new buildah into podman and incorporated the treadmill
+        # commit. Since this is a perpetual-motion workflow, in which we
+        # keep an in-progress PR open at all times, we need a baseline
+        # commit even if it's empty.
+        progress("Rebasing on podman main...");
+        git('rebase', '--empty=keep', 'main');
+        # FIXME: rebase can fail after --pick. If it does, offer instructions.
+        $rebased = 1;
+    }
+
+    # We're now back on our treadmill branch, with one commit on top of main.
+    # Now vendor in latest buildah.
+    progress("Vendoring in buildah...");
+    system('go', 'mod', 'edit', '--require' => "${Buildah}\@main") == 0
+        or die "$ME: go mod edit failed\n";
+    system('make', 'vendor') == 0
+        or die "$ME: make vendor failed\n";
+    my $buildah_new = vendored_buildah();
+    print "-> buildah new = $buildah_new\n";
+    git('commit', '-as', '-m', <<"END_COMMIT_MESSAGE");
+[DO NOT MERGE] vendor in buildah \@ $buildah_new
+
+This is a JUNK COMMIT from $ME v$VERSION.
+
+DO NOT MERGE. This is just a way to keep the buildah-podman
+vendoring in sync. See script --help for details.
+END_COMMIT_MESSAGE
+
+    # if buildah is unchanged, and we did not pull main, exit cleanly
+    my $change_message = '';
+    if ($buildah_new eq $buildah_old) {
+        if (! $rebased) {
+            progress("Nothing has changed (same buildah, same podman). Bye!");
+            exit 0;
+        }
+        $change_message = "Podman has bumped, but Buildah is unchanged. There's probably not much point to testing this.";
+    }
+    else {
+        my $samenew = ($rebased ? 'new' : 'same');
+        $change_message = "New buildah, $samenew podman. Good candidate for pushing.";
+    }
+    progress($change_message);
+
+    build_and_check_podman();
+
+    progress("All OK. It's now up to you to 'git push --force'");
+    progress(" --- Reminder: $change_message");
+}
+
+#########################
+#  is_treadmill_commit  #  ARG (HEAD or HEAD^) commit message =~ treadmill
+#########################
+sub is_treadmill_commit {
+    my $commit_message = git('log', '-1', '--format=%s', @_);
+    print "[$commit_message]\n"         if $verbose;
+    $commit_message =~ /buildah.*treadmill/;
+}
+
+###############
+#  pull_main  #  Switch to main, and pull latest from github
+###############
+sub pull_main {
+    progress("Pulling podman main...");
+    git('checkout', '-q', 'main');
+    git('pull', '-r', git_upstream(), 'main');
+}
+
+############################
+#  build_and_check_podman  #  Run quick (local) sanity checks before pushing
+############################
+sub build_and_check_podman {
+    my $errs = 0;
+
+    # Confirm that we can still build podman
+    progress("Running 'make' to confirm that podman builds cleanly...");
+    system('make') == 0
+        or die "$ME: 'make' failed with new buildah. Cannot continue.\n";
+
+    # See if any new options need man pages
+    progress('Cross-checking man pages...');
+    $errs += system('hack/xref-helpmsgs-manpages');
+
+    # Confirm that buildah-bud patches still apply. This requires knowing
+    # the name of the directory created by the bud-tests script.
+    progress("Confirming that buildah-bud-tests patches still apply...");
+    system('rm -rf test-buildah-*');
+    $errs += system('test/buildah-bud/run-buildah-bud-tests', '--no-test');
+    # Clean up
+    system('rm -rf test-buildah-*');
+
+    return if !$errs;
+    warn "$ME: Errors found. Please address, then add to HEAD^ commit\n";
+    die  "    ...see $ME --help for more information.\n";
+}
+
+# END   sync and its helpers
+###############################################################################
+# BEGIN pick and its helpers
+#
+# This is what gets used on a real vendor-new-buildah PR
+
+sub do_pick {
+    my $current_branch = git_current_branch();
+
+    # Confirm that current branch is a buildah-vendor one
+    head_is_buildah_vendor_commit(1);
+    progress("HEAD is a buildah vendor commit. Good.");
+
+    # Identify and pull the treadmill PR
+    my $treadmill_pr = treadmill_pr();
+    my $treadmill_branch = "$ME/pr$treadmill_pr/tmp$$";
+    progress("Fetching treadmill PR $treadmill_pr into $treadmill_branch");
+    git('fetch', git_upstream(), "pull/$treadmill_pr/head:$treadmill_branch");
+
+    # read buildah go.mod from it, and from current tree, and compare
+    my $buildah_on_treadmill = vendored_buildah($treadmill_branch);
+    my $buildah_here         = vendored_buildah();
+    if ($buildah_on_treadmill ne $buildah_here) {
+        warn "$ME: Warning: buildah version mismatch:\n";
+        warn "$ME: on treadmill:   $buildah_on_treadmill\n";
+        warn "$ME: on this branch: $buildah_here\n";
+        # FIXME: should this require --force? A yes/no prompt?
+        # FIXME: I think not, because usual case will be a true tagged version
+        warn "$ME: Continuing anyway\n";
+    }
+
+    cherry_pick($treadmill_pr, $treadmill_branch);
+
+    # Clean up
+    git('branch', '-D', $treadmill_branch);
+
+    build_and_check_podman();
+
+    progress("Looks good! Please 'git commit --amend' before pushing.");
+}
+
+##################
+#  treadmill_pr  #  Returns ID of open podman PR with the desired subject
+##################
+sub treadmill_pr {
+    my $query = <<'END_QUERY';
+{
+  search(
+    query: "buildah vendor treadmill repo:containers/podman",
+    type: ISSUE,
+    first: 10
+  ) {
+    edges { node { ... on PullRequest { number state title } } }
+  }
+}
+END_QUERY
+
+    my $ua = LWP::UserAgent->new;
+    $ua->agent("$ME " . $ua->agent);              # Identify ourself
+
+    my %headers = (
+        'Accept'        => "application/vnd.github.antiope-preview+json",
+        'Content-Type'  => "application/json",
+    );
+
+    # Use github token if available, but don't require it. (All it does is
+    # bump up our throttling limit, which shouldn't be an issue) (unless
+    # someone invokes this script hundreds of times per minute).
+    if (my $token = $ENV{GITHUB_TOKEN}) {
+        $headers{Authorization} = "bearer $token";
+    }
+    $ua->default_header($_ => $headers{$_}) for keys %headers;
+
+    # Massage the query: escape quotes, put it all in one line, collapse spaces
+    $query =~ s/\"/\\"/g;
+    $query =~ s/\n/\\n/g;
+    $query =~ s/\s+/ /g;
+    # ...and now one more massage
+    my $postquery = qq/{ "query": \"$query\" }/;
+
+    print $postquery, "\n"            if $debug;
+    my $res = $ua->post($API_URL, Content => $postquery);
+    if ((my $code = $res->code) != 200) {
+        print $code, " ", $res->message, "\n";
+        exit 1;
+    }
+
+    # Got something. Confirm that it has all our required fields
+    my $content = decode_json($res->content);
+    use Data::Dump; dd $content         if $debug;
+    exists $content->{data}
+        or die "$ME: No '{data}' section in response\n";
+    exists $content->{data}{search}
+        or die "$ME: No '{data}{search}' section in response\n";
+    exists $content->{data}{search}{edges}
+        or die "$ME: No '{data}{search}{edges}' section in response\n";
+
+    # Confirm that there is exactly one such PR
+    my @prs = @{ $content->{data}{search}{edges} };
+    @prs > 0
+        or die "$ME: WEIRD! No 'buildah vendor treadmill' PRs found!\n";
+    @prs = grep { $_->{node}{title} eq $Treadmill_PR_Title } @prs
+        or die "$ME: No PRs found with title '$Treadmill_PR_Title'\n";
+    @prs = grep { $_->{node}{state} eq 'OPEN' } @prs
+        or die "$ME: Found '$Treadmill_PR_Title' PRs, but none are OPEN\n";
+    @prs == 1
+        or die "$ME: Multiple OPEN '$Treadmill_PR_Title' PRs found!\n";
+
+    # Yay. Found exactly one.
+    return $prs[0]{node}{number};
+}
+
+#################
+#  cherry_pick  #  cherry-pick a commit, updating its commit message
+#################
+sub cherry_pick {
+    my $treadmill_pr     = shift;       # e.g., 12345
+    my $treadmill_branch = shift;       # e.g., b-v-p/pr12345/tmpNNN
+
+    progress("Cherry-picking from $treadmill_pr^");
+
+    # Create a temp script. Do so in /var/tmp because sometimes $TMPDIR
+    # (e.g. /tmp) has noexec.
+    my ($fh, $editor) = tempfile( "$ME.edit-commit-message.XXXXXXXX", DIR => "/var/tmp" );
+    printf { $fh } <<'END_EDIT_SCRIPT', $ME, $VERSION, $treadmill_pr;
+#!/bin/bash
+
+if [[ -z "$1" ]]; then
+    echo "FATAL: Did not get called with an arg" >&2
+    exit 1
+fi
+
+msgfile=$1
+if [[ ! -e $msgfile ]]; then
+    echo "FATAL: git-commit file does not exist: $msgfile" >&2
+    exit 1
+fi
+
+tmpfile=$msgfile.tmp
+rm -f $tmpfile
+
+cat >$tmpfile <<EOF
+WIP: Fixes for vendoring Buildah
+
+This commit was automatically cherry-picked
+by %s v%s
+from the buildah vendor treadmill PR, #%s
+
+/vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv
+> The git commit message from that PR is below. Please review it,
+> edit as necessary, then remove this comment block.
+\^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+EOF
+
+# Strip the "DO NOT MERGE" header from the treadmill PR, print only
+# the "Changes as of YYYY-MM-DD" and subsequent lines
+sed -ne '/^Changes as of/,$ p' <$msgfile >>$tmpfile
+mv $tmpfile $msgfile
+
+END_EDIT_SCRIPT
+    close $fh
+        or die "$ME: Error writing $editor: $!\n";
+    chmod 0755 => $editor;
+    local $ENV{EDITOR} = $editor;
+    git('cherry-pick', '--allow-empty', '--edit', "$treadmill_branch^");
+    unlink $editor;
+}
+
+# END   pick and its helpers
+###############################################################################
+# BEGIN general-purpose helpers
+
+##############
+#  progress  #  Progris riport Dr Strauss says I shud rite down what I think
+##############
+sub progress {
+    print $Highlight, "|\n+---> @_\n", $Reset;
+}
+
+#######################
+#  assert_clean_repo  #  Don't even think of running with local changes
+#######################
+sub assert_clean_repo {
+    my @changed = git('status', '--porcelain', '--untracked=no')
+        or return;
+
+    warn "$ME: Modified files in repo:\n";
+    warn "    $_\n" for @changed;
+    exit 1;
+}
+
+########################
+#  git_current_branch  #  e.g., 'vendor_buildah'
+########################
+sub git_current_branch() {
+    my $b = git('rev-parse', '--abbrev-ref=strict', 'HEAD');
+
+    # There is no circumstance in which we can ever be called from main
+    die "$ME: must run from side branch, not main\n" if $b eq 'main';
+    return $b;
+}
+
+##################
+#  git_upstream  #  Name of true github upstream
+##################
+sub git_upstream {
+    for my $line (git('remote', '-v')) {
+        my ($remote, $url, $type) = split(' ', $line);
+        if ($url =~ m!github\.com.*containers/(podman|libpod)!) {
+            if ($type =~ /fetch/) {
+                return $remote;
+            }
+        }
+    }
+
+    die "$ME: did not find a remote with 'github.com/containers/podman'\n";
+}
+
+
+#########
+#  git  #  Run a git command
+#########
+sub git {
+    my @cmd = ('git', @_);
+    print "\$ @cmd\n"                   if $verbose || $debug;
+    open my $fh, '-|', @cmd
+        or die "$ME: Cannot fork: $!\n";
+    my @results;
+    while (my $line = <$fh>) {
+        chomp $line;
+        push @results, $line;
+    }
+    close $fh
+        or die "$ME: command failed: @cmd\n";
+
+    return wantarray ? @results : join("\n", @results);
+}
+
+###################################
+#  head_is_buildah_vendor_commit  #  Returns 1 if HEAD is buildah vendor
+###################################
+sub head_is_buildah_vendor_commit {
+    my $fatal = shift;                  # in: if true, die upon anything missing
+
+    my @deltas = git('diff', '--name-only', 'HEAD^', 'HEAD');
+
+    # It's OK if there are more modified files than just these.
+    # It's not OK if any of these are missing.
+    my @expect = qw(go.mod go.sum vendor/modules.txt);
+    my @missing;
+    for my $expect (@expect) {
+        if (! grep { $_ eq $expect } @deltas) {
+            push @missing, "$expect is unchanged";
+        }
+    }
+
+    if (! grep { m!^vendor/\Q$Buildah\E/! } @deltas) {
+        push @missing, "no changes under $Buildah";
+    }
+
+    if (@missing) {
+        if ($fatal || $verbose) {
+            warn "$ME: HEAD does not look like a buildah vendor commit:\n";
+            warn "$ME:  - $_\n" for @missing;
+            if ($fatal) {
+                die "$ME: Cannot continue\n";
+            }
+            warn "$ME: ...this might be okay, continuing anyway...\n";
+        }
+        return;
+    }
+
+    return 1;
+}
+
+######################
+#  vendored_buildah  #  Returns currently-vendored buildah
+######################
+sub vendored_buildah {
+    my $gomod_file = 'go.mod';
+    my @gomod;
+    if (@_) {
+        # Called with a branch argument; fetch that version of go.mod
+        $gomod_file = "@_:$gomod_file";
+        @gomod = git('show', $gomod_file);
+    }
+    else {
+        # No branch argument, read file
+        open my $fh, '<', $gomod_file
+          or die "$ME: Cannot read $gomod_file: $!\n";
+        while (my $line = <$fh>) {
+            chomp $line;
+            push @gomod, $line;
+        }
+        close $fh;
+    }
+
+    for my $line (@gomod) {
+        if ($line =~ m!^\s+\Q$Buildah\E\s+(\S+)!) {
+            return $1;
+        }
+    }
+
+    die "$ME: Could not find buildah in $gomod_file!\n";
+}
+
+# END   general-purpose helpers
+###############################################################################