#!/usr/bin/perl # # logformatter - highlight a Cirrus test log (ginkgo or bats) # # Adapted from https://raw.githubusercontent.com/edsantiago/greasemonkey/podman-ginkgo-highlight # package LibPod::CI::LogFormatter; use v5.14; use utf8; # Grumble. CI system doesn't have 'open' binmode STDIN, ':utf8'; binmode STDOUT, ':utf8'; use strict; use warnings; (our $ME = $0) =~ s|.*/||; our $VERSION = '0.1'; # Autoflush stdout $| = 1; # For debugging, show data structures using DumpTree($var) #use Data::TreeDumper; $Data::TreeDumper::Displayaddress = 0; ############################################################################### # BEGIN user-customizable section # Stylesheet for highlighting or de-highlighting parts of lines our $CSS = <<'END_CSS'; /* wrap long lines - don't require user to scroll right */ pre { line-break: normal; overflow-wrap: normal; white-space: pre-wrap; } /* synopsis table at top */ table.synopsis { border: none; border-collapse: collapse; margin-left: 2em; margin-top: 2ex; } .synopsis th { font-weight: normal; font-size: 110%; text-align: right; } .synopsis td { font-weight: bold; font-size: 120%; font-family: monospace; } /* test results */ .testname { font-size: 125%; color: #444; } .boring { color: #999; } .timestamp { color: #999; } .log-debug { color: #999; } .log-info { color: #333; } .log-warn { color: #f60; } .log-error { color: #900; font-weight: bold; } .log-skip { color: #F90; } .log-slow { background: #FF0; color: #000; font-weight: bold; } .subtest { background: #eee; } .subsubtest { color: #F39; font-weight: bold; } .string { color: #00c; } .command { font-weight: bold; color: #000; } .changed { color: #000; font-weight: bold; } /* links to source files: not as prominent as links to errors */ a.codelink:link { color: #000; } a.codelink:visited { color: #666; } a.codelink:hover { background: #000; color: #999; } /* The timing tests at bottom: remove underline, it's too cluttery. */ a.timing { text-decoration: none; } .timing:hover { background: #FF9; } /* highlight row for easy reading */ /* BATS styles */ .bats-passed { color: #393; } .bats-failed { color: #F00; font-weight: bold; } .bats-skipped { color: #F90; } .bats-log { color: #900; } .bats-log-esm { color: #b00; font-weight: bold; } .bats-summary { font-size: 150%; } /* error titles: display next to timestamp, not on separate line */ h2 { display: inline; } END_CSS # END user-customizable section ############################################################################### ############################################################################### # BEGIN boilerplate args checking, usage messages sub usage { print <<"END_USAGE"; Usage: $ME [OPTIONS] TEST_NAME $ME is a filter; it HTMLifies an input stream (presumably Ginkgo or BATS log results), writing HTML results to an output file but passing stdin unmodified to stdout. It is intended to run in the Cirrus CI environment. Parameters: TEST_NAME descriptive name; output file will be TEST_NAME.log.html OPTIONS: --help display this message --man display program man page --version display program name and version END_USAGE exit; } # Command-line options. Note that this operates directly on @ARGV ! our $debug = 0; our $force = 0; our $verbose = 0; our $NOT = ''; # print "blahing the blah$NOT\n" if $debug sub handle_opts { use Getopt::Long; GetOptions( 'debug!' => \$debug, 'dry-run|n!' => sub { $NOT = ' [NOT]' }, 'force' => \$force, 'verbose|v' => \$verbose, help => \&usage, version => sub { print "$ME version $VERSION\n"; exit 0 }, ) or die "Try `$ME --help' for help\n"; } # END boilerplate args checking, usage messages ############################################################################### ############################## CODE BEGINS HERE ############################### # The term is "modulino". __PACKAGE__->main() unless caller(); # Main code. sub main { # Note that we operate directly on @ARGV, not on function parameters. # This is deliberate: it's because Getopt::Long only operates on @ARGV # and there's no clean way to make it use @_. handle_opts(); # will set package globals # In case someone is tempted to run us on the command line die "$ME: this is a filter, not an interactive script\n" if -t *STDIN; # Fetch command-line arguments. Barf if too many. my $test_name = shift(@ARGV) or die "$ME: missing TEST_NAME argument; try $ME --help\n"; warn "$ME: Too many arguments; ignoring extras. try $ME --help\n" if @ARGV; format_log($test_name); } sub format_log { my $test_name = shift; # in: e.g. 'integration_test' my $outfile = "$test_name.log.html"; my $out_tmp = "$outfile.tmp.$$"; open my $out_fh, '>:utf8', $out_tmp or warn "$ME: Cannot create $out_tmp: $!\n"; # Boilerplate: HTML headers for output file print { $out_fh } <<"END_HTML" if $out_fh; $test_name END_HTML # Synopsis of this job: show job environment, links to PR and Cirrus print { $out_fh } "

Synopsis

\n
\n", job_synopsis($test_name), "
\n"; # FOR DEBUGGING: dump environment, but in HTML comments to not clutter # This is safe. There is a TOKEN envariable, but it's not sensitive. # There are no sensitive/secret values in our execution environment, # but we're careful anyway. $SECRET_ENV_RE is set in lib.sh my $filter_re = $ENV{SECRET_ENV_RE} || 'ACCOUNT|GC[EP]|PASSW|SECRET|TOKEN'; $filter_re .= '|BASH_FUNC'; # These are long and un-useful print { $out_fh } "\n"; for my $e (sort keys %ENV) { next if $e =~ /$filter_re/; my $val = escapeHTML($ENV{$e}); $val =~ s/--/--/g; # double dash not valid in comments printf { $out_fh } "\n", $e, $val; } # State variables my $previous_timestamp = ''; # timestamp of previous line my $cirrus_task; # Cirrus task number, used for linking my $git_commit; # git SHA, used for linking to source files my $in_failure; # binary flag: are we in an error dump? my $in_timing; # binary flag: are we in the timing section? my $after_divider = 0; # Count of lines after seeing '-----' my $current_output; # for removing duplication my $looks_like_bats; # binary flag: for detecting BATS results my $looks_like_python; # " " " : for colorizing python tests my %bats_count; # For summary line: count of pass/fail/skip # When running in cirrus, we have the commit SHA $git_commit = $ENV{CIRRUS_CHANGE_IN_REPO}; print { $out_fh } "
 \n";

    # Assume rootful prompt, check for rootless (here and in log itself, below)
    my $Prompt = '#';
    $Prompt = '$' if $test_name =~ /rootless/;

    # Main loop: read input, one line at a time, and write out reformatted
  LINE:
    while (my $line = ) {
        print $line;                    # Immediately dump back to stdout

        $Prompt = '$' if $line =~ /Runner executing .* as rootless /;

        # Remain robust in face of errors: always write stdout even if no HTML
        next LINE if ! $out_fh;

        chomp $line;
        $line =~ s/\0//g;               # Some log files have NULs????
        $line = escapeHTML($line);

        # Temporarily strip off leading timestamp
        $line =~ s/^(\[\+\d+s\]\s)//;
        my $timestamp = $1 || '';
        if ($previous_timestamp && $timestamp eq $previous_timestamp) {
            $timestamp = ' ' x length($timestamp);
        }
        elsif ($timestamp) {
            $previous_timestamp = $timestamp;
        }

        # Try to identify the git commit we're working with...
        if ($line =~ m!/define.gitCommit=([0-9a-f]+)!) {
            $git_commit = $1;
        }
        # ...so we can link to specific lines in source files
        if ($git_commit) {
            #           1  12  3                 34     4 5   526  6
            $line =~ s{^(.*)(\/(containers\/[^/]+)(\/\S+):(\d+))(.*)$}
                      {$1$2$6};

            # Same, for python errors
            #           1  12  3                 34         4             5   526
            $line =~ s{^(.*)(\/(containers\/[^/]+)(\/\S+\.py).*,\s+line\s+(\d+))(,\s+in.*)$}
                      {$1$2$6};
        }

        # Try to identify the cirrus task
        if ($line =~ /cirrus-task-(\d+)/) {
            $cirrus_task = $1;
        }

        # BATS handling. This will recognize num_tests both at start and end
        if ($line =~ /^1\.\.(\d+)$/) {
            $looks_like_bats = 1;
            $bats_count{expected_total} = $1;
            undef $looks_like_python;
        }
        # Since the number of tests can't always be predicted, recognize
        # some leading text strings that indicate BATS output to come.
        elsif ($line =~ /^TAP\s+version\s/ || $line =~ m!/test-apiv2!) {
            $looks_like_bats = 1;
            $bats_count{expected_total} = -1; # Expect to be overridden at end!
            undef $looks_like_python;
        }

        # 'python -m unittest' means we're starting some pythony stuff
        elsif ($line =~ m!/python.*\sunittest\s!) {
            $looks_like_python = 1;
            undef $looks_like_bats;
        }
        elsif ($looks_like_python && $line =~ m!Ran\s+(\d+)\s+tests\s+in\s!) {
            # End of python tests. However, we're still likely to see a
            # summary line saying 'OK' or 'FAILED'. Deal with that by
            # resetting $looks_like_python to 0, which the next elsif catches
            $bats_count{expected_total} += $1;
            $looks_like_python = 0;
            print { $out_fh } "\n"            if $in_failure;
            undef $in_failure;
        }
        elsif (defined($looks_like_python) && !$looks_like_python) {
            # The final python summary line. Show it in its appropriate color.
            if ($line =~ /^\s*(OK|FAILED)\s+\(/) {
                undef $looks_like_python;
                my $css = ($1 eq 'OK' ? 'passed' : 'failed');
                print { $out_fh } "$timestamp"
                    if $timestamp;
                print { $out_fh } "", $line, "\n";
                next LINE;
            }
        }

        if ($looks_like_bats) {
            my $css;

            # Readability: /long/path/to/podman -> podman (hover for full path)
            # Also make it boldface, to make commands stand out
            $line =~ s{^(#\s+(#|\$)\s+)(\S+/)(podman\S*)(\s.*)}
                      {$1$4$5};

            if    ($line =~ /^ok\s.*\s# skip/) { $css = 'skipped' }
            elsif ($line =~ /^ok\s/)           { $css = 'passed'  }
            elsif ($line =~ /^not\s+ok\s/)     { $css = 'failed'  }
            elsif ($line =~ /^#\s#\|\s/)       { $css = 'log-esm' }
            elsif ($line =~ /^#\s/)            { $css = 'log'     }

            # Link to source file. This is ugly: we have to hardcode 'podman'
            # and 'test/system' because there's no way to get them from log.
            #
            #          1  2      2               13     4         43           5
            $line =~ s{(in(\stest)?\s+file\s+\S+/)(\S+\.(bats|bash)),\s+line\s+(\d+)}{$1$3, line $5};

            if ($css) {
                # Make it linkable, e.g. foo.html#t--00001
                if ($line =~ /^(not\s+)?ok\s+(\d+)\s+(.*)/) {
                    $line = sprintf("%s", $2, $line);

                    push @{$bats_count{__fail_list}}, [ $2, $3 ] if $1;
                }
                $line = "$line";

                $bats_count{$css}++;
            }

            print { $out_fh } "$timestamp"
                if $timestamp;
            print { $out_fh } $line, "\n";
            next LINE;
        }
        elsif ($looks_like_python) {
            my $css;

            if    ($line =~ /\s\.\.\.\sskipped/) { $css = 'skipped' }
            elsif ($line =~ /\s\.\.\.\sok\s*$/)  { $css = 'passed'  }
            elsif ($line =~ /\s\.\.\.\sFAIL/)    { $css = 'failed'  }
            elsif ($line =~ /^\s*={40}/)         {
                # Begins a block of multiple lines including a stack trace
                print { $out_fh } "
\n" unless $in_failure; $in_failure = 1; } if ($css) { $line = "$line"; $bats_count{$css}++; } print { $out_fh } "$timestamp" if $timestamp; print { $out_fh } $line, "\n"; next LINE; } # Timing section at the bottom of the page if ($line =~ / timing results\s*$/) { $in_timing = 1; } elsif ($in_timing) { if ($line =~ /^(\S.*\S)\s+(\d+\.\d+)\s*$/) { my ($name, $time) = ($1, $2); my $id = make_id($1, 'timing'); # Try to column-align the timing numbers. Some test names # will be longer than our max - oh well. my $spaces = 80 - length(unescapeHTML($name)); $spaces = 1 if $spaces < 1; $spaces++ if $time < 10; my $spacing = ' ' x $spaces; $line = qq{$name$spacing$time}; } else { $in_timing = 0; } } # # Ginkgo error reformatting # if ($line =~ /^.{1,4} (Failure|Panic)( in .*)? \[/) { # Begins a block of multiple lines including a stack trace print { $out_fh } "
\n"; $in_failure = 1; } elsif ($line =~ /^-----------/) { if ($in_failure) { # Ends a stack trace block $in_failure = 0; print { $out_fh } "
\n"; } $after_divider = 1; print { $out_fh } "
\n
\n
\n";
            # Always show timestamp at start of each new test
            $previous_timestamp = '';
            next LINE;
        }
        # (bindings test sometimes emits 'Running' with leading bullet char)
        elsif ($line =~ s!^•?Running:!$Prompt!) {
            # Highlight the important (non-boilerplate) podman command.
            $line =~ s/\s+--remote\s+/ /g;      # --remote takes no args
            # Strip out the global podman options, but show them on hover
            $line =~ s{(\S+\/podman(-remote)?)((\s+--(root|runroot|runtime|tmpdir|storage-opt|conmon|cgroup-manager|network-backend|network-config-dir|storage-driver|events-backend|url) \S+)*)(.*)}{
                my ($full_path, $remote, $options, $args) = ($1, $2||'', $3, $6);

                $options =~ s/^\s+//;
                # Separate each '--foo bar' with newlines for readability
                $options =~ s/ --/\n--/g;
                qq{podman$remote [options]$args};
            }e;
            $current_output = '';
        }
        # Grrr. 'output:' usually just tells us what we already know.
        elsif ($line =~ /^output:/) {
            $current_output =~ s!^\s+|\s+$!!g;  # Trim leading/trailing blanks
            $current_output =~ s/\s+/ /g;       # Collapse multiple spaces
            if ($line eq "output: $current_output" || $line eq 'output: ') {
                next LINE;
            }
        }
        elsif ($line =~ /^Error:/ || $line =~ / level=(warning|error) /) {
            $line = "" . $line . "";
        }
        elsif ($line =~ /^panic:/) {
            $line = "" . $line . "";
        }
        else {
            $current_output .= ' ' . $line;
        }


        # Two lines after each divider, there's a test name. Make it
        # an anchor so we can link to it later.
        if ($after_divider++ == 2) {
            # Sigh. There is no actual marker. Assume that anything with
            ## two leading spaces then alpha or hyphen (not slashes) is
            ## a test name.
            if ($line =~ /^  [a-zA-Z-]/) {
                my $id = make_id($line, 'anchor');

                $line = "

$line

"; } } # Make SKIPPING and SLOW TEST visible $line =~ s!(\[SKIPPING\].*)!$1!; $line =~ s!(\[SLOW TEST.*\])!$1!; # Highlight test name when it appears in the middle of commands. # But make it boring, because we already have the test name in large # bold just above. (Except in skipped tests). $line =~ s!^(\s*)(\[It\]\s+.*)!$1$2!; # Failure name corresponds to a previously-seen block. # 1 2 2 3 3 14 4 if ($line =~ /^(\[(Fail|Panic!)\] .* \[(It|BeforeEach)\] )([A-Za-z-].*)/) { my ($lhs, $type, $ginkgo_fluff, $testname) = ($1, $2, $3, $4); my $id = make_id($testname, 'link'); $line = "$lhs$testname"; } print { $out_fh } "$timestamp" if $timestamp; print { $out_fh } $line, "\n"; } my $have_formatted_log; # Set on success if ($out_fh) { # Summary line for BATS tests if (keys %bats_count) { print { $out_fh } "
Summary:"; my $total = 0; my $comma = ''; for my $class (qw(passed failed skipped)) { if (my $n = $bats_count{$class}) { printf { $out_fh } "%s %d %s", $comma, $class, $n, ucfirst($class); $total += $n; $comma = ','; } } printf { $out_fh } ". Total tests: $total"; if (my $expected_total = $bats_count{expected_total}) { if ($total != $expected_total) { print { $out_fh } " (WARNING: expected $expected_total)"; } } print { $out_fh } "\n"; } print { $out_fh } "
\n"; # Did we find a cirrus task? Link back. if ($cirrus_task) { print { $out_fh } <<"END_HTML";

Cirrus task $cirrus_task

END_HTML } # FIXME: need a safe way to get TZ printf { $out_fh } <<"END_HTML", scalar(CORE::localtime);
Processed %s by $ME v$VERSION END_HTML if (close $out_fh) { if (rename $out_tmp => $outfile) { $have_formatted_log = 1; } else { warn "$ME: Could not rename $out_tmp: $!\n"; } } else { warn "$ME: Error writing $out_tmp: $!\n"; } } # Grumble. Github only shows the last N lines of the log... which is # anti-helpful when you want a quick synopsis of what failed. Write a # summary at the tail, to make it easier for humans to see what went wrong. if (my $fails = $bats_count{__fail_list}) { print "\n"; printf "Failed tests (%d):\n", scalar(@$fails); printf " - %d %s\n", @$_ for @$fails; } # If Cirrus magic envariables are available, write a link to results. # FIXME: it'd be so nice to make this a clickable live link. # # As of June 2022 we use the Cirrus API[1] as the source of our logs, # instead of linking directly to googleapis.com. This will allow us # to abstract cloud-specific details, so we can one day use Amazon cloud. # See #14569 for more info. # # [1] https://cirrus-ci.org/guide/writing-tasks/#latest-build-artifacts if ($have_formatted_log && $ENV{CIRRUS_TASK_ID}) { my $URL_BASE = "https://api.cirrus-ci.com"; my $task_id = $ENV{CIRRUS_TASK_ID}; # Link by *taskID*, not buildID + taskname. First, this is shorter # and less duplicaty. Second, and more important, buildID + taskname # is non-unique, and a link to a flake log will be clobbered. my $URL = "${URL_BASE}/v1/artifact/task/$task_id/html/${outfile}"; print "\n\nAnnotated results:\n $URL\n"; } } ############# # make_id # Given a test name, generate an anchor link name ############# sub make_id { my $name = shift; # in: test title my $type = shift; # in: differentiator (anchor, link) state %counter; $name =~ s/^\s+|\s+$//g; # strip leading/trailing whitespace $name =~ s/\&#\d+;//g; # 'doesn't' -> 'doesnt' $name =~ s/\"/-/g; # '"path"' -> '-path-' $name =~ s/[^a-zA-Z0-9_-]/-/g; # Convert non-alphanumeric to dash $name =~ s/-{3,}/-/g; # '------' to just '-' # Keep a running tally of how many times we've seen this identifier # for this given type! This lets us cross-match, in the bottom of # the page, the first/second/third failure of a given test. $name .= "--" . ++$counter{$type}{$name}; $name; } ############################################################################### # BEGIN job_synopsis and related helpers ################## # job_synopsis # Job details, links to github/cirrus ################## sub job_synopsis { my $subtest_name = shift; # e.g. integration_test my $s = <<"END_SYNOPSIS"; END_SYNOPSIS # PR 1234 - title of the pr my $pr_title = escapeHTML(_env_replace("{CIRRUS_CHANGE_TITLE}")); $s .= _tr("GitHub PR", sprintf("%s - %s", _a("{CIRRUS_PR}", "https://{CIRRUS_REPO_CLONE_HOST}/{CIRRUS_REPO_FULL_NAME}/pull/{CIRRUS_PR}"), $pr_title)); # PR author, if signed-off-by if (my $msg = _env_replace("{CIRRUS_COMMIT_MESSAGE}")) { while ($msg =~ /^Signed-off-by:\s+(\S.*\S)$/gmi) { $s .= _tr("Author", escapeHTML($1)); } } # eg "test fedora", "special_testing_rootless" # WARNING: As of 2020-10-05, $CIRRUS_TASK_NAME reflects the same # descriptive content as our $subtest_name argument (confirm via # cross-checking runner.sh:logformatter() vs cirrus.yml:&std_name_fmt). # If this ever becomes untrue, just add _tr("Subtest", $subtest_name). my $test_name = _env_replace("{CIRRUS_TASK_NAME}"); # (Special-case cleanup: Cirrus\ quotes\ spaces; remove for readability). $test_name =~ s/\\\s+/ /g; $s .= _tr("Test name", $test_name); # Link to further Cirrus results, e.g. other runs. # Build is mostly boring, it's usually TASK that we want to see. $s .= _tr("Cirrus", sprintf("Build %s / Task %s", _a("{CIRRUS_BUILD_ID}", "https://cirrus-ci.com/build/{CIRRUS_BUILD_ID}"), _a("{CIRRUS_TASK_ID}", "https://cirrus-ci.com/task/{CIRRUS_TASK_ID}"))); $s .= "
\n"; return $s; } sub _tr { my ($th, $td) = @_; return "$th:$td\n"; } sub _a { my ($name, $href) = map { _env_replace($_) } @_; if ($href =~ /UNDEFINED/) { return "$name ($href)"; } return "$name"; } sub _env_replace { my $s_in = shift; $s_in =~ s[\{(.*?)\}][$ENV{$1} || "[$1 UNDEFINED]"]ge; return $s_in; } # END job_synopsis and related helpers ############################################################################### # BEGIN html-formatting helpers sub escapeHTML { my $s = shift; state %chars; %chars = ('&' => '&', '<' => '<', '>' => '>', '"' => '"', "'" => ''') if keys(%chars) == 0; my $class = join('', sort keys %chars); $s =~ s/([$class])/$chars{$1}/ge; return $s; } sub unescapeHTML { my $s = shift; # We don't actually care about the character, only its length $s =~ s/\&\#?[a-z0-9]+;/./g; return $s; } # END html-formatting helpers ############################################################################### 1;