diff options
Diffstat (limited to 'contrib')
-rw-r--r-- | contrib/cirrus/lib.sh | 107 | ||||
-rwxr-xr-x | contrib/cirrus/logformatter | 47 | ||||
-rwxr-xr-x | contrib/cirrus/logformatter.t | 189 | ||||
-rw-r--r-- | contrib/cirrus/packer/fedora_packaging.sh | 141 | ||||
-rw-r--r-- | contrib/cirrus/packer/fedora_setup.sh | 120 | ||||
-rw-r--r-- | contrib/cirrus/packer/libpod_base_images.yml | 12 | ||||
-rw-r--r-- | contrib/cirrus/packer/libpod_images.yml | 5 | ||||
-rw-r--r-- | contrib/cirrus/packer/ubuntu_packaging.sh | 168 | ||||
-rw-r--r-- | contrib/cirrus/packer/ubuntu_setup.sh | 168 | ||||
-rwxr-xr-x | contrib/cirrus/setup_environment.sh | 8 | ||||
-rw-r--r-- | contrib/dependencies.txt | 34 | ||||
-rw-r--r-- | contrib/gate/Dockerfile | 39 | ||||
-rw-r--r-- | contrib/gate/README.md | 8 | ||||
-rw-r--r-- | contrib/podmanimage/stable/Dockerfile | 16 | ||||
-rw-r--r-- | contrib/podmanimage/stable/containers.conf | 11 | ||||
-rw-r--r-- | contrib/podmanimage/stable/manual/Containerfile | 13 | ||||
-rw-r--r-- | contrib/podmanimage/testing/Dockerfile | 16 | ||||
-rw-r--r-- | contrib/podmanimage/upstream/Dockerfile | 18 | ||||
-rw-r--r-- | contrib/spec/podman.spec.in | 23 |
19 files changed, 703 insertions, 440 deletions
diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh index 2031432b9..dd4f66f56 100644 --- a/contrib/cirrus/lib.sh +++ b/contrib/cirrus/lib.sh @@ -6,6 +6,11 @@ # Global details persist here source /etc/environment # not always loaded under all circumstances +# Automation environment doesn't automatically load for Ubuntu 18 +if [[ -r '/usr/share/automation/environment' ]]; then + source '/usr/share/automation/environment' +fi + # Under some contexts these values are not set, make sure they are. export USER="$(whoami)" export HOME="$(getent passwd $USER | cut -d : -f 6)" @@ -34,7 +39,6 @@ PACKER_BASE=${PACKER_BASE:-./contrib/cirrus/packer} # Important filepaths SETUP_MARKER_FILEPATH="${SETUP_MARKER_FILEPATH:-/var/tmp/.setup_environment_sh_complete}" AUTHOR_NICKS_FILEPATH="${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/git_authors_to_irc_nicks.csv" -BUILDAH_PACKAGES_FILEPATH="./contrib/cirrus/packages.sh" # in buildah repo. # Log remote-client system test varlink output here export VARLINK_LOG=/var/tmp/varlink.log @@ -60,23 +64,28 @@ PACKER_VER="1.4.2" # CSV of cache-image names to build (see $PACKER_BASE/libpod_images.json) # Base-images rarely change, define them here so they're out of the way. -export PACKER_BUILDS="${PACKER_BUILDS:-ubuntu-18,ubuntu-19,fedora-31,fedora-30}" +export PACKER_BUILDS="${PACKER_BUILDS:-ubuntu-18,ubuntu-19,fedora-32,fedora-31}" # Manually produced base-image names (see $SCRIPT_BASE/README.md) export UBUNTU_BASE_IMAGE="ubuntu-1910-eoan-v20200211" export PRIOR_UBUNTU_BASE_IMAGE="ubuntu-1804-bionic-v20200218" # Manually produced base-image names (see $SCRIPT_BASE/README.md) -export FEDORA_BASE_IMAGE="fedora-cloud-base-31-1-9-1578586410" -export PRIOR_FEDORA_BASE_IMAGE="fedora-cloud-base-30-1-2-1578586410" +export FEDORA_BASE_IMAGE="fedora-cloud-base-32-n-0-1586202964" +export PRIOR_FEDORA_BASE_IMAGE="fedora-cloud-base-31-1-9-1586202964" export BUILT_IMAGE_SUFFIX="${BUILT_IMAGE_SUFFIX:--$CIRRUS_REPO_NAME-${CIRRUS_BUILD_ID}}" # IN_PODMAN container image IN_PODMAN_IMAGE="quay.io/libpod/in_podman:$DEST_BRANCH" # Image for uploading releases UPLDREL_IMAGE="quay.io/libpod/upldrel:master" +# This is needed under some environments/contexts +SUDO='' +[[ "$UID" -eq 0 ]] || \ + SUDO='sudo -E' + # Avoid getting stuck waiting for user input export DEBIAN_FRONTEND="noninteractive" -SUDOAPTGET="ooe.sh sudo -E apt-get -qq --yes" -SUDOAPTADD="ooe.sh sudo -E add-apt-repository --yes" +SUDOAPTGET="$SUDO apt-get -qq --yes" +SUDOAPTADD="$SUDO add-apt-repository --yes" # Regex that finds enabled periodic apt configuration items PERIODIC_APT_RE='^(APT::Periodic::.+")1"\;' # Short-cuts for retrying/timeout calls @@ -110,6 +119,9 @@ OS_REL_VER="${OS_RELEASE_ID}-${OS_RELEASE_VER}" # Type of filesystem used for cgroups CG_FS_TYPE="$(stat -f -c %T /sys/fs/cgroup)" +# When building images, the version of automation tooling to install +INSTALL_AUTOMATION_VERSION=1.1.3 + # Installed into cache-images, supports overrides # by user-data in case of breakage or for debugging. CUSTOM_CLOUD_CONFIG_DEFAULTS="$GOSRC/$PACKER_BASE/cloud-init/$OS_RELEASE_ID/cloud.cfg.d" @@ -355,25 +367,18 @@ setup_rootless() { die 11 "Timeout exceeded waiting for localhost ssh capability" } -# Helper/wrapper script to only show stderr/stdout on non-zero exit -install_ooe() { - req_env_var SCRIPT_BASE - echo "Installing script to mask stdout/stderr unless non-zero exit." - sudo install -D -m 755 "$GOSRC/$SCRIPT_BASE/ooe.sh" /usr/local/bin/ooe.sh -} - # Grab a newer version of git from software collections # https://www.softwarecollections.org/en/ # and use it with a wrapper install_scl_git() { echo "Installing SoftwareCollections updated 'git' version." - ooe.sh sudo yum -y install rh-git29 - cat << "EOF" | sudo tee /usr/bin/git + ooe.sh $SUDO yum -y install rh-git29 + cat << "EOF" | $SUDO tee /usr/bin/git #!/bin/bash scl enable rh-git29 -- git $@ EOF - sudo chmod 755 /usr/bin/git + $SUDO chmod 755 /usr/bin/git } install_test_configs() { @@ -389,8 +394,7 @@ install_test_configs() { install -v -D -m 644 ./test/registries.conf /etc/containers/ } -# Remove all files (except conmon, for now) provided by the distro version of podman. -# Except conmon, for now as it's expected to eventually be packaged separately. +# Remove all files provided by the distro version of podman. # All VM cache-images used for testing include the distro podman because (1) it's # required for podman-in-podman testing and (2) it somewhat simplifies the task # of pulling in necessary prerequisites packages as the set can change over time. @@ -416,9 +420,9 @@ remove_packaged_podman_files() { if [[ "$OS_RELEASE_ID" =~ "ubuntu" ]] then - LISTING_CMD="sudo -E dpkg-query -L podman" + LISTING_CMD="$SUDO dpkg-query -L podman" else - LISTING_CMD='sudo rpm -ql podman' + LISTING_CMD='$SUDO rpm -ql podman' fi # yum/dnf/dpkg may list system directories, only remove files @@ -426,7 +430,7 @@ remove_packaged_podman_files() { do # Sub-directories may contain unrelated/valuable stuff if [[ -d "$fullpath" ]]; then continue; fi - ooe.sh sudo rm -vf "$fullpath" + ooe.sh $SUDO rm -vf "$fullpath" done # Be super extra sure and careful vs performant and completely safe @@ -449,63 +453,60 @@ systemd_banish() { $GOSRC/$PACKER_BASE/systemd_banish.sh } -install_buildah_packages() { - git clone https://github.com/containers/buildah.git /tmp/buildah - if [[ -r "$BUILDAH_PACKAGES_FILEPATH" ]]; then - source "$BUILDAH_PACKAGES_FILEPATH" - req_env_var UBUNTU_BUILDAH_PACKAGES FEDORA_BUILDAH_PACKAGES OS_RELEASE_ID - case "$OS_RELEASE_ID" in - fedora) - $BIGTO ooe.sh sudo dnf install -y ${FEDORA_BUILDAH_PACKAGES[@]} - ;; - ubuntu) - $LILTO $SUDOAPTGET update - $BIGTO $SUDOAPTGET install ${UBUNTU_BUILDAH_PACKAGES[@]} - ;; - *) bad_os_id_ver ;; - esac - else - warn "Could not find $BUILDAH_PACKAGES_FILEPATH in buildah repository root." +# This can be removed when the kernel bug fix is included in Fedora +workaround_bfq_bug() { + if [[ "$OS_RELEASE_ID" == "fedora" ]] && [[ $OS_RELEASE_VER -le 32 ]]; then + warn "Switching io scheduler to 'deadline' to avoid RHBZ 1767539" + warn "aka https://bugzilla.kernel.org/show_bug.cgi?id=205447" + echo "mq-deadline" | sudo tee /sys/block/sda/queue/scheduler > /dev/null + echo -n "IO Scheduler set to: " + $SUDO cat /sys/block/sda/queue/scheduler fi } +# Warning: DO NOT USE. +# This is called by other functions as the very last step during the VM Image build +# process. It's purpose is to "reset" the image, so all the first-boot operations +# happen at test runtime (like generating new ssh host keys, resizing partitions, etc.) _finalize() { set +e # Don't fail at the very end if [[ -d "$CUSTOM_CLOUD_CONFIG_DEFAULTS" ]] then echo "Installing custom cloud-init defaults" - sudo cp -v "$CUSTOM_CLOUD_CONFIG_DEFAULTS"/* /etc/cloud/cloud.cfg.d/ + $SUDO cp -v "$CUSTOM_CLOUD_CONFIG_DEFAULTS"/* /etc/cloud/cloud.cfg.d/ else echo "Could not find any files in $CUSTOM_CLOUD_CONFIG_DEFAULTS" fi echo "Re-initializing so next boot does 'first-boot' setup again." cd / - sudo rm -rf /var/lib/cloud/instanc* - sudo rm -rf /root/.ssh/* - sudo rm -rf /etc/ssh/*key* - sudo rm -rf /etc/ssh/moduli - sudo rm -rf /home/* - sudo rm -rf /tmp/* - sudo rm -rf /tmp/.??* - sudo sync - sudo fstrim -av + $SUDO rm -rf /var/lib/cloud/instanc* + $SUDO rm -rf /root/.ssh/* + $SUDO rm -rf /etc/ssh/*key* + $SUDO rm -rf /etc/ssh/moduli + $SUDO rm -rf /home/* + $SUDO rm -rf /tmp/* + $SUDO rm -rf /tmp/.??* + $SUDO sync + $SUDO fstrim -av } +# Called during VM Image setup, not intended for general use. rh_finalize() { set +e # Don't fail at the very end echo "Resetting to fresh-state for usage as cloud-image." PKG=$(type -P dnf || type -P yum || echo "") - sudo $PKG clean all - sudo rm -rf /var/cache/{yum,dnf} - sudo rm -f /etc/udev/rules.d/*-persistent-*.rules - sudo touch /.unconfigured # force firstboot to run + $SUDO $PKG clean all + $SUDO rm -rf /var/cache/{yum,dnf} + $SUDO rm -f /etc/udev/rules.d/*-persistent-*.rules + $SUDO touch /.unconfigured # force firstboot to run _finalize } +# Called during VM Image setup, not intended for general use. ubuntu_finalize() { set +e # Don't fail at the very end echo "Resetting to fresh-state for usage as cloud-image." $LILTO $SUDOAPTGET autoremove - sudo rm -rf /var/cache/apt + $SUDO rm -rf /var/cache/apt _finalize } diff --git a/contrib/cirrus/logformatter b/contrib/cirrus/logformatter index 738d2e19d..4bfe7b97f 100755 --- a/contrib/cirrus/logformatter +++ b/contrib/cirrus/logformatter @@ -52,12 +52,14 @@ a.codelink:hover { background: #000; color: #999; } a.timing { text-decoration: none; } /* BATS styles */ -.bats-ok { color: #393; } -.bats-notok { color: #F00; font-weight: bold; } -.bats-skip { color: #F90; } +.bats-passed { color: #393; } +.bats-failed { color: #F00; font-weight: bold; } +.bats-skipped { color: #F90; } .bats-log { color: #900; } .bats-log-esm { color: #b00; font-weight: bold; } +.bats-summary { font-size: 150%; } + /* error titles: display next to timestamp, not on separate line */ h2 { display: inline; } END_CSS @@ -169,7 +171,7 @@ window.addEventListener("load", scrollToBottom, false); </script> </head> <body> -<pre> +<pre> <!-- begin processed output --> END_HTML # State variables @@ -181,6 +183,7 @@ END_HTML my $after_divider = 0; # Count of lines after seeing '-----' my $current_output; # for removing duplication my $looks_like_bats; # binary flag: for detecting BATS results + my %bats_count; # For summary line: count of pass/fail/skip # Main loop: read input, one line at a time, and write out reformatted LINE: @@ -221,15 +224,16 @@ END_HTML } # BATS handling (used also for apiv2 tests, which emit TAP output) - if ($line =~ /^1\.\.\d+$/ || $line =~ m!/test-apiv2!) { + if ($line =~ /^1\.\.(\d+)$/ || $line =~ m!/test-apiv2!) { $looks_like_bats = 1; + $bats_count{expected_total} = $1; } if ($looks_like_bats) { my $css; - if ($line =~ /^ok\s.*\s# skip/) { $css = 'skip' } - elsif ($line =~ /^ok\s/) { $css = 'ok' } - elsif ($line =~ /^not\s+ok\s/) { $css = 'notok' } + if ($line =~ /^ok\s.*\s# skip/) { $css = 'skipped' } + elsif ($line =~ /^ok\s/) { $css = 'passed' } + elsif ($line =~ /^not\s+ok\s/) { $css = 'failed' } elsif ($line =~ /^#\s#\|\s/) { $css = 'log-esm' } elsif ($line =~ /^#\s/) { $css = 'log' } @@ -239,6 +243,8 @@ END_HTML $line = sprintf("<a name='t--%05d'>%s</a>", $2, $line); } $line = "<span class='bats-$css'>$line</span>"; + + $bats_count{$css}++; } print { $out_fh } "<span class=\"timestamp\">$timestamp</span>" @@ -354,7 +360,30 @@ END_HTML my $have_formatted_log; # Set on success if ($out_fh) { - print { $out_fh } "</pre>\n"; + # Summary line for BATS tests + if (keys %bats_count) { + print { $out_fh } "<hr/><span class='bats-summary'>Summary:"; + my $total = 0; + my $comma = ''; + for my $class (qw(passed failed skipped)) { + if (my $n = $bats_count{$class}) { + printf { $out_fh } "%s <span class='bats-%s'>%d %s</span>", + $comma, $class, $n, ucfirst($class); + $total += $n; + $comma = ','; + } + } + + printf { $out_fh } ". Total tests: $total"; + if (my $expected_total = $bats_count{expected_total}) { + if ($total != $expected_total) { + print { $out_fh } " <span class='bats-failed'>(WARNING: expected $expected_total)</span>"; + } + } + print { $out_fh } "</span>\n"; + } + + print { $out_fh } "</pre> <!-- end processed output -->\n"; # Did we find a cirrus task? Link back. if ($cirrus_task) { diff --git a/contrib/cirrus/logformatter.t b/contrib/cirrus/logformatter.t new file mode 100755 index 000000000..79c4563c2 --- /dev/null +++ b/contrib/cirrus/logformatter.t @@ -0,0 +1,189 @@ +#!/usr/bin/perl +# +# tests for logformatter +# +(our $ME = $0) =~ s|^.*/||; + +use v5.14; +use strict; +use warnings; + +use FindBin; +use File::Temp qw(tempdir); +use Test::More; + +# +# Read the test cases (see __END__ section below) +# +my @tests; +my $context = ''; +while (my $line = <DATA>) { + chomp $line; + + if ($line =~ /^==\s+(.*)/) { + push @tests, { name => $1, input => [], expect => [] }; + $context = ''; + } + elsif ($line =~ /^<<</) { + $context = 'input'; + } + elsif ($line =~ /^>>>/) { + $context = 'expect'; + } + elsif (@tests && $line) { + push @{ $tests[-1]{$context} }, $line; + } +} + +plan tests => scalar(@tests); + +my $tempdir = tempdir("logformatter-test.XXXXXX", TMPDIR => 1, CLEANUP => !$ENV{DEBUG}); + +chdir $tempdir + or die "$ME: Could not cd $tempdir: $!\n"; + +for my $t (@tests) { + my $name = $t->{name}; + (my $fname = $name) =~ s/\s+/_/g; + + open my $fh_out, '>', "$fname.txt" + or die "$ME: Cannot create $tempdir/$fname.txt: $!\n"; + print { $fh_out } "$_\n" for @{$t->{input}}; + close $fh_out + or die "$ME: Error writing $tempdir/$fname.txt: $!\n"; + + system("$FindBin::Bin/logformatter $fname <$fname.txt >/dev/null"); + open my $fh_in, '<', "$fname.log.html" + or die "$ME: Fatal: $fname: logformatter did not create .log.html\n"; + my @actual; + while (my $line = <$fh_in>) { + chomp $line; + push @actual, $line if $line =~ / begin processed output / .. $line =~ / end processed output /; + } + close $fh_in; + + # Strip off leading and trailing "<pre>" + shift @actual; pop @actual; + + # For debugging: preserve expected results + if ($ENV{DEBUG}) { + open my $fh_out, '>', "$fname.expect"; + print { $fh_out } "$_\n" for @{$t->{expect}}; + close $fh_out; + } + + is_deeply \@actual, $t->{expect}, $name; +} + +chdir '/'; + + + +__END__ + +== simple bats + +<<< +1..4 +ok 1 hi +ok 2 bye # skip no reason +not ok 3 fail +ok 4 blah +>>> +1..4 +<span class='bats-passed'><a name='t--00001'>ok 1 hi</a></span> +<span class='bats-skipped'><a name='t--00002'>ok 2 bye # skip no reason</a></span> +<span class='bats-failed'><a name='t--00003'>not ok 3 fail</a></span> +<span class='bats-passed'><a name='t--00004'>ok 4 blah</a></span> +<hr/><span class='bats-summary'>Summary: <span class='bats-passed'>2 Passed</span>, <span class='bats-failed'>1 Failed</span>, <span class='bats-skipped'>1 Skipped</span>. Total tests: 4</span> + + + + + + + +== simple ginkgo + +<<< +$SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP} +[08:26:19] START - All [+xxxx] lines that follow are relative to right now. +[+0002s] GO111MODULE=on go build -mod=vendor -gcflags 'all=-trimpath=/var/tmp/go/src/github.com/containers/libpod' -asmflags 'all=-trimpath=/var/tmp/go/src/github.com/containers/libpod' -ldflags '-X github.com/containers/libpod/libpod/define.gitCommit=40f5d8b1becd381c4e8283ed3940d09193e4fe06 -X github.com/containers/libpod/libpod/define.buildInfo=1582809981 -X github.com/containers/libpod/libpod/config._installPrefix=/usr/local -X github.com/containers/libpod/libpod/config._etcDir=/etc -extldflags ""' -tags " selinux systemd exclude_graphdriver_devicemapper seccomp varlink" -o bin/podman github.com/containers/libpod/cmd/podman +[+0103s] • +[+0103s] ------------------------------ +[+0103s] Podman pod restart +[+0103s] podman pod restart single empty pod +[+0103s] /var/tmp/go/src/github.com/containers/libpod/test/e2e/pod_restart_test.go:41 +[+0103s] [BeforeEach] Podman pod restart +[+0103s] /var/tmp/go/src/github.com/containers/libpod/test/e2e/pod_restart_test.go:18 +[+0103s] [It] podman pod restart single empty pod +[+0103s] /var/tmp/go/src/github.com/containers/libpod/test/e2e/pod_restart_test.go:41 +[+0103s] Running: /var/tmp/go/src/github.com/containers/libpod/bin/podman --storage-opt vfs.imagestore=/tmp/podman/imagecachedir --root /tmp/podman_test553496330/crio --runroot /tmp/podman_test553496330/crio-run --runtime /usr/bin/runc --conmon /usr/bin/conmon --cni-config-dir /etc/cni/net.d --cgroup-manager systemd --tmpdir /tmp/podman_test553496330 --events-backend file --storage-driver vfs pod create --infra=false --share +[+0103s] 4810be0cfbd42241e349dbe7d50fbc54405cd320a6637c65fd5323f34d64af89 +[+0103s] output: 4810be0cfbd42241e349dbe7d50fbc54405cd320a6637c65fd5323f34d64af89 +[+0103s] Running: /var/tmp/go/src/github.com/containers/libpod/bin/podman --storage-opt vfs.imagestore=/tmp/podman/imagecachedir --root /tmp/podman_test553496330/crio --runroot /tmp/podman_test553496330/crio-run --runtime /usr/bin/runc --conmon /usr/bin/conmon --cni-config-dir /etc/cni/net.d --cgroup-manager systemd --tmpdir /tmp/podman_test553496330 --events-backend file --storage-driver vfs pod restart 4810be0cfbd42241e349dbe7d50fbc54405cd320a6637c65fd5323f34d64af89 +[+0103s] Error: no containers in pod 4810be0cfbd42241e349dbe7d50fbc54405cd320a6637c65fd5323f34d64af89 have no dependencies, cannot start pod: no such container +[+0103s] output: +[+0103s] [AfterEach] Podman pod restart +[+0103s] /var/tmp/go/src/github.com/containers/libpod/test/e2e/pod_restart_test.go:28 +[+0103s] Running: /var/tmp/go/src/github.com/containers/libpod/bin/podman --storage-opt vfs.imagestore=/tmp/podman/imagecachedir --root /tmp/podman_test553496330/crio --runroot /tmp/podman_test553496330/crio-run --runtime /usr/bin/runc --conmon /usr/bin/conmon --cni-config-dir /etc/cni/net.d --cgroup-manager systemd --tmpdir /tmp/podman_test553496330 --events-backend file --storage-driver vfs pod rm -fa +[+0103s] 4810be0cfbd42241e349dbe7d50fbc54405cd320a6637c65fd5323f34d64af89 +[+0107s] • +[+0107s] ------------------------------ +[+0107s] podman system reset +>>> +$SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP} +[08:26:19] START - All [+xxxx] lines that follow are relative to right now. +<span class="timestamp">[+0002s] </span>GO111MODULE=on go build -mod=vendor -gcflags 'all=-trimpath=/var/tmp/go/src/github.com/containers/libpod' -asmflags 'all=-trimpath=/var/tmp/go/src/github.com/containers/libpod' -ldflags '-X github.com/containers/libpod/libpod/define.gitCommit=40f5d8b1becd381c4e8283ed3940d09193e4fe06 -X github.com/containers/libpod/libpod/define.buildInfo=1582809981 -X github.com/containers/libpod/libpod/config._installPrefix=/usr/local -X github.com/containers/libpod/libpod/config._etcDir=/etc -extldflags ""' -tags " selinux systemd exclude_graphdriver_devicemapper seccomp varlink" -o bin/podman github.com/containers/libpod/cmd/podman +<span class="timestamp">[+0103s] </span>• +</pre> +<hr /> +<pre> +<span class="timestamp">[+0103s] </span>Podman pod restart +<span class="timestamp"> </span><a name='t--podman-pod-restart-single-empty-pod--1'><h2> podman pod restart single empty pod</h2></a> +<span class="timestamp"> </span> /var/tmp/go/src/github.com<a class="codelink" href='https://github.com/containers/libpod/blob/40f5d8b1becd381c4e8283ed3940d09193e4fe06/test/e2e/pod_restart_test.go#L41'>/containers/libpod/test/e2e/pod_restart_test.go:41</a> +<span class="timestamp"> </span>[BeforeEach] Podman pod restart +<span class="timestamp"> </span> /var/tmp/go/src/github.com<a class="codelink" href='https://github.com/containers/libpod/blob/40f5d8b1becd381c4e8283ed3940d09193e4fe06/test/e2e/pod_restart_test.go#L18'>/containers/libpod/test/e2e/pod_restart_test.go:18</a> +<span class="timestamp"> </span>[It] podman pod restart single empty pod +<span class="timestamp"> </span> /var/tmp/go/src/github.com<a class="codelink" href='https://github.com/containers/libpod/blob/40f5d8b1becd381c4e8283ed3940d09193e4fe06/test/e2e/pod_restart_test.go#L41'>/containers/libpod/test/e2e/pod_restart_test.go:41</a> +<span class="timestamp"> </span>Running: <span title="/var/tmp/go/src/github.com/containers/libpod/bin/podman"><b>podman</b></span> <span class="boring" title="--storage-opt vfs.imagestore=/tmp/podman/imagecachedir +--root /tmp/podman_test553496330/crio +--runroot /tmp/podman_test553496330/crio-run +--runtime /usr/bin/runc +--conmon /usr/bin/conmon +--cni-config-dir /etc/cni/net.d +--cgroup-manager systemd +--tmpdir /tmp/podman_test553496330 +--events-backend file +--storage-driver vfs">[options]</span><b> pod create --infra=false --share</b> +<span class="timestamp"> </span>4810be0cfbd42241e349dbe7d50fbc54405cd320a6637c65fd5323f34d64af89 +<span class="timestamp"> </span>Running: <span title="/var/tmp/go/src/github.com/containers/libpod/bin/podman"><b>podman</b></span> <span class="boring" title="--storage-opt vfs.imagestore=/tmp/podman/imagecachedir +--root /tmp/podman_test553496330/crio +--runroot /tmp/podman_test553496330/crio-run +--runtime /usr/bin/runc +--conmon /usr/bin/conmon +--cni-config-dir /etc/cni/net.d +--cgroup-manager systemd +--tmpdir /tmp/podman_test553496330 +--events-backend file +--storage-driver vfs">[options]</span><b> pod restart 4810be0cfbd42241e349dbe7d50fbc54405cd320a6637c65fd5323f34d64af89</b> +<span class="timestamp"> </span><span class='log-warn'>Error: no containers in pod 4810be0cfbd42241e349dbe7d50fbc54405cd320a6637c65fd5323f34d64af89 have no dependencies, cannot start pod: no such container</span> +<span class="timestamp"> </span>output: +<span class="timestamp"> </span>[AfterEach] Podman pod restart +<span class="timestamp"> </span> /var/tmp/go/src/github.com<a class="codelink" href='https://github.com/containers/libpod/blob/40f5d8b1becd381c4e8283ed3940d09193e4fe06/test/e2e/pod_restart_test.go#L28'>/containers/libpod/test/e2e/pod_restart_test.go:28</a> +<span class="timestamp"> </span>Running: <span title="/var/tmp/go/src/github.com/containers/libpod/bin/podman"><b>podman</b></span> <span class="boring" title="--storage-opt vfs.imagestore=/tmp/podman/imagecachedir +--root /tmp/podman_test553496330/crio +--runroot /tmp/podman_test553496330/crio-run +--runtime /usr/bin/runc +--conmon /usr/bin/conmon +--cni-config-dir /etc/cni/net.d +--cgroup-manager systemd +--tmpdir /tmp/podman_test553496330 +--events-backend file +--storage-driver vfs">[options]</span><b> pod rm -fa</b> +<span class="timestamp"> </span>4810be0cfbd42241e349dbe7d50fbc54405cd320a6637c65fd5323f34d64af89 +<span class="timestamp">[+0107s] </span>• +</pre> +<hr /> +<pre> +<span class="timestamp">[+0107s] </span>podman system reset diff --git a/contrib/cirrus/packer/fedora_packaging.sh b/contrib/cirrus/packer/fedora_packaging.sh new file mode 100644 index 000000000..e80d48bc8 --- /dev/null +++ b/contrib/cirrus/packer/fedora_packaging.sh @@ -0,0 +1,141 @@ +#!/bin/bash + +# This script is called from fedora_setup.sh and various Dockerfiles. +# It's not intended to be used outside of those contexts. It assumes the lib.sh +# library has already been sourced, and that all "ground-up" package-related activity +# needs to be done, including repository setup and initial update. + +set -e + +echo "Updating/Installing repos and packages for $OS_REL_VER" + +source $GOSRC/$SCRIPT_BASE/lib.sh + +# Pre-req. to install automation tooing +$LILTO $SUDO dnf install -y git + +# Install common automation tooling (i.e. ooe.sh) +curl --silent --show-error --location \ + --url "https://raw.githubusercontent.com/containers/automation/master/bin/install_automation.sh" | \ + $SUDO env INSTALL_PREFIX=/usr/share /bin/bash -s - "$INSTALL_AUTOMATION_VERSION" +# Reload installed environment right now (happens automatically in a new process) +source /usr/share/automation/environment + +# Set this to 1 to NOT enable updates-testing repository +DISABLE_UPDATES_TESTING=${DISABLE_UPDATES_TESTING:0} + +# Do not enable update-stesting on the previous Fedora release +if ((DISABLE_UPDATES_TESTING!=0)); then + warn "Enabling updates-testing repository for image based on $FEDORA_BASE_IMAGE" + $LILTO $SUDO ooe.sh dnf install -y 'dnf-command(config-manager)' + $LILTO $SUDO ooe.sh dnf config-manager --set-enabled updates-testing +else + warn "NOT enabling updates-testing repository for image based on $PRIOR_FEDORA_BASE_IMAGE" +fi + +$BIGTO ooe.sh $SUDO dnf update -y + +REMOVE_PACKAGES=() +INSTALL_PACKAGES=(\ + autoconf + automake + bash-completion + bats + bridge-utils + btrfs-progs-devel + buildah + bzip2 + conmon + container-selinux + containernetworking-plugins + containers-common + criu + device-mapper-devel + dnsmasq + emacs-nox + file + findutils + fuse3 + fuse3-devel + gcc + git + glib2-devel + glibc-static + gnupg + go-md2man + golang + gpgme-devel + iproute + iptables + jq + libassuan-devel + libcap-devel + libmsi1 + libnet + libnet-devel + libnl3-devel + libseccomp + libseccomp-devel + libselinux-devel + libtool + libvarlink-util + lsof + make + msitools + nmap-ncat + ostree-devel + pandoc + podman + procps-ng + protobuf + protobuf-c + protobuf-c-devel + protobuf-devel + python + python3-dateutil + python3-psutil + python3-pytoml + rsync + selinux-policy-devel + skopeo + skopeo-containers + slirp4netns + unzip + vim + wget + which + xz + zip +) + +case "$OS_RELEASE_VER" in + 30) + INSTALL_PACKAGES+=(\ + atomic-registries + golang-github-cpuguy83-go-md2man + python2-future + runc + ) + REMOVE_PACKAGES+=(crun) + ;; + 31) + INSTALL_PACKAGES+=(crun) + REMOVE_PACKAGES+=(runc) + ;; + 32) + INSTALL_PACKAGES+=(crun) + REMOVE_PACKAGES+=(runc) + ;; + *) + bad_os_id_ver ;; +esac + +echo "Installing general build/test dependencies for Fedora '$OS_RELEASE_VER'" +$BIGTO ooe.sh $SUDO dnf install -y ${INSTALL_PACKAGES[@]} + +[[ ${#REMOVE_PACKAGES[@]} -eq 0 ]] || \ + $LILTO ooe.sh $SUDO dnf erase -y ${REMOVE_PACKAGES[@]} + +export GOPATH="$(mktemp -d)" +trap "$SUDO rm -rf $GOPATH" EXIT +ooe.sh $SUDO $GOSRC/hack/install_catatonit.sh diff --git a/contrib/cirrus/packer/fedora_setup.sh b/contrib/cirrus/packer/fedora_setup.sh index 81a46b13f..3830b3bc4 100644 --- a/contrib/cirrus/packer/fedora_setup.sh +++ b/contrib/cirrus/packer/fedora_setup.sh @@ -6,130 +6,26 @@ set -e # Load in library (copied by packer, before this script was run) -source /tmp/libpod/$SCRIPT_BASE/lib.sh +source $GOSRC/$SCRIPT_BASE/lib.sh -req_env_var SCRIPT_BASE PACKER_BUILDER_NAME GOSRC FEDORA_BASE_IMAGE OS_RELEASE_ID OS_RELEASE_VER +req_env_var SCRIPT_BASE PACKER_BASE INSTALL_AUTOMATION_VERSION PACKER_BUILDER_NAME GOSRC FEDORA_BASE_IMAGE OS_RELEASE_ID OS_RELEASE_VER -install_ooe - -export GOPATH="$(mktemp -d)" -trap "sudo rm -rf $GOPATH" EXIT - -$BIGTO ooe.sh sudo dnf update -y +workaround_bfq_bug # Do not enable update-stesting on the previous Fedora release if [[ "$FEDORA_BASE_IMAGE" =~ "${OS_RELEASE_ID}-cloud-base-${OS_RELEASE_VER}" ]]; then - warn "Enabling updates-testing repository for image based on $FEDORA_BASE_IMAGE" - $LILTO ooe.sh sudo dnf install -y 'dnf-command(config-manager)' - $LILTO ooe.sh sudo dnf config-manager --set-enabled updates-testing + DISABLE_UPDATES_TESTING=0 else - warn "NOT enabling updates-testing repository for image based on $PRIOR_FEDORA_BASE_IMAGE" + DISABLE_UPDATES_TESTING=1 fi -REMOVE_PACKAGES=() -INSTALL_PACKAGES=(\ - autoconf - automake - bash-completion - bats - bridge-utils - btrfs-progs-devel - bzip2 - conmon - container-selinux - containernetworking-plugins - containers-common - criu - device-mapper-devel - dnsmasq - emacs-nox - file - findutils - fuse3 - fuse3-devel - gcc - git - glib2-devel - glibc-static - gnupg - go-md2man - golang - gpgme-devel - iproute - iptables - jq - libassuan-devel - libcap-devel - libmsi1 - libnet - libnet-devel - libnl3-devel - libseccomp - libseccomp-devel - libselinux-devel - libtool - libvarlink-util - lsof - make - msitools - nmap-ncat - ostree-devel - pandoc - podman - procps-ng - protobuf - protobuf-c - protobuf-c-devel - protobuf-devel - protobuf-python - python - python3-dateutil - python3-psutil - python3-pytoml - rsync - runc - selinux-policy-devel - skopeo - skopeo-containers - slirp4netns - unzip - vim - wget - which - xz - zip -) -case "$OS_RELEASE_VER" in - 30) - INSTALL_PACKAGES+=(\ - atomic-registries - golang-github-cpuguy83-go-md2man - python2-future - runc - ) - REMOVE_PACKAGES+=(crun) - ;; - 31) - INSTALL_PACKAGES+=(crun) - REMOVE_PACKAGES+=(runc) - ;; - *) - bad_os_id_ver ;; -esac - -echo "Installing general build/test dependencies for Fedora '$OS_RELEASE_VER'" -$BIGTO ooe.sh sudo dnf install -y ${INSTALL_PACKAGES[@]} - -install_buildah_packages - -[[ "${#REMOVE_PACKAGES[@]}" -eq "0" ]] || \ - $LILTO ooe.sh sudo dnf erase -y ${REMOVE_PACKAGES[@]} +bash $PACKER_BASE/fedora_packaging.sh +# Load installed environment right now (happens automatically in a new process) +source /usr/share/automation/environment echo "Enabling cgroup management from containers" ooe.sh sudo setsebool container_manage_cgroup true -ooe.sh sudo /tmp/libpod/hack/install_catatonit.sh - # Ensure there are no disruptive periodic services enabled by default in image systemd_banish diff --git a/contrib/cirrus/packer/libpod_base_images.yml b/contrib/cirrus/packer/libpod_base_images.yml index 255723d57..a66fac31c 100644 --- a/contrib/cirrus/packer/libpod_base_images.yml +++ b/contrib/cirrus/packer/libpod_base_images.yml @@ -17,14 +17,14 @@ variables: PRIOR_UBUNTU_BASE_IMAGE: # Latest Fedora release - FEDORA_IMAGE_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/31/Cloud/x86_64/images/Fedora-Cloud-Base-31-1.9.x86_64.qcow2" - FEDORA_CSUM_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/31/Cloud/x86_64/images/Fedora-Cloud-31-1.9-x86_64-CHECKSUM" - FEDORA_BASE_IMAGE_NAME: 'fedora-cloud-base-31-1-9' + FEDORA_IMAGE_URL: "https://dl.fedoraproject.org/pub/fedora/linux/development/32/Cloud/x86_64/images/Fedora-Cloud-Base-32-20200406.n.0.x86_64.qcow2" + FEDORA_CSUM_URL: "https://dl.fedoraproject.org/pub/fedora/linux/development/32/Cloud/x86_64/images/Fedora-Cloud-32-x86_64-20200406.n.0-CHECKSUM" + FEDORA_BASE_IMAGE_NAME: 'fedora-cloud-base-32-n-0' # Prior Fedora release - PRIOR_FEDORA_IMAGE_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/30/Cloud/x86_64/images/Fedora-Cloud-Base-30-1.2.x86_64.qcow2" - PRIOR_FEDORA_CSUM_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/30/Cloud/x86_64/images/Fedora-Cloud-30-1.2-x86_64-CHECKSUM" - PRIOR_FEDORA_BASE_IMAGE_NAME: 'fedora-cloud-base-30-1-2' + PRIOR_FEDORA_IMAGE_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/31/Cloud/x86_64/images/Fedora-Cloud-Base-31-1.9.x86_64.qcow2" + PRIOR_FEDORA_CSUM_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/31/Cloud/x86_64/images/Fedora-Cloud-31-1.9-x86_64-CHECKSUM" + PRIOR_FEDORA_BASE_IMAGE_NAME: 'fedora-cloud-base-31-1-9' # The name of the image in GCE used for packer build libpod_images.yml IBI_BASE_NAME: 'image-builder-image' diff --git a/contrib/cirrus/packer/libpod_images.yml b/contrib/cirrus/packer/libpod_images.yml index 074a813af..e33ad775e 100644 --- a/contrib/cirrus/packer/libpod_images.yml +++ b/contrib/cirrus/packer/libpod_images.yml @@ -51,12 +51,12 @@ builders: source_image_family: 'prior-ubuntu-base' - <<: *gce_hosted_image - name: 'fedora-31' + name: 'fedora-32' source_image: '{{user `FEDORA_BASE_IMAGE`}}' source_image_family: 'fedora-base' - <<: *gce_hosted_image - name: 'fedora-30' + name: 'fedora-31' source_image: '{{user `PRIOR_FEDORA_BASE_IMAGE`}}' source_image_family: 'prior-fedora-base' @@ -71,6 +71,7 @@ provisioners: environment_vars: - 'PACKER_BUILDER_NAME={{build_name}}' - 'GOSRC=/tmp/libpod' + - 'PACKER_BASE={{user `PACKER_BASE`}}' - 'SCRIPT_BASE={{user `SCRIPT_BASE`}}' post-processors: diff --git a/contrib/cirrus/packer/ubuntu_packaging.sh b/contrib/cirrus/packer/ubuntu_packaging.sh new file mode 100644 index 000000000..b57bc95e9 --- /dev/null +++ b/contrib/cirrus/packer/ubuntu_packaging.sh @@ -0,0 +1,168 @@ +#!/bin/bash + +# This script is called from ubuntu_setup.sh and various Dockerfiles. +# It's not intended to be used outside of those contexts. It assumes the lib.sh +# library has already been sourced, and that all "ground-up" package-related activity +# needs to be done, including repository setup and initial update. + +set -e + +echo "Updating/Installing repos and packages for $OS_REL_VER" + +source $GOSRC/$SCRIPT_BASE/lib.sh + +echo "Updating/configuring package repositories." +$BIGTO $SUDOAPTGET update + +echo "Installing deps to add third-party repositories and automation tooling" +$LILTO $SUDOAPTGET install software-properties-common git curl + +# Install common automation tooling (i.e. ooe.sh) +curl --silent --show-error --location \ + --url "https://raw.githubusercontent.com/containers/automation/master/bin/install_automation.sh" | \ + $SUDO env INSTALL_PREFIX=/usr/share /bin/bash -s - "$INSTALL_AUTOMATION_VERSION" +# Reload installed environment right now (happens automatically in a new process) +source /usr/share/automation/environment + +$LILTO ooe.sh $SUDOAPTADD ppa:criu/ppa + +# Install newer version of golang +if [[ "$OS_RELEASE_VER" -eq "18" ]] +then + $LILTO ooe.sh $SUDOAPTADD ppa:longsleep/golang-backports +fi + +echo "Configuring/Instaling deps from Open build server" +VERSION_ID=$(source /etc/os-release; echo $VERSION_ID) +echo "deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_$VERSION_ID/ /" \ + | ooe.sh $SUDO tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list +ooe.sh curl -L -o /tmp/Release.key "https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/xUbuntu_${VERSION_ID}/Release.key" +ooe.sh $SUDO apt-key add - < /tmp/Release.key + +INSTALL_PACKAGES=(\ + apparmor + aufs-tools + autoconf + automake + bash-completion + bison + build-essential + buildah + bzip2 + conmon + containernetworking-plugins + containers-common + coreutils + cri-o-runc + criu + curl + dnsmasq + e2fslibs-dev + emacs-nox + file + gawk + gcc + gettext + git + go-md2man + golang + iproute2 + iptables + jq + libaio-dev + libapparmor-dev + libcap-dev + libdevmapper-dev + libdevmapper1.02.1 + libfuse-dev + libfuse2 + libglib2.0-dev + libgpgme11-dev + liblzma-dev + libnet1 + libnet1-dev + libnl-3-dev + libprotobuf-c-dev + libprotobuf-dev + libseccomp-dev + libseccomp2 + libselinux-dev + libsystemd-dev + libtool + libudev-dev + libvarlink + lsof + make + netcat + openssl + pkg-config + podman + protobuf-c-compiler + protobuf-compiler + python-future + python-minimal + python-protobuf + python3-dateutil + python3-pip + python3-psutil + python3-pytoml + python3-setuptools + rsync + runc + scons + skopeo + slirp4netns + socat + sudo + unzip + vim + wget + xz-utils + yum-utils + zip + zlib1g-dev +) + +if [[ $OS_RELEASE_VER -ge 19 ]] +then + INSTALL_PACKAGES+=(\ + bats + btrfs-progs + fuse3 + libbtrfs-dev + libfuse3-dev + ) +else + echo "Downloading version of bats with fix for a \$IFS related bug in 'run' command" + cd /tmp + BATS_URL='http://launchpadlibrarian.net/438140887/bats_1.1.0+git104-g1c83a1b-1_all.deb' + curl -L -O "$BATS_URL" + cd - + INSTALL_PACKAGES+=(\ + /tmp/$(basename $BATS_URL) + btrfs-tools + ) +fi + +# Do this at the last possible moment to avoid dpkg lock conflicts +echo "Upgrading all packages" +$BIGTO ooe.sh $SUDOAPTGET upgrade + +echo "Installing general testing and system dependencies" +# Necessary to update cache of newly added repos +$LILTO ooe.sh $SUDOAPTGET update +$BIGTO ooe.sh $SUDOAPTGET install ${INSTALL_PACKAGES[@]} + +export GOPATH="$(mktemp -d)" +trap "$SUDO rm -rf $GOPATH" EXIT +echo "Installing cataonit and libseccomp.sudo" +cd $GOSRC +ooe.sh $SUDO hack/install_catatonit.sh +ooe.sh $SUDO make install.libseccomp.sudo + +CRIO_RUNC_PATH="/usr/lib/cri-o-runc/sbin/runc" +if $SUDO dpkg -L cri-o-runc | grep -m 1 -q "$CRIO_RUNC_PATH" +then + echo "Linking $CRIO_RUNC_PATH to /usr/bin/runc for ease of testing." + $SUDO ln -f "$CRIO_RUNC_PATH" "/usr/bin/runc" +fi diff --git a/contrib/cirrus/packer/ubuntu_setup.sh b/contrib/cirrus/packer/ubuntu_setup.sh index 46e7a620f..2febbd265 100644 --- a/contrib/cirrus/packer/ubuntu_setup.sh +++ b/contrib/cirrus/packer/ubuntu_setup.sh @@ -8,171 +8,27 @@ set -e # Load in library (copied by packer, before this script was run) source $GOSRC/$SCRIPT_BASE/lib.sh -req_env_var SCRIPT_BASE +req_env_var SCRIPT_BASE PACKER_BASE INSTALL_AUTOMATION_VERSION PACKER_BUILDER_NAME GOSRC UBUNTU_BASE_IMAGE OS_RELEASE_ID OS_RELEASE_VER -install_ooe - -export GOPATH="$(mktemp -d)" -trap "sudo rm -rf $GOPATH" EXIT +# Ensure there are no disruptive periodic services enabled by default in image +systemd_banish # Stop disruption upon boot ASAP after booting echo "Disabling all packaging activity on boot" -# Don't let sed process sed's temporary files -_FILEPATHS=$(sudo ls -1 /etc/apt/apt.conf.d) -for filename in $_FILEPATHS; do \ +for filename in $(sudo ls -1 /etc/apt/apt.conf.d); do \ echo "Checking/Patching $filename" sudo sed -i -r -e "s/$PERIODIC_APT_RE/"'\10"\;/' "/etc/apt/apt.conf.d/$filename"; done -echo "Updating/configuring package repositories." -$BIGTO $SUDOAPTGET update - -echo "Upgrading all packages" -$BIGTO $SUDOAPTGET upgrade - -echo "Adding third-party repositories and PPAs" -$LILTO $SUDOAPTGET install software-properties-common -$LILTO $SUDOAPTADD ppa:criu/ppa -if [[ "$OS_RELEASE_VER" -eq "18" ]] -then - $LILTO $SUDOAPTADD ppa:longsleep/golang-backports -fi - -echo "Configuring/Instaling deps from Open build server" -VERSION_ID=$(source /etc/os-release; echo $VERSION_ID) -echo "deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_$VERSION_ID/ /" \ - | ooe.sh sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list -ooe.sh curl -L -o /tmp/Release.key "https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/xUbuntu_${VERSION_ID}/Release.key" -ooe.sh sudo apt-key add - < /tmp/Release.key - -INSTALL_PACKAGES=(\ - apparmor - aufs-tools - autoconf - automake - bash-completion - bison - build-essential - bzip2 - conmon - containernetworking-plugins - containers-common - coreutils - cri-o-runc - criu - curl - dnsmasq - e2fslibs-dev - emacs-nox - file - gawk - gcc - gettext - git - go-md2man - golang - iproute2 - iptables - jq - libaio-dev - libapparmor-dev - libcap-dev - libdevmapper-dev - libdevmapper1.02.1 - libfuse-dev - libfuse2 - libglib2.0-dev - libgpgme11-dev - liblzma-dev - libnet1 - libnet1-dev - libnl-3-dev - libprotobuf-c-dev - libprotobuf-dev - libseccomp-dev - libseccomp2 - libselinux-dev - libsystemd-dev - libtool - libudev-dev - libvarlink - lsof - make - netcat - openssl - pkg-config - podman - protobuf-c-compiler - protobuf-compiler - python-future - python-minimal - python-protobuf - python3-dateutil - python3-pip - python3-psutil - python3-pytoml - python3-setuptools - rsync - runc - scons - skopeo - slirp4netns - socat - unzip - vim - wget - xz-utils - yum-utils - zip - zlib1g-dev -) +bash $PACKER_BASE/ubuntu_packaging.sh -if [[ "$OS_RELEASE_VER" -ge "19" ]] -then - INSTALL_PACKAGES+=(\ - bats - btrfs-progs - fuse3 - libbtrfs-dev - libfuse3-dev - ) -else - echo "Downloading version of bats with fix for a \$IFS related bug in 'run' command" - cd /tmp - BATS_URL='http://launchpadlibrarian.net/438140887/bats_1.1.0+git104-g1c83a1b-1_all.deb' - curl -L -O "$BATS_URL" - cd - - INSTALL_PACKAGES+=(\ - /tmp/$(basename $BATS_URL) - btrfs-tools - ) - - echo "Forced Ubuntu 18 kernel to enable cgroup swap accounting." - SEDCMD='s/^GRUB_CMDLINE_LINUX="(.*)"/GRUB_CMDLINE_LINUX="\1 cgroup_enable=memory swapaccount=1"/g' - ooe.sh sudo sed -re "$SEDCMD" -i /etc/default/grub.d/* - ooe.sh sudo sed -re "$SEDCMD" -i /etc/default/grub - ooe.sh sudo update-grub -fi - -echo "Installing general testing and system dependencies" -# Necessary to update cache of newly added repos -$LILTO $SUDOAPTGET update -$BIGTO $SUDOAPTGET install ${INSTALL_PACKAGES[@]} - -install_buildah_packages - -echo "Installing cataonit and libseccomp.sudo" -ooe.sh sudo /tmp/libpod/hack/install_catatonit.sh -ooe.sh sudo make -C /tmp/libpod install.libseccomp.sudo - -# Ensure there are no disruptive periodic services enabled by default in image -systemd_banish +# Load installed environment right now (happens automatically in a new process) +source /usr/share/automation/environment -CRIO_RUNC_PATH="/usr/lib/cri-o-runc/sbin/runc" -if sudo dpkg -L cri-o-runc | grep -m 1 -q "$CRIO_RUNC_PATH" -then - echo "Linking $CRIO_RUNC_PATH to /usr/bin/runc for ease of testing." - sudo ln -f "$CRIO_RUNC_PATH" "/usr/bin/runc" -fi +echo "Making Ubuntu kernel to enable cgroup swap accounting as it is not the default." +SEDCMD='s/^GRUB_CMDLINE_LINUX="(.*)"/GRUB_CMDLINE_LINUX="\1 cgroup_enable=memory swapaccount=1"/g' +ooe.sh sudo sed -re "$SEDCMD" -i /etc/default/grub.d/* +ooe.sh sudo sed -re "$SEDCMD" -i /etc/default/grub +ooe.sh sudo update-grub ubuntu_finalize diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh index eceb80b00..6bec9625e 100755 --- a/contrib/cirrus/setup_environment.sh +++ b/contrib/cirrus/setup_environment.sh @@ -43,14 +43,8 @@ case "${OS_RELEASE_ID}" in fedora) # All SELinux distros need this for systemd-in-a-container setsebool container_manage_cgroup true - if [[ "$ADD_SECOND_PARTITION" == "true" ]]; then - bash "$SCRIPT_BASE/add_second_partition.sh" - fi - warn "Switching io scheduler to 'deadline' to avoid RHBZ 1767539" - warn "aka https://bugzilla.kernel.org/show_bug.cgi?id=205447" - echo "mq-deadline" > /sys/block/sda/queue/scheduler - cat /sys/block/sda/queue/scheduler + workaround_bfq_bug if [[ "$ADD_SECOND_PARTITION" == "true" ]]; then bash "$SCRIPT_BASE/add_second_partition.sh" diff --git a/contrib/dependencies.txt b/contrib/dependencies.txt new file mode 100644 index 000000000..5a6fa9834 --- /dev/null +++ b/contrib/dependencies.txt @@ -0,0 +1,34 @@ +# Fedora dependencies for building podman + +btrfs-progs-devel +bzip2 +container-selinux +containernetworking-cni +device-mapper-devel +findutils +git +glib2-devel +glibc-static +golang +gpgme-devel +iptables +libassuan-devel +libseccomp-devel +libselinux-devel +lsof +make +nmap-ncat +procps-ng +python +python3-dateutil +python3-pip +python3-psutil +python3-pytoml +python3-pyyaml +python3-varlink +rsync +slirp4netns +unzip +which +xz +zip diff --git a/contrib/gate/Dockerfile b/contrib/gate/Dockerfile index 4fddae557..f7cd8f2b3 100644 --- a/contrib/gate/Dockerfile +++ b/contrib/gate/Dockerfile @@ -1,38 +1,4 @@ FROM fedora:31 -RUN dnf -y install \ - btrfs-progs-devel \ - bzip2 \ - container-selinux \ - containernetworking-cni \ - device-mapper-devel \ - findutils \ - git \ - glib2-devel \ - glibc-static \ - golang \ - gpgme-devel \ - iptables \ - libassuan-devel \ - libseccomp-devel \ - libselinux-devel \ - lsof \ - make \ - nmap-ncat \ - procps-ng \ - python \ - python3-dateutil \ - python3-pip \ - python3-psutil \ - python3-pytoml \ - python3-pyyaml \ - python3-varlink \ - rsync \ - slirp4netns \ - unzip \ - which \ - xz \ - zip \ - && dnf clean all ENV GOPATH="/var/tmp/go" \ GOBIN="/var/tmp/go/bin" \ @@ -43,6 +9,11 @@ ENV GOPATH="/var/tmp/go" \ # Only needed for installing build-time dependencies, then will be removed COPY / $GOSRC +# Install packages from dependencies.txt, ignoring commented lines +RUN dnf -y install \ + $(grep "^[^#]" $GOSRC/contrib/dependencies.txt) \ + && dnf clean all + # Install dependencies RUN set -x && \ mkdir -p "$GOBIN" && \ diff --git a/contrib/gate/README.md b/contrib/gate/README.md index fe1205dc5..b2bc56023 100644 --- a/contrib/gate/README.md +++ b/contrib/gate/README.md @@ -1,6 +1,6 @@ ![PODMAN logo](../../logo/podman-logo-source.svg) -A standard container image for lint-checking and validating changes to the libpod -repository. The -[contributors guide contains the documentation for usage.](https://github.com/containers/libpod/blob/master/CONTRIBUTING.md#go-format-and-lint). Note that this container image is also utilized -in automation, see the file [.cirrus.yml](.cirrus.yml) +The "gate" image is a standard container image for lint-checking and validating +changes to the libpod repository. It must be built from the repository root as +[described in the contibutors guide](https://github.com/containers/libpod/blob/master/CONTRIBUTING.md#go-format-and-lint). +The image is also used in [CI/CD automation](../../.cirrus.yml). diff --git a/contrib/podmanimage/stable/Dockerfile b/contrib/podmanimage/stable/Dockerfile index c0c07d9d2..912d16e1c 100644 --- a/contrib/podmanimage/stable/Dockerfile +++ b/contrib/podmanimage/stable/Dockerfile @@ -11,16 +11,12 @@ FROM fedora:latest # Don't include container-selinux and remove # directories used by yum that are just taking # up space. -RUN useradd build; yum -y update; yum -y reinstall shadow-utils; yum -y install podman fuse-overlayfs --exclude container-selinux; rm -rf /var/cache /var/log/dnf* /var/log/yum.* +RUN useradd podman; yum -y update; yum -y reinstall shadow-utils; yum -y install podman fuse-overlayfs --exclude container-selinux; rm -rf /var/cache /var/log/dnf* /var/log/yum.* -# Adjust storage.conf to enable Fuse storage. -RUN sed -i -e 's|^#mount_program|mount_program|g' -e '/additionalimage.*/a "/var/lib/shared",' /etc/containers/storage.conf -RUN mkdir -p /var/lib/shared/overlay-images /var/lib/shared/overlay-layers; touch /var/lib/shared/overlay-images/images.lock; touch /var/lib/shared/overlay-layers/layers.lock +ADD https://raw.githubusercontent.com/containers/libpod/master/contrib/podmanimage/stable/containers.conf /etc/containers/ -# Adjust libpod.conf to write logging to a file -RUN sed -i 's/# events_logger = "journald"/events_logger = "file"/g' /usr/share/containers/libpod.conf +# chmod containers.conf and adjust storage.conf to enable Fuse storage. +RUN chmod 644 /etc/containers/containers.conf; sed -i -e 's|^#mount_program|mount_program|g' -e '/additionalimage.*/a "/var/lib/shared",' /etc/containers/storage.conf +RUN mkdir -p /var/lib/shared/overlay-images /var/lib/shared/overlay-layers; touch /var/lib/shared/overlay-images/images.lock; touch /var/lib/shared/overlay-layers/layers.lock -# Set up environment variables to note that this is -# not starting with usernamespace and default to -# isolate the filesystem with chroot. -ENV _BUILDAH_STARTED_IN_USERNS="" BUILDAH_ISOLATION=chroot +ENV _CONTAINERS_USERNS_CONFIGURED="" diff --git a/contrib/podmanimage/stable/containers.conf b/contrib/podmanimage/stable/containers.conf new file mode 100644 index 000000000..e6b806da3 --- /dev/null +++ b/contrib/podmanimage/stable/containers.conf @@ -0,0 +1,11 @@ +[containers] +netns="host" +userns="host" +ipcns="host" +utsns="host" +cgroupns="host" +cgroups="disabled" +[engine] +cgroup_manager = "cgroupfs" +events_logger="file" +runtime="crun" diff --git a/contrib/podmanimage/stable/manual/Containerfile b/contrib/podmanimage/stable/manual/Containerfile index d76d6d9b4..4375ea4f4 100644 --- a/contrib/podmanimage/stable/manual/Containerfile +++ b/contrib/podmanimage/stable/manual/Containerfile @@ -26,14 +26,11 @@ FROM fedora:latest COPY /tmp/podman-1.7.0-3.fc30.x86_64.rpm /tmp RUN yum -y install /tmp/podman-1.7.0-3.fc30.x86_64.rpm fuse-overlayfs --exclude container-selinux; rm -rf /var/cache /var/log/dnf* /var/log/yum.* /tmp/podman*.rpm -# Adjust storage.conf to enable Fuse storage. -RUN sed -i -e 's|^#mount_program|mount_program|g' -e '/additionalimage.*/a "/var/lib/shared",' /etc/containers/storage.conf +ADD https://raw.githubusercontent.com/containers/libpod/master/contrib/podmanimage/stable/containers.conf /etc/containers/ + +# chmod containers.conf and adjust storage.conf to enable Fuse storage. +RUN chmod 644 /etc/containers/containers.conf; sed -i -e 's|^#mount_program|mount_program|g' -e '/additionalimage.*/a "/var/lib/shared",' /etc/containers/storage.conf RUN mkdir -p /var/lib/shared/overlay-images /var/lib/shared/overlay-layers; touch /var/lib/shared/overlay-images/images.lock; touch /var/lib/shared/overlay-layers/layers.lock -# Adjust libpod.conf to write logging to a file -RUN sed -i 's/events_logger = "journald"/events_logger = "file"/g' /usr/share/containers/libpod.conf; mkdir -p /run/systemd/journal -# Set up environment variables to note that this is -# not starting with usernamespace and default to -# isolate the filesystem with chroot. -ENV _BUILDAH_STARTED_IN_USERNS="" BUILDAH_ISOLATION=chroot +ENV _CONTAINERS_USERNS_CONFIGURED="" diff --git a/contrib/podmanimage/testing/Dockerfile b/contrib/podmanimage/testing/Dockerfile index a8e7653f6..31265a0ea 100644 --- a/contrib/podmanimage/testing/Dockerfile +++ b/contrib/podmanimage/testing/Dockerfile @@ -13,16 +13,12 @@ FROM fedora:latest # Don't include container-selinux and remove # directories used by yum that are just taking # up space. -RUN useradd build; yum -y update; yum -y reinstall shadow-utils; yum -y install podman fuse-overlayfs --exclude container-selinux --enablerepo updates-testing; rm -rf /var/cache /var/log/dnf* /var/log/yum.* +RUN useradd podman; yum -y update; yum -y reinstall shadow-utils; yum -y install podman fuse-overlayfs --exclude container-selinux --enablerepo updates-testing; rm -rf /var/cache /var/log/dnf* /var/log/yum.* -# Adjust storage.conf to enable Fuse storage. -RUN sed -i -e 's|^#mount_program|mount_program|g' -e '/additionalimage.*/a "/var/lib/shared",' /etc/containers/storage.conf -RUN mkdir -p /var/lib/shared/overlay-images /var/lib/shared/overlay-layers; touch /var/lib/shared/overlay-images/images.lock; touch /var/lib/shared/overlay-layers/layers.lock +ADD https://raw.githubusercontent.com/containers/libpod/master/contrib/podmanimage/stable/containers.conf /etc/containers/ -# Adjust libpod.conf to write logging to a file -RUN sed -i 's/# events_logger = "journald"/events_logger = "file"/g' /usr/share/containers/libpod.conf +# chmod containers.conf and adjust storage.conf to enable Fuse storage. +RUN chmod 644 /etc/containers/containers.conf; sed -i -e 's|^#mount_program|mount_program|g' -e '/additionalimage.*/a "/var/lib/shared",' /etc/containers/storage.conf +RUN mkdir -p /var/lib/shared/overlay-images /var/lib/shared/overlay-layers; touch /var/lib/shared/overlay-images/images.lock; touch /var/lib/shared/overlay-layers/layers.lock -# Set up environment variables to note that this is -# not starting with usernamespace and default to -# isolate the filesystem with chroot. -ENV _BUILDAH_STARTED_IN_USERNS="" BUILDAH_ISOLATION=chroot +ENV _CONTAINERS_USERNS_CONFIGURED="" diff --git a/contrib/podmanimage/upstream/Dockerfile b/contrib/podmanimage/upstream/Dockerfile index 847097920..541670aa2 100644 --- a/contrib/podmanimage/upstream/Dockerfile +++ b/contrib/podmanimage/upstream/Dockerfile @@ -17,7 +17,7 @@ ENV GOPATH=/root/podman # to the container. # Finally remove the podman directory and a few other packages # that are needed for building but not running Podman -RUN useradd build; yum -y update; yum -y reinstall shadow-utils; yum -y install --exclude container-selinux \ +RUN useradd podman; yum -y update; yum -y reinstall shadow-utils; yum -y install --exclude container-selinux \ --enablerepo=updates-testing \ btrfs-progs-devel \ containernetworking-cni \ @@ -37,7 +37,7 @@ RUN useradd build; yum -y update; yum -y reinstall shadow-utils; yum -y install libselinux-devel \ make \ pkgconfig \ - runc \ + crun \ fuse-overlayfs \ fuse3 \ containers-common; \ @@ -59,18 +59,14 @@ RUN useradd build; yum -y update; yum -y reinstall shadow-utils; yum -y install mkdir -p /etc/cni/net.d; \ curl -qsSL https://raw.githubusercontent.com/containers/libpod/master/cni/87-podman-bridge.conflist | tee /etc/cni/net.d/99-loopback.conf; \ mkdir -p /usr/share/containers; \ - cp $GOPATH/src/github.com/containers/libpod/libpod.conf /usr/share/containers; \ - # Adjust libpod.conf to write logging to a file - sed -i 's/# events_logger = "journald"/events_logger = "file"/g' /usr/share/containers/libpod.conf; \ rm -rf /root/podman/*; \ yum -y remove git golang go-md2man make; \ yum clean all; -# Adjust storage.conf to enable Fuse storage. -RUN sed -i -e 's|^#mount_program|mount_program|g' -e '/additionalimage.*/a "/var/lib/shared",' /etc/containers/storage.conf +ADD https://raw.githubusercontent.com/containers/libpod/master/contrib/podmanimage/stable/containers.conf /etc/containers/ + +# chmod containers.conf and adjust storage.conf to enable Fuse storage. +RUN chmod 644 /etc/containers/containers.conf; sed -i -e 's|^#mount_program|mount_program|g' -e '/additionalimage.*/a "/var/lib/shared",' /etc/containers/storage.conf RUN mkdir -p /var/lib/shared/overlay-images /var/lib/shared/overlay-layers; touch /var/lib/shared/overlay-images/images.lock; touch /var/lib/shared/overlay-layers/layers.lock -# Set up environment variables to note that this is -# not starting with usernamespace and default to -# isolate the filesystem with chroot. -ENV _BUILDAH_STARTED_IN_USERNS="" BUILDAH_ISOLATION=chroot +ENV _CONTAINERS_USERNS_CONFIGURED="" diff --git a/contrib/spec/podman.spec.in b/contrib/spec/podman.spec.in index afc50f854..1dfbdf208 100644 --- a/contrib/spec/podman.spec.in +++ b/contrib/spec/podman.spec.in @@ -377,12 +377,6 @@ Man pages for the %{name} commands # untar conmon tar zxf %{SOURCE1} -sed -i 's/install.remote: podman-remote/install.remote:/' Makefile -sed -i 's/install.bin: podman/install.bin:/' Makefile -%if %{with doc} -sed -i 's/install.man: docs/install.man:/' Makefile -%endif - %build mkdir _build pushd _build @@ -417,22 +411,15 @@ popd %install install -dp %{buildroot}%{_unitdir} install -dp %{buildroot}%{_usr}/lib/systemd/user -%if %{with doc} -PODMAN_VERSION=%{version} %{__make} PREFIX=%{buildroot}%{_prefix} ETCDIR=%{buildroot}%{_sysconfdir} \ - install.bin \ - install.remote \ - install.man \ - install.cni \ - install.systemd \ - install.completions -%else PODMAN_VERSION=%{version} %{__make} PREFIX=%{buildroot}%{_prefix} ETCDIR=%{buildroot}%{_sysconfdir} \ - install.bin \ - install.remote \ + install.bin-nobuild \ + install.remote-nobuild \ +%if %{with doc} + install.man-nobuild \ +%endif install.cni \ install.systemd \ install.completions -%endif mv pkg/hooks/README.md pkg/hooks/README-hooks.md |