diff options
Diffstat (limited to 'test')
| -rwxr-xr-x | test/buildah-bud/apply-podman-deltas | 34 | ||||
| -rw-r--r-- | test/buildah-bud/buildah-tests.diff | 36 | ||||
| -rw-r--r-- | test/e2e/benchmarks_test.go | 79 |
3 files changed, 99 insertions, 50 deletions
diff --git a/test/buildah-bud/apply-podman-deltas b/test/buildah-bud/apply-podman-deltas index b30c95d59..66e8687de 100755 --- a/test/buildah-bud/apply-podman-deltas +++ b/test/buildah-bud/apply-podman-deltas @@ -136,32 +136,17 @@ errmsg "no such file or directory" \ ############################################################################### # BEGIN tests that don't make sense under podman due to fundamental differences + # TODO -# Some of the podman tests in CI expects exit code 125, which might not be true -# since exit code from runtime is relayed as it is without any modification both -# in `buildah` and `podman`. Following behaviour is seen when PR https://github.com/containers/buildah/pull/3809 -# added a test here https://github.com/containers/buildah/blob/main/tests/bud.bats#L3183 -# which relays exit code from runtime as it is, in case of both `podman` and `buildah`. -# However apart from this test case no other test case was able to trigger this behavior -# hence marking this test as an anomaly. Since its debatable if we should override this -# returned error number or not hence adding a note here. -skip "podman CI expects all exit code to be 125 this test has anomaly behaviour" \ - "bud with --add-host" - -skip "podman CI expects all exit code to be 125 this test has anomaly behaviour" \ +# Normally, when buildah exits 1 on error, podman exits 125. +# These tests are the exception. They exit 1 under podman. +skip "these tests exit 1 under podman, not 125" \ + "bud with --add-host" \ "bud - invalid runtime flags test" skip "does not work under podman" \ "bud without any arguments should succeed" -skip "podman requires a directory, not a Dockerfile" \ - "bud with specified context should succeed if context contains existing Dockerfile" - -# ...or due to Ed's laziness -skip "Too much effort to spin up a local registry" \ - "bud with encrypted FROM image" \ - "bud --authfile" - # ...or due to a fundamental arg-parsing difference between buildah and podman # which we could and perhaps should fix in the buildah repo via: # - ... ${TESTSDIR}/bud/layers-squash/Dockerfile.hardlinks @@ -172,11 +157,6 @@ skip "FIXME FIXME FIXME: argument-order incompatible with podman" \ skip "FIXME FIXME FIXME: this passes on Ed's laptop, fails in CI??" \ "bud-multi-stage-nocache-nocommit" -# This will probably never work: buildah and podman have incompatible defaults -# Documented in https://github.com/containers/podman/issues/10412 -skip "buildah runs with --cgroup-manager=cgroupfs, podman with systemd" \ - "bud with --cgroup-parent" - ############################################################################### # BEGIN tests which are skipped because they make no sense under podman-remote @@ -200,6 +180,10 @@ skip_if_remote "volumes don't work with podman-remote" \ "buildah bud --volume" \ "buildah-bud-policy" +# Requires a local file outside context dir +skip_if_remote "local keyfile not sent to podman-remote" \ + "bud with encrypted FROM image" + # See podman #9890 for discussion skip_if_remote "--stdin option will not be implemented in podman-remote" \ "bud test no --stdin" diff --git a/test/buildah-bud/buildah-tests.diff b/test/buildah-bud/buildah-tests.diff index ca38b5a10..d8b7e177f 100644 --- a/test/buildah-bud/buildah-tests.diff +++ b/test/buildah-bud/buildah-tests.diff @@ -1,15 +1,15 @@ -From 1a5562929a85074a7498165191558f434bb30bf1 Mon Sep 17 00:00:00 2001 +From a2cf44e9b58955d682626e95b493450242fb8394 Mon Sep 17 00:00:00 2001 From: Ed Santiago <santiago@redhat.com> Date: Tue, 9 Feb 2021 17:28:05 -0700 Subject: [PATCH] tweaks for running buildah tests under podman Signed-off-by: Ed Santiago <santiago@redhat.com> --- - tests/helpers.bash | 73 +++++++++++++++++++++++++++++++++++++++++++--- - 1 file changed, 69 insertions(+), 4 deletions(-) + tests/helpers.bash | 69 ++++++++++++++++++++++++++++++++++++++++++++-- + 1 file changed, 66 insertions(+), 3 deletions(-) diff --git a/tests/helpers.bash b/tests/helpers.bash -index 2bb6b429..0dc98a85 100644 +index b5713550..34fcfa3a 100644 --- a/tests/helpers.bash +++ b/tests/helpers.bash @@ -43,6 +43,23 @@ EOF @@ -36,9 +36,9 @@ index 2bb6b429..0dc98a85 100644 } function starthttpd() { -@@ -85,6 +102,12 @@ function teardown_tests() { - stophttpd +@@ -86,6 +103,12 @@ function teardown_tests() { stop_git_daemon + stop_registry + if [[ -n "$PODMAN_SERVER_PID" ]]; then + kill $PODMAN_SERVER_PID @@ -49,22 +49,18 @@ index 2bb6b429..0dc98a85 100644 # Workaround for #1991 - buildah + overlayfs leaks mount points. # Many tests leave behind /var/tmp/.../root/overlay and sub-mounts; # let's find those and clean them up, otherwise 'rm -rf' fails. -@@ -157,7 +180,13 @@ function copy() { +@@ -178,6 +201,10 @@ function podman() { + command ${PODMAN_BINARY:-podman} ${PODMAN_REGISTRY_OPTS} ${ROOTDIR_OPTS} "$@" } - function podman() { -- command podman ${PODMAN_REGISTRY_OPTS} ${ROOTDIR_OPTS} "$@" -+ echo "# ... podman $*" >&3 -+ ${PODMAN_BINARY} ${PODMAN_REGISTRY_OPTS} ${ROOTDIR_OPTS} "$@" ++function podman-remote() { ++ command ${PODMAN_BINARY:-podman-remote} ${ROOTDIR_OPTS} "$@" +} + -+function podman-remote() { -+ echo "# ... podman-remote $*" >&3 -+ ${PODMAN_BINARY} ${ROOTDIR_OPTS} "$@" - } - # There are various scenarios where we would like to execute `tests` as rootless user, however certain commands like `buildah mount` -@@ -221,8 +250,35 @@ function run_buildah() { + # do not work in rootless session since a normal user cannot mount a filesystem unless they're in a user namespace along with its + # own mount namespace. In order to run such specific commands from a rootless session we must perform `buildah unshare`. +@@ -239,8 +266,35 @@ function run_buildah() { --retry) retry=3; shift;; # retry network flakes esac @@ -101,7 +97,7 @@ index 2bb6b429..0dc98a85 100644 # If session is rootless and `buildah mount` is invoked, perform unshare, # since normal user cannot mount a filesystem unless they're in a user namespace along with its own mount namespace. -@@ -236,8 +292,8 @@ function run_buildah() { +@@ -254,8 +308,8 @@ function run_buildah() { retry=$(( retry - 1 )) # stdout is only emitted upon error; this echo is to help a debugger @@ -112,7 +108,7 @@ index 2bb6b429..0dc98a85 100644 # without "quotes", multiple lines are glommed together into one if [ -n "$output" ]; then echo "$output" -@@ -545,6 +601,15 @@ function skip_if_no_docker() { +@@ -587,6 +641,15 @@ function skip_if_no_docker() { fi } @@ -129,5 +125,5 @@ index 2bb6b429..0dc98a85 100644 daemondir=${TESTDIR}/git-daemon mkdir -p ${daemondir}/repo -- -2.31.1 +2.35.1 diff --git a/test/e2e/benchmarks_test.go b/test/e2e/benchmarks_test.go index c631b06ee..9653cee3b 100644 --- a/test/e2e/benchmarks_test.go +++ b/test/e2e/benchmarks_test.go @@ -11,10 +11,12 @@ import ( "strconv" "strings" + podmanRegistry "github.com/containers/podman/v4/hack/podman-registry-go" . "github.com/containers/podman/v4/test/utils" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" . "github.com/onsi/gomega/gexec" + "github.com/sirupsen/logrus" ) var ( @@ -30,21 +32,23 @@ type benchmark struct { name string // The function to execute. main func() - // Function is run before `main`. - init func() + // Allows for extending a benchmark. + options newBenchmarkOptions } // Allows for customizing the benchnmark in an easy to extend way. type newBenchmarkOptions struct { // Sets the benchmark's init function. init func() + // Run a local registry for this benchmark. + needsRegistry bool } // Queue a new benchmark. func newBenchmark(name string, main func(), options *newBenchmarkOptions) { bm := benchmark{name: name, main: main} if options != nil { - bm.init = options.init + bm.options = *options } allBenchmarks = append(allBenchmarks, bm) } @@ -109,8 +113,23 @@ var _ = Describe("Podman Benchmark Suite", func() { for i := range allBenchmarks { setup() bm := allBenchmarks[i] - if bm.init != nil { - bm.init() + + // Start a local registry if requested. + var registry *podmanRegistry.Registry + if bm.options.needsRegistry { + reg, err := podmanRegistry.Start() + if err != nil { + logrus.Errorf("Error starting registry: %v", err) + os.Exit(1) + } + registry = reg + os.Setenv(podmanRegistry.UserKey, reg.User) + os.Setenv(podmanRegistry.PassKey, reg.Password) + os.Setenv(podmanRegistry.PortKey, reg.Port) + } + + if bm.options.init != nil { + bm.options.init() } // Set the time dir only for the main() function. @@ -120,6 +139,18 @@ var _ = Describe("Podman Benchmark Suite", func() { mem := totalMemoryInKb() b.RecordValueWithPrecision("[MEM] "+bm.name, float64(mem), "KB", 1) + + // Stop the local registry. + if bm.options.needsRegistry { + os.Unsetenv(podmanRegistry.UserKey) + os.Unsetenv(podmanRegistry.PassKey) + os.Unsetenv(podmanRegistry.PortKey) + if err := registry.Stop(); err != nil { + logrus.Errorf("Error stopping registry: %v", err) + os.Exit(1) + } + } + cleanup() } }, numBenchmarkSamples) @@ -142,6 +173,44 @@ var _ = Describe("Podman Benchmark Suite", func() { Expect(session).Should(Exit(0)) }, nil) + newBenchmark("podman load [docker]", func() { + session := podmanTest.Podman([]string{"load", "-i", "./testdata/docker-two-images.tar.xz"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + }, nil) + + newBenchmark("podman load [oci]", func() { + session := podmanTest.Podman([]string{"load", "-i", "./testdata/oci-registry-name.tar.gz"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + }, nil) + + newBenchmark("podman save", func() { + session := podmanTest.Podman([]string{"save", ALPINE, "-o", path.Join(podmanTest.TempDir, "alpine.tar")}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + }, nil) + + newBenchmark("podman image inspect", func() { + session := podmanTest.Podman([]string{"inspect", ALPINE}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + }, nil) + + newBenchmark("podman login + logout", func() { + user := os.Getenv(podmanRegistry.UserKey) + pass := os.Getenv(podmanRegistry.PassKey) + port := os.Getenv(podmanRegistry.PortKey) + + session := podmanTest.Podman([]string{"login", "-u", user, "-p", pass, "--tls-verify=false", "localhost:" + port}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + + session = podmanTest.Podman([]string{"logout", "localhost:" + port}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + }, &newBenchmarkOptions{needsRegistry: true}) + // -------------------------------------------------------------------------- // CONTAINER BENCHMARKS // -------------------------------------------------------------------------- |