diff options
| -rwxr-xr-x | .papr.sh | 95 | ||||
| -rw-r--r-- | .papr.yml | 12 | ||||
| -rw-r--r-- | .papr_prepare.sh | 16 | ||||
| -rw-r--r-- | Dockerfile | 16 | ||||
| -rw-r--r-- | Dockerfile.CentOSDev | 78 | ||||
| -rw-r--r-- | Dockerfile.Fedora | 67 | ||||
| -rw-r--r-- | Makefile | 8 | ||||
| -rw-r--r-- | test/e2e/info_test.go | 35 | ||||
| -rw-r--r-- | test/e2e/libpod_suite_test.go | 308 | ||||
| -rw-r--r-- | test/e2e/rmi_test.go | 70 | ||||
| -rw-r--r-- | test/e2e/run_test.go | 187 | ||||
| -rw-r--r-- | test/e2e/wait_test.go | 63 | ||||
| -rw-r--r-- | test/podman_rmi.bats | 52 | ||||
| -rw-r--r-- | test/podman_run.bats | 162 | ||||
| -rw-r--r-- | test/podman_wait.bats | 43 |
15 files changed, 860 insertions, 352 deletions
@@ -1,85 +1,10 @@ #!/bin/bash set -xeuo pipefail -DIST=$(cat /etc/redhat-release | awk '{print $1}') -IMAGE=registry.fedoraproject.org/fedora:27 -PACKAGER=dnf -NOTEST=${NOTEST:-0} -if [[ ${DIST} != "Fedora" ]]; then - PACKAGER=yum - IMAGE=registry.centos.org/centos/centos:7 -fi - -if test -z "${INSIDE_CONTAINER:-}"; then - source /etc/os-release - - if [ -f /run/ostree-booted ]; then - - # by default, the root LV on AH is only 3G, but we need a - # bit more for our tests. Only do resize on centos and fedora - # versions less than 27 - if [[ "$VERSION_ID" != "27" ]]; then - lvresize -r -L +4G atomicos/root - fi - - if [ ! -e /var/tmp/ostree-unlock-ovl.* ]; then - ostree admin unlock - fi - fi - # Restarting docker helps with permissions related to above. - systemctl restart docker - - # somewhat mimic the spec conditional - if [ "$ID" == fedora ]; then - PYTHON=python3 - else - PYTHON=python - fi - docker run --rm \ - --privileged \ - -v $PWD:/go/src/github.com/projectatomic/libpod \ - -v /etc/yum.repos.d:/etc/yum.repos.d.host:ro \ - -v /usr:/host/usr \ - -v /etc:/host/etc \ - -v /host:/host/var \ - --workdir /go/src/github.com/projectatomic/libpod \ - -e INSIDE_CONTAINER=1 \ - -e PYTHON=$PYTHON \ - -e NOTEST=$NOTEST \ - ${IMAGE} /go/src/github.com/projectatomic/libpod/.papr.sh - if [[ "${NOTEST}" -eq 1 ]]; then - exit - fi - systemd-detect-virt - script -qefc ./test/test_runner.sh - exit 0 -fi - export GOPATH=/go export PATH=$HOME/gopath/bin:$PATH export GOSRC=/$GOPATH/src/github.com/projectatomic/libpod -${PACKAGER} install -y \ - btrfs-progs-devel \ - bzip2 \ - device-mapper-devel \ - findutils \ - git \ - glib2-devel \ - gnupg \ - golang \ - gpgme-devel \ - libassuan-devel \ - libseccomp-devel \ - libselinux-devel \ - skopeo-containers \ - runc \ - make \ - ostree-devel \ - python \ - which\ - golang-github-cpuguy83-go-md2man - # PAPR adds a merge commit, for testing, which fails the # short-commit-subject validation test, so tell git-validate.sh to only check @@ -87,25 +12,25 @@ ${PACKAGER} install -y \ export GITVALIDATE_TIP=$(cd $GOSRC; git log -2 --pretty='%H' | tail -n 1) export TAGS="seccomp $($GOSRC/hack/btrfs_tag.sh) $($GOSRC/hack/libdm_tag.sh) $($GOSRC/hack/btrfs_installed_tag.sh) $($GOSRC/hack/ostree_tag.sh) $($GOSRC/hack/selinux_tag.sh)" -if [[ "${NOTEST}" -eq 0 ]]; then - make gofmt TAGS="${TAGS}" - make testunit TAGS="${TAGS}" -fi +make gofmt TAGS="${TAGS}" +make testunit TAGS="${TAGS}" make install.tools TAGS="${TAGS}" # Only check lint and gitvalidation on more recent # distros with updated git and tooling -if [[ ${PACKAGER} != "yum" ]] && [[ "${NOTEST}" -eq 0 ]]; then +if [[ ${DIST} == "Fedora" ]]; then HEAD=$GITVALIDATE_TIP make -C $GOSRC .gitvalidation TAGS="${TAGS}" make lint fi +# Make and install podman make TAGS="${TAGS}" -make TAGS="${TAGS}" install PREFIX=/host/usr ETCDIR=/host/etc +make TAGS="${TAGS}" install PREFIX=/usr ETCDIR=/etc make TAGS="${TAGS}" test-binaries -# Add the new bats core instead of the deprecated bats -git clone https://github.com/bats-core/bats-core /bats-core -cd /bats-core -install -D -m 755 libexec/* /host/usr/bin/ +# Run the ginkgo integration tests +GOPATH=/go ginkgo test/e2e/. +# Run the bats integration tests +script -qefc ./test/test_runner.sh +exit 0 @@ -7,20 +7,13 @@ host: distro: fedora/27/atomic specs: ram: 8192 + cpus: 4 required: true timeout: 45m tests: - - CRIO_ROOT=/var/tmp/checkout PODMAN_BINARY=/usr/bin/podman CONMON_BINARY=/usr/libexec/crio/conmon PAPR=1 sh .papr.sh - -packages: - - cri-o - - containernetworking-cni - -extra-repos: - - name: updatestesting - baseurl: http://download.fedoraproject.org/pub/fedora/linux/updates/testing/27/x86_64/ + - sh .papr_prepare.sh --- @@ -29,6 +22,7 @@ host: distro: centos/7/atomic/smoketested specs: ram: 8192 + cpus: 4 extra-repos: - name: epel metalink: https://mirrors.fedoraproject.org/metalink?repo=epel-7&arch=$basearch diff --git a/.papr_prepare.sh b/.papr_prepare.sh new file mode 100644 index 000000000..730cd6540 --- /dev/null +++ b/.papr_prepare.sh @@ -0,0 +1,16 @@ +#!/bin/bash +set -xeuo pipefail + +DIST=$(cat /etc/redhat-release | awk '{print $1}') +IMAGE=fedorapodmanbuild +PYTHON=python3 +if [[ ${DIST} != "Fedora" ]]; then + IMAGE=centospodmanbuild + PYTHON=python +fi + +# Build the test image +docker build -t ${IMAGE} -f Dockerfile.${DIST} . + +# Run the tests +docker run --rm --privileged -v $PWD:/go/src/github.com/projectatomic/libpod --workdir /go/src/github.com/projectatomic/libpod -e PYTHON=$PYTHON -e STORAGE_OPTIONS="--storage-driver=vfs" -e CRIO_ROOT="/go/src/github.com/projectatomic/libpod" -e PODMAN_BINARY="/usr/bin/podman" -e CONMON_BINARY="/usr/libexec/crio/conmon" -e DIST=$DIST $IMAGE sh .papr.sh diff --git a/Dockerfile b/Dockerfile index c046c6ad3..affcf8b39 100644 --- a/Dockerfile +++ b/Dockerfile @@ -103,6 +103,22 @@ RUN set -x \ && cp "$GOPATH"/bin/crictl /usr/bin/ \ && rm -rf "$GOPATH" +# Install ginkgo +RUN set -x \ + && export GOPATH=/go \ + && go get -u github.com/onsi/ginkgo/ginkgo \ + && install -D -m 755 "$GOPATH"/bin/ginkgo /usr/bin/ + +# Install gomega +RUN set -x \ + && export GOPATH=/go \ + && go get github.com/onsi/gomega/... + +# Install cni config +#RUN make install.cni +RUN mkdir -p /etc/cni/net.d/ +COPY cni/87-podman-bridge.conflist /etc/cni/net.d/87-podman-bridge.conflist + # Make sure we have some policy for pulling images RUN mkdir -p /etc/containers COPY test/policy.json /etc/containers/policy.json diff --git a/Dockerfile.CentOSDev b/Dockerfile.CentOSDev new file mode 100644 index 000000000..3bb834c10 --- /dev/null +++ b/Dockerfile.CentOSDev @@ -0,0 +1,78 @@ +FROM registry.centos.org/centos/centos:7 + +RUN yum -y install btrfs-progs-devel \ + bzip2 \ + device-mapper-devel \ + findutils \ + git \ + glib2-devel \ + gnupg \ + golang \ + gpgme-devel \ + libassuan-devel \ + libseccomp-devel \ + libselinux-devel \ + skopeo-containers \ + runc \ + make \ + ostree-devel \ + python \ + which\ + golang-github-cpuguy83-go-md2man \ + iptables && yum clean all + +# install bats +RUN cd /tmp \ + && git clone https://github.com/sstephenson/bats.git \ + && cd bats \ + && git reset --hard 03608115df2071fff4eaaff1605768c275e5f81f \ + && ./install.sh /usr/local \ + && rm -fr /tmp/bats + +# Install CNI plugins +ENV CNI_COMMIT 7480240de9749f9a0a5c8614b17f1f03e0c06ab9 +RUN set -x \ + && export GOPATH="$(mktemp -d)" \ + && git clone https://github.com/containernetworking/plugins.git "$GOPATH/src/github.com/containernetworking/plugins" \ + && cd "$GOPATH/src/github.com/containernetworking/plugins" \ + && git checkout -q "$CNI_COMMIT" \ + && ./build.sh \ + && mkdir -p /usr/libexec/cni \ + && cp bin/* /usr/libexec/cni \ + && rm -rf "$GOPATH" + +# Install ginkgo +RUN set -x \ + && export GOPATH=/go \ + && go get -u github.com/onsi/ginkgo/ginkgo \ + && install -D -m 755 "$GOPATH"/bin/ginkgo /usr/bin/ + +# Install gomega +RUN set -x \ + && export GOPATH=/go \ + && go get github.com/onsi/gomega/... + +# Install conmon +ENV CRIO_COMMIT 814c6ab0913d827543696b366048056a31d9529c +RUN set -x \ + && export GOPATH="$(mktemp -d)" \ + && git clone https://github.com/kubernetes-incubator/cri-o.git "$GOPATH/src/github.com/kubernetes-incubator/cri-o.git" \ + && cd "$GOPATH/src/github.com/kubernetes-incubator/cri-o.git" \ + && git fetch origin --tags \ + && git checkout -q "$CRIO_COMMIT" \ + && mkdir bin \ + && make conmon \ + && install -D -m 755 bin/conmon /usr/libexec/crio/conmon \ + && rm -rf "$GOPATH" + +# Install cni config +#RUN make install.cni +RUN mkdir -p /etc/cni/net.d/ +COPY cni/87-podman-bridge.conflist /etc/cni/net.d/87-podman-bridge.conflist + +# Make sure we have some policy for pulling images +RUN mkdir -p /etc/containers +COPY test/policy.json /etc/containers/policy.json +COPY test/redhat_sigstore.yaml /etc/containers/registries.d/registry.access.redhat.com.yaml + +WORKDIR /go/src/github.com/projectatomic/libpod diff --git a/Dockerfile.Fedora b/Dockerfile.Fedora new file mode 100644 index 000000000..1dbc1a02e --- /dev/null +++ b/Dockerfile.Fedora @@ -0,0 +1,67 @@ +FROM registry.fedoraproject.org/fedora:27 + +RUN dnf -y install btrfs-progs-devel \ + bzip2 \ + device-mapper-devel \ + findutils \ + git \ + glib2-devel \ + gnupg \ + golang \ + gpgme-devel \ + libassuan-devel \ + libseccomp-devel \ + libselinux-devel \ + skopeo-containers \ + runc \ + make \ + ostree-devel \ + python \ + which\ + golang-github-cpuguy83-go-md2man \ + crio \ + procps-ng \ + iptables && dnf clean all + +# install bats +RUN cd /tmp \ + && git clone https://github.com/sstephenson/bats.git \ + && cd bats \ + && git reset --hard 03608115df2071fff4eaaff1605768c275e5f81f \ + && ./install.sh /usr/local \ + && rm -fr /tmp/bats + +# Install CNI plugins +ENV CNI_COMMIT 7480240de9749f9a0a5c8614b17f1f03e0c06ab9 +RUN set -x \ + && export GOPATH="$(mktemp -d)" \ + && git clone https://github.com/containernetworking/plugins.git "$GOPATH/src/github.com/containernetworking/plugins" \ + && cd "$GOPATH/src/github.com/containernetworking/plugins" \ + && git checkout -q "$CNI_COMMIT" \ + && ./build.sh \ + && mkdir -p /usr/libexec/cni \ + && cp bin/* /usr/libexec/cni \ + && rm -rf "$GOPATH" + +# Install ginkgo +RUN set -x \ + && export GOPATH=/go \ + && go get -u github.com/onsi/ginkgo/ginkgo \ + && install -D -m 755 "$GOPATH"/bin/ginkgo /usr/bin/ + +# Install gomega +RUN set -x \ + && export GOPATH=/go \ + && go get github.com/onsi/gomega/... + +# Install cni config +#RUN make install.cni +RUN mkdir -p /etc/cni/net.d/ +COPY cni/87-podman-bridge.conflist /etc/cni/net.d/87-podman-bridge.conflist + +# Make sure we have some policy for pulling images +RUN mkdir -p /etc/containers +COPY test/policy.json /etc/containers/policy.json +COPY test/redhat_sigstore.yaml /etc/containers/registries.d/registry.access.redhat.com.yaml + +WORKDIR /go/src/github.com/projectatomic/libpod @@ -18,7 +18,7 @@ BASHINSTALLDIR=${PREFIX}/share/bash-completion/completions OCIUMOUNTINSTALLDIR=$(PREFIX)/share/oci-umount/oci-umount.d SELINUXOPT ?= $(shell test -x /usr/sbin/selinuxenabled && selinuxenabled && echo -Z) -PACKAGES ?= $(shell go list -tags "${BUILDTAGS}" ./... | grep -v github.com/projectatomic/libpod/vendor) +PACKAGES ?= $(shell go list -tags "${BUILDTAGS}" ./... | grep -v github.com/projectatomic/libpod/vendor | grep -v e2e) COMMIT_NO := $(shell git rev-parse HEAD 2> /dev/null || true) GIT_COMMIT := $(if $(shell git status --porcelain --untracked-files=no),"${COMMIT_NO}-dirty","${COMMIT_NO}") @@ -110,10 +110,16 @@ dbuild: libpodimage integration: libpodimage docker run -e STORAGE_OPTIONS="--storage-driver=vfs" -e TESTFLAGS -e TRAVIS -t --privileged --rm -v ${CURDIR}:/go/src/${PROJECT} ${LIBPOD_IMAGE} make localintegration +integration.fedora: + DIST=Fedora sh .papr_prepare.sh testunit: $(GO) test -tags "$(BUILDTAGS)" -cover $(PACKAGES) +ginkgo: + ginkgo -v test/e2e/ + localintegration: test-binaries + ginkgo -v test/e2e/. bash -i ./test/test_runner.sh ${TESTFLAGS} vagrant-check: diff --git a/test/e2e/info_test.go b/test/e2e/info_test.go new file mode 100644 index 000000000..965dffaae --- /dev/null +++ b/test/e2e/info_test.go @@ -0,0 +1,35 @@ +package integration + +import ( + "os" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("Podman Info", func() { + var ( + tempdir string + err error + podmanTest PodmanTest + ) + + BeforeEach(func() { + tempdir, err = CreateTempDirInTempDir() + if err != nil { + os.Exit(1) + } + podmanTest = PodmanCreate(tempdir) + }) + + AfterEach(func() { + podmanTest.Cleanup() + }) + + It("podman info json output", func() { + session := podmanTest.Podman([]string{"info", "--format=json"}) + session.Wait() + Expect(session.ExitCode()).To(Equal(0)) + + }) +}) diff --git a/test/e2e/libpod_suite_test.go b/test/e2e/libpod_suite_test.go new file mode 100644 index 000000000..2f541244c --- /dev/null +++ b/test/e2e/libpod_suite_test.go @@ -0,0 +1,308 @@ +package integration + +import ( + "fmt" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/containers/image/copy" + "github.com/containers/image/signature" + "github.com/containers/image/storage" + "github.com/containers/image/transports/alltransports" + "github.com/containers/image/types" + sstorage "github.com/containers/storage" + "github.com/containers/storage/pkg/reexec" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/onsi/gomega/gexec" + "github.com/pkg/errors" +) + +// - CRIO_ROOT=/var/tmp/checkout PODMAN_BINARY=/usr/bin/podman CONMON_BINARY=/usr/libexec/crio/conmon PAPR=1 sh .papr.sh +// PODMAN_OPTIONS="--root $TESTDIR/crio $STORAGE_OPTIONS --runroot $TESTDIR/crio-run --runtime ${RUNTIME_BINARY} --conmon ${CONMON_BINARY} --cni-config-dir ${LIBPOD_CNI_CONFIG}" + +//TODO do the image caching +// "$COPYIMG_BINARY" --root "$TESTDIR/crio" $STORAGE_OPTIONS --runroot "$TESTDIR/crio-run" --image-name=${IMAGES[${key}]} --import-from=dir:"$ARTIFACTS_PATH"/${key} --add-name=${IMAGES[${key}]} +//TODO whats the best way to clean up after a test + +var ( + PODMAN_BINARY string + CONMON_BINARY string + CNI_CONFIG_DIR string + RUNC_BINARY string + INTEGRATION_ROOT string + STORAGE_OPTIONS = "--storage-driver vfs" + ARTIFACT_DIR = "/tmp/.artifacts" + IMAGES = []string{"alpine", "busybox"} + ALPINE = "docker.io/library/alpine:latest" + BB_GLIBC = "docker.io/library/busybox:glibc" + fedoraMinimal = "registry.fedoraproject.org/fedora-minimal:latest" +) + +// PodmanSession wrapps the gexec.session so we can extend it +type PodmanSession struct { + *gexec.Session +} + +// PodmanTest struct for command line options +type PodmanTest struct { + PodmanBinary string + ConmonBinary string + CrioRoot string + CNIConfigDir string + RunCBinary string + RunRoot string + StorageOptions string + SignaturePolicyPath string + ArtifactPath string + TempDir string +} + +// TestLibpod ginkgo master function +func TestLibpod(t *testing.T) { + if reexec.Init() { + os.Exit(1) + } + RegisterFailHandler(Fail) + RunSpecs(t, "Libpod Suite") +} + +var _ = BeforeSuite(func() { + //Cache images + cwd, _ := os.Getwd() + INTEGRATION_ROOT = filepath.Join(cwd, "../../") + podman := PodmanCreate("/tmp") + podman.ArtifactPath = ARTIFACT_DIR + if _, err := os.Stat(ARTIFACT_DIR); os.IsNotExist(err) { + if err = os.Mkdir(ARTIFACT_DIR, 0777); err != nil { + fmt.Printf("%q\n", err) + os.Exit(1) + } + } + for _, image := range IMAGES { + fmt.Printf("Caching %s...\n", image) + if err := podman.CreateArtifact(image); err != nil { + fmt.Printf("%q\n", err) + os.Exit(1) + } + } + +}) + +// CreateTempDirin +func CreateTempDirInTempDir() (string, error) { + return ioutil.TempDir("", "podman_test") +} + +// PodmanCreate creates a PodmanTest instance for the tests +func PodmanCreate(tempDir string) PodmanTest { + cwd, _ := os.Getwd() + + podmanBinary := filepath.Join(cwd, "../../bin/podman") + if os.Getenv("PODMAN_BINARY") != "" { + podmanBinary = os.Getenv("PODMAN_BINARY") + } + conmonBinary := filepath.Join("/usr/libexec/crio/conmon") + if os.Getenv("CONMON_BINARY") != "" { + conmonBinary = os.Getenv("CONMON_BINARY") + } + storageOptions := STORAGE_OPTIONS + if os.Getenv("STORAGE_OPTIONS") != "" { + storageOptions = os.Getenv("STORAGE_OPTIONS") + } + + runCBinary := "/usr/bin/runc" + CNIConfigDir := "/etc/cni/net.d" + + return PodmanTest{ + PodmanBinary: podmanBinary, + ConmonBinary: conmonBinary, + CrioRoot: filepath.Join(tempDir, "crio"), + CNIConfigDir: CNIConfigDir, + RunCBinary: runCBinary, + RunRoot: filepath.Join(tempDir, "crio-run"), + StorageOptions: storageOptions, + SignaturePolicyPath: filepath.Join(INTEGRATION_ROOT, "test/policy.json"), + ArtifactPath: ARTIFACT_DIR, + TempDir: tempDir, + } +} + +//MakeOptions assembles all the podman main options +func (p *PodmanTest) MakeOptions() []string { + return strings.Split(fmt.Sprintf("--root %s --runroot %s --runtime %s --conmon %s --cni-config-dir %s", + p.CrioRoot, p.RunRoot, p.RunCBinary, p.ConmonBinary, p.CNIConfigDir), " ") +} + +// Podman is the exec call to podman on the filesystem +func (p *PodmanTest) Podman(args []string) *PodmanSession { + podmanOptions := p.MakeOptions() + podmanOptions = append(podmanOptions, strings.Split(p.StorageOptions, " ")...) + podmanOptions = append(podmanOptions, args...) + fmt.Printf("Running: %s %s\n", p.PodmanBinary, strings.Join(podmanOptions, " ")) + command := exec.Command(p.PodmanBinary, podmanOptions...) + session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) + if err != nil { + Fail(fmt.Sprintf("unable to run podman command: %s", strings.Join(podmanOptions, " "))) + } + return &PodmanSession{session} +} + +// Cleanup cleans up the temporary store +func (p *PodmanTest) Cleanup() { + // Remove all containers + session := p.Podman([]string{"rm", "-fa"}) + session.Wait() + // Nuke tempdir + if err := os.RemoveAll(p.TempDir); err != nil { + fmt.Printf("%q\n", err) + } +} + +// GrepString takes session output and behaves like grep. it returns a bool +// if successful and an array of strings on positive matches +func (s *PodmanSession) GrepString(term string) (bool, []string) { + var ( + greps []string + matches bool + ) + + for _, line := range strings.Split(s.OutputToString(), "\n") { + if strings.Contains(line, term) { + matches = true + greps = append(greps, line) + } + } + return matches, greps +} + +// Pull Images pulls multiple images +func (p *PodmanTest) PullImages(images []string) error { + for _, i := range images { + p.PullImage(i) + } + return nil +} + +// Pull Image a single image +// TODO should the timeout be configurable? +func (p *PodmanTest) PullImage(image string) error { + session := p.Podman([]string{"pull", image}) + session.Wait(60) + Expect(session.ExitCode()).To(Equal(0)) + return nil +} + +// OutputToString formats session output to string +func (s *PodmanSession) OutputToString() string { + fields := strings.Fields(fmt.Sprintf("%s", s.Out.Contents())) + return strings.Join(fields, " ") +} + +// SystemExec is used to exec a system command to check its exit code or output +func (p *PodmanTest) SystemExec(command string, args []string) *PodmanSession { + c := exec.Command(command, args...) + session, err := gexec.Start(c, GinkgoWriter, GinkgoWriter) + if err != nil { + Fail(fmt.Sprintf("unable to run command: %s %s", command, strings.Join(args, " "))) + } + return &PodmanSession{session} +} + +// CreateArtifact creates a cached image in the artifact dir +func (p *PodmanTest) CreateArtifact(image string) error { + imageName := fmt.Sprintf("docker://%s", image) + systemContext := types.SystemContext{ + SignaturePolicyPath: p.SignaturePolicyPath, + } + policy, err := signature.DefaultPolicy(&systemContext) + if err != nil { + return errors.Errorf("error loading signature policy: %v", err) + } + policyContext, err := signature.NewPolicyContext(policy) + if err != nil { + return errors.Errorf("error loading signature policy: %v", err) + } + defer func() { + _ = policyContext.Destroy() + }() + options := ©.Options{} + + importRef, err := alltransports.ParseImageName(imageName) + if err != nil { + return errors.Errorf("error parsing image name %v: %v", image, err) + } + + exportTo := filepath.Join("dir:", p.ArtifactPath, image) + exportRef, err := alltransports.ParseImageName(exportTo) + if err != nil { + return errors.Errorf("error parsing image name %v: %v", exportTo, err) + } + + return copy.Image(policyContext, exportRef, importRef, options) + + return nil +} + +// RestoreArtifact puts the cached image into our test store +func (p *PodmanTest) RestoreArtifact(image string) error { + storeOptions := sstorage.DefaultStoreOptions + storeOptions.GraphDriverName = "vfs" + //storeOptions.GraphDriverOptions = storageOptions + storeOptions.GraphRoot = p.CrioRoot + storeOptions.RunRoot = p.RunRoot + store, err := sstorage.GetStore(storeOptions) + + options := ©.Options{} + if err != nil { + return errors.Errorf("error opening storage: %v", err) + } + defer func() { + _, _ = store.Shutdown(false) + }() + + storage.Transport.SetStore(store) + ref, err := storage.Transport.ParseStoreReference(store, image) + if err != nil { + return errors.Errorf("error parsing image name: %v", err) + } + + importFrom := fmt.Sprintf("dir:%s", filepath.Join(p.ArtifactPath, image)) + importRef, err := alltransports.ParseImageName(importFrom) + if err != nil { + return errors.Errorf("error parsing image name %v: %v", image, err) + } + systemContext := types.SystemContext{ + SignaturePolicyPath: p.SignaturePolicyPath, + } + policy, err := signature.DefaultPolicy(&systemContext) + if err != nil { + return errors.Errorf("error loading signature policy: %v", err) + } + policyContext, err := signature.NewPolicyContext(policy) + if err != nil { + return errors.Errorf("error loading signature policy: %v", err) + } + defer func() { + _ = policyContext.Destroy() + }() + err = copy.Image(policyContext, ref, importRef, options) + if err != nil { + return errors.Errorf("error importing %s: %v", importFrom, err) + } + return nil +} + +// RestoreAllArtifacts unpacks all cached images +func (p *PodmanTest) RestoreAllArtifacts() error { + for _, image := range IMAGES { + if err := p.RestoreArtifact(image); err != nil { + return err + } + } + return nil +} diff --git a/test/e2e/rmi_test.go b/test/e2e/rmi_test.go new file mode 100644 index 000000000..895c812c6 --- /dev/null +++ b/test/e2e/rmi_test.go @@ -0,0 +1,70 @@ +package integration + +import ( + "os" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("Podman rmi", func() { + var ( + tempdir string + err error + podmanTest PodmanTest + image1 = "docker.io/library/alpine:latest" + image3 = "docker.io/library/busybox:glibc" + ) + + BeforeEach(func() { + tempdir, err = CreateTempDirInTempDir() + if err != nil { + os.Exit(1) + } + podmanTest = PodmanCreate(tempdir) + podmanTest.RestoreAllArtifacts() + }) + + AfterEach(func() { + podmanTest.Cleanup() + + }) + + It("podman rmi bogus image", func() { + session := podmanTest.Podman([]string{"rmi", "debian:6.0.10"}) + session.Wait() + Expect(session.ExitCode()).To(Equal(125)) + + }) + + It("podman rmi with fq name", func() { + session := podmanTest.Podman([]string{"rmi", image1}) + session.Wait() + Expect(session.ExitCode()).To(Equal(0)) + + }) + + It("podman rmi with short name", func() { + session := podmanTest.Podman([]string{"rmi", "alpine"}) + session.Wait() + Expect(session.ExitCode()).To(Equal(0)) + + }) + + It("podman rmi all images", func() { + podmanTest.PullImages([]string{image3}) + session := podmanTest.Podman([]string{"rmi", "-a"}) + session.Wait() + Expect(session.ExitCode()).To(Equal(0)) + + }) + + It("podman rmi all images forceably with short options", func() { + podmanTest.PullImages([]string{image3}) + session := podmanTest.Podman([]string{"rmi", "-fa"}) + session.Wait() + Expect(session.ExitCode()).To(Equal(0)) + + }) + +}) diff --git a/test/e2e/run_test.go b/test/e2e/run_test.go new file mode 100644 index 000000000..8f1aa0053 --- /dev/null +++ b/test/e2e/run_test.go @@ -0,0 +1,187 @@ +package integration + +import ( + "fmt" + "os" + "path/filepath" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("Podman run", func() { + var ( + tempdir string + err error + podmanTest PodmanTest + ) + + BeforeEach(func() { + tempdir, err = CreateTempDirInTempDir() + if err != nil { + os.Exit(1) + } + podmanTest = PodmanCreate(tempdir) + podmanTest.RestoreAllArtifacts() + }) + + AfterEach(func() { + podmanTest.Cleanup() + }) + + It("podman run a container based on local image", func() { + session := podmanTest.Podman([]string{"run", ALPINE, "ls"}) + session.Wait(10) + Expect(session.ExitCode()).To(Equal(0)) + + }) + + It("podman run a container based on local image with short options", func() { + session := podmanTest.Podman([]string{"run", "-dt", ALPINE, "ls"}) + session.Wait(10) + Expect(session.ExitCode()).To(Equal(0)) + + }) + + It("podman run a container based on remote image", func() { + session := podmanTest.Podman([]string{"run", "-dt", BB_GLIBC, "ls"}) + session.Wait(45) + Expect(session.ExitCode()).To(Equal(0)) + + }) + + It("podman run selinux grep test", func() { + selinux := podmanTest.SystemExec("ls", []string{"/usr/sbin/selinuxenabled"}) + if selinux.ExitCode() != 0 { + Skip("SELinux not enabled") + } + session := podmanTest.Podman([]string{"run", "-it", "--security-opt", "label=level:s0:c1,c2", ALPINE, "cat", "/proc/self/attr/current"}) + session.Wait(30) + Expect(session.ExitCode()).To(Equal(0)) + match, _ := session.GrepString("s0:c1,c2") + Expect(match).Should(BeTrue()) + }) + + It("podman run capabilities test", func() { + session := podmanTest.Podman([]string{"run", "--rm", "--cap-add", "all", ALPINE, "cat", "/proc/self/status"}) + session.Wait(10) + Expect(session.ExitCode()).To(Equal(0)) + + session = podmanTest.Podman([]string{"run", "--rm", "--cap-add", "sys_admin", ALPINE, "cat", "/proc/self/status"}) + session.Wait(10) + Expect(session.ExitCode()).To(Equal(0)) + + session = podmanTest.Podman([]string{"run", "--rm", "--cap-drop", "all", ALPINE, "cat", "/proc/self/status"}) + session.Wait(10) + Expect(session.ExitCode()).To(Equal(0)) + + session = podmanTest.Podman([]string{"run", "--rm", "--cap-drop", "setuid", ALPINE, "cat", "/proc/self/status"}) + session.Wait(10) + Expect(session.ExitCode()).To(Equal(0)) + }) + + It("podman run environment test", func() { + session := podmanTest.Podman([]string{"run", "--rm", "--env", "FOO=BAR", ALPINE, "printenv", "FOO"}) + session.Wait(10) + Expect(session.ExitCode()).To(Equal(0)) + match, _ := session.GrepString("BAR") + Expect(match).Should(BeTrue()) + + session = podmanTest.Podman([]string{"run", "--rm", "--env", "PATH=/bin", ALPINE, "printenv", "PATH"}) + session.Wait(10) + Expect(session.ExitCode()).To(Equal(0)) + match, _ = session.GrepString("/bin") + Expect(match).Should(BeTrue()) + + os.Setenv("FOO", "BAR") + session = podmanTest.Podman([]string{"run", "--rm", "--env", "FOO", ALPINE, "printenv", "FOO"}) + session.Wait(10) + Expect(session.ExitCode()).To(Equal(0)) + match, _ = session.GrepString("BAR") + Expect(match).Should(BeTrue()) + os.Unsetenv("FOO") + + session = podmanTest.Podman([]string{"run", "--rm", ALPINE, "printenv"}) + session.Wait(10) + Expect(session.ExitCode()).To(Equal(0)) + + // This currently does not work + // Re-enable when hostname is an env variable + //session = podmanTest.Podman([]string{"run", "--rm", ALPINE, "sh", "-c", "printenv"}) + //session.Wait(10) + //Expect(session.ExitCode()).To(Equal(0)) + //match, _ = session.GrepString("HOSTNAME") + //Expect(match).Should(BeTrue()) + }) + + It("podman run limits test", func() { + session := podmanTest.Podman([]string{"run", "--rm", "--ulimit", "rtprio=99", "--cap-add=sys_nice", fedoraMinimal, "cat", "/proc/self/sched"}) + session.Wait(45) + Expect(session.ExitCode()).To(Equal(0)) + + session = podmanTest.Podman([]string{"run", "--rm", "--ulimit", "nofile=2048:2048", fedoraMinimal, "ulimit", "-n"}) + session.Wait(45) + Expect(session.ExitCode()).To(Equal(0)) + Expect(session.OutputToString()).To(ContainSubstring("2048")) + + session = podmanTest.Podman([]string{"run", "--rm", "--ulimit", "nofile=1024:1028", fedoraMinimal, "ulimit", "-n"}) + session.Wait(45) + Expect(session.ExitCode()).To(Equal(0)) + Expect(session.OutputToString()).To(ContainSubstring("1024")) + + session = podmanTest.Podman([]string{"run", "--rm", "--oom-kill-disable=true", fedoraMinimal, "echo", "memory-hog"}) + session.Wait(45) + Expect(session.ExitCode()).To(Equal(0)) + + session = podmanTest.Podman([]string{"run", "--rm", "--oom-score-adj=100", fedoraMinimal, "cat", "/proc/self/oom_score_adj"}) + session.Wait(45) + Expect(session.ExitCode()).To(Equal(0)) + Expect(session.OutputToString()).To(ContainSubstring("100")) + }) + + It("podman run with volume flag", func() { + Skip("Skip until we diagnose the regression of volume mounts") + mountPath := filepath.Join(podmanTest.TempDir, "secrets") + os.Mkdir(mountPath, 0755) + session := podmanTest.Podman([]string{"run", "--rm", "-v", fmt.Sprintf("%s:/run/test", mountPath), ALPINE, "cat", "/proc/self/mountinfo"}) + session.Wait(10) + Expect(session.ExitCode()).To(Equal(0)) + Expect(session.OutputToString()).To(ContainSubstring("/run/test rw,relatime")) + + mountPath = filepath.Join(podmanTest.TempDir, "secrets") + os.Mkdir(mountPath, 0755) + session = podmanTest.Podman([]string{"run", "--rm", "-v", fmt.Sprintf("%s:/run/test:ro", mountPath), ALPINE, "cat", "/proc/self/mountinfo"}) + session.Wait(10) + Expect(session.ExitCode()).To(Equal(0)) + Expect(session.OutputToString()).To(ContainSubstring("/run/test ro,relatime")) + + mountPath = filepath.Join(podmanTest.TempDir, "secrets") + os.Mkdir(mountPath, 0755) + session = podmanTest.Podman([]string{"run", "--rm", "-v", fmt.Sprintf("%s:/run/test:shared", mountPath), ALPINE, "cat", "/proc/self/mountinfo"}) + session.Wait(10) + Expect(session.ExitCode()).To(Equal(0)) + Expect(session.OutputToString()).To(ContainSubstring("/run/test rw,relatime, shared")) + }) + + It("podman run with cidfile", func() { + session := podmanTest.Podman([]string{"run", "--cidfile", "/tmp/cidfile", ALPINE, "ls"}) + session.Wait(10) + Expect(session.ExitCode()).To(Equal(0)) + err := os.Remove("/tmp/cidfile") + Expect(err).To(BeNil()) + }) + + It("podman run sysctl test", func() { + session := podmanTest.Podman([]string{"run", "--rm", "--sysctl", "net.core.somaxconn=65535", ALPINE, "sysctl", "net.core.somaxconn"}) + session.Wait(10) + Expect(session.ExitCode()).To(Equal(0)) + Expect(session.OutputToString()).To(ContainSubstring("net.core.somaxconn = 65535")) + }) + + It("podman run blkio-weight test", func() { + session := podmanTest.Podman([]string{"run", "--rm", "--blkio-weight=15", ALPINE, "cat", "/sys/fs/cgroup/blkio/blkio.weight"}) + session.Wait(10) + Expect(session.ExitCode()).To(Equal(0)) + Expect(session.OutputToString()).To(ContainSubstring("15")) + }) +}) diff --git a/test/e2e/wait_test.go b/test/e2e/wait_test.go new file mode 100644 index 000000000..5448b5ae4 --- /dev/null +++ b/test/e2e/wait_test.go @@ -0,0 +1,63 @@ +package integration + +import ( + "os" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("Podman wait", func() { + var ( + tempdir string + err error + podmanTest PodmanTest + ) + + BeforeEach(func() { + tempdir, err = CreateTempDirInTempDir() + if err != nil { + os.Exit(1) + } + podmanTest = PodmanCreate(tempdir) + podmanTest.RestoreAllArtifacts() + }) + + AfterEach(func() { + podmanTest.Cleanup() + + }) + + It("podman wait on bogus container", func() { + session := podmanTest.Podman([]string{"wait", "1234"}) + session.Wait() + Expect(session.ExitCode()).To(Equal(125)) + + }) + + It("podman wait on a stopped container", func() { + session := podmanTest.Podman([]string{"run", "-d", ALPINE, "ls"}) + session.Wait(10) + cid := session.OutputToString() + Expect(session.ExitCode()).To(Equal(0)) + session = podmanTest.Podman([]string{"wait", cid}) + session.Wait() + }) + + It("podman wait on a sleeping container", func() { + session := podmanTest.Podman([]string{"run", "-d", ALPINE, "sleep", "10"}) + session.Wait(20) + cid := session.OutputToString() + Expect(session.ExitCode()).To(Equal(0)) + session = podmanTest.Podman([]string{"wait", cid}) + session.Wait(20) + }) + + It("podman wait on latest container", func() { + session := podmanTest.Podman([]string{"run", "-d", ALPINE, "sleep", "10"}) + session.Wait(20) + Expect(session.ExitCode()).To(Equal(0)) + session = podmanTest.Podman([]string{"wait", "-l"}) + session.Wait(20) + }) +}) diff --git a/test/podman_rmi.bats b/test/podman_rmi.bats deleted file mode 100644 index e800a0388..000000000 --- a/test/podman_rmi.bats +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env bats - -load helpers - -IMAGE1="docker.io/library/alpine:latest" -IMAGE2="docker.io/library/busybox:latest" -IMAGE3="docker.io/library/busybox:glibc" - -function teardown() { - cleanup_test -} - -function pullImages() { - ${PODMAN_BINARY} $PODMAN_OPTIONS pull $IMAGE1 - ${PODMAN_BINARY} $PODMAN_OPTIONS pull $IMAGE2 - ${PODMAN_BINARY} $PODMAN_OPTIONS pull $IMAGE3 -} - -@test "podman rmi bogus image" { - run ${PODMAN_BINARY} $PODMAN_OPTIONS rmi debian:6.0.10 - echo "$output" - [ "$status" -eq 125 ] -} - -@test "podman rmi image with fq name" { - pullImages - run ${PODMAN_BINARY} $PODMAN_OPTIONS rmi $IMAGE1 - echo "$output" - [ "$status" -eq 0 ] -} - -@test "podman rmi image with short name" { - pullImages - run ${PODMAN_BINARY} $PODMAN_OPTIONS rmi alpine - echo "$output" - [ "$status" -eq 0 ] -} - -@test "podman rmi all images" { - pullImages - run ${PODMAN_BINARY} $PODMAN_OPTIONS rmi -a - echo "$output" - [ "$status" -eq 0 ] -} - -@test "podman rmi all images forceably with short options" { - pullImages - ${PODMAN_BINARY} $PODMAN_OPTIONS create ${IMAGE1} ls - run ${PODMAN_BINARY} $PODMAN_OPTIONS rmi -af - echo "$output" - [ "$status" -eq 0 ] -} diff --git a/test/podman_run.bats b/test/podman_run.bats deleted file mode 100644 index 9fa048439..000000000 --- a/test/podman_run.bats +++ /dev/null @@ -1,162 +0,0 @@ -#!/usr/bin/env bats - -load helpers - -function teardown() { - cleanup_test -} - -function setup() { - copy_images -} - -@test "run a container based on local image" { - run ${PODMAN_BINARY} ${PODMAN_OPTIONS} run $BB ls - echo "$output" - [ "$status" -eq 0 ] -} - -@test "run a container based on local image with short options" { - run ${PODMAN_BINARY} ${PODMAN_OPTIONS} run -dt $BB ls - echo "$output" - [ "$status" -eq 0 ] -} - -@test "run a container based on a remote image" { - run ${PODMAN_BINARY} ${PODMAN_OPTIONS} run ${BB_GLIBC} ls - echo "$output" - [ "$status" -eq 0 ] -} - -@test "run selinux test" { - if [ ! -e /usr/sbin/selinuxenabled ] || [ ! /usr/sbin/selinuxenabled ]; then - skip "SELinux not enabled" - fi - - run ${PODMAN_BINARY} ${PODMAN_OPTIONS} run ${ALPINE} cat /proc/self/attr/current - echo "$output" - firstLabel=$output - - run ${PODMAN_BINARY} ${PODMAN_OPTIONS} run ${ALPINE} cat /proc/self/attr/current - echo "$output" - [ "$output" != "${firstLabel}" ] -} - - -@test "run selinux grep test" { - skip "Until console issues worked out" - run bash -c "${PODMAN_BINARY} ${PODMAN_OPTIONS} run -t -i --security-opt label=level:s0:c1,c2 ${ALPINE} cat /proc/self/attr/current | grep s0:c1,c2" - echo "$output" - [ "$status" -eq 0 ] - -} - -@test "run capabilities test" { - run ${PODMAN_BINARY} ${PODMAN_OPTIONS} run --cap-add all ${ALPINE} cat /proc/self/status - echo "$output" - [ "$status" -eq 0 ] - - run ${PODMAN_BINARY} ${PODMAN_OPTIONS} run --cap-add sys_admin ${ALPINE} cat /proc/self/status - echo "$output" - [ "$status" -eq 0 ] - - run ${PODMAN_BINARY} ${PODMAN_OPTIONS} run --cap-drop all ${ALPINE} cat /proc/self/status - echo "$output" - [ "$status" -eq 0 ] - - run ${PODMAN_BINARY} ${PODMAN_OPTIONS} run --cap-drop setuid ${ALPINE} cat /proc/self/status - echo "$output" - [ "$status" -eq 0 ] - -} - -@test "run environment test" { - run bash -c "${PODMAN_BINARY} ${PODMAN_OPTIONS} run --env FOO=BAR ${ALPINE} printenv FOO | tr -d '\r'" - echo "$output" - [ "$status" -eq 0 ] - [ $output = "BAR" ] - - run bash -c "${PODMAN_BINARY} ${PODMAN_OPTIONS} run --env PATH="/bin" ${ALPINE} printenv PATH | tr -d '\r'" - echo "$output" - [ "$status" -eq 0 ] - [ $output = "/bin" ] - - run bash -c "export FOO=BAR; ${PODMAN_BINARY} ${PODMAN_OPTIONS} run --env FOO ${ALPINE} printenv FOO | tr -d '\r'" - echo "$output" - [ "$status" -eq 0 ] - [ "$output" = "BAR" ] - - run ${PODMAN_BINARY} ${PODMAN_OPTIONS} run --env FOO ${ALPINE} printenv - echo "$output" - [ "$status" -ne 0 ] - -# We don't currently set the hostname in containers, since we are not setting up -# networking. As soon as podman run gets network support we need to uncomment this -# test. -# run bash -c "${PODMAN_BINARY} ${PODMAN_OPTIONS} run ${ALPINE} sh -c printenv | grep HOSTNAME" -# echo "$output" -# [ "$status" -eq 0 ] -} - -IMAGE="docker.io/library/fedora:latest" - -@test "run limits test" { - run ${PODMAN_BINARY} ${PODMAN_OPTIONS} run --ulimit rtprio=99 --cap-add=sys_nice ${IMAGE} cat /proc/self/sched - echo $output - [ "$status" -eq 0 ] - - run bash -c "${PODMAN_BINARY} ${PODMAN_OPTIONS} run --ulimit nofile=2048:2048 ${IMAGE} ulimit -n | tr -d '\r'" - echo $output - [ "$status" -eq 0 ] - [ "$output" = 2048 ] - - run bash -c "${PODMAN_BINARY} ${PODMAN_OPTIONS} run --ulimit nofile=1024:1028 ${IMAGE} ulimit -n | tr -d '\r'" - echo $output - [ "$status" -eq 0 ] - [ "$output" = 1024 ] - - run ${PODMAN_BINARY} ${PODMAN_OPTIONS} run --oom-kill-disable=true ${IMAGE} echo memory-hog - echo $output - [ "$status" -eq 0 ] - - run bash -c "${PODMAN_BINARY} ${PODMAN_OPTIONS} run --oom-score-adj=100 ${IMAGE} cat /proc/self/oom_score_adj | tr -d '\r'" - echo $output - [ "$status" -eq 0 ] - [ "$output" = 100 ] - -} - -@test "podman run with volume flag" { - run bash -c "${PODMAN_BINARY} ${PODMAN_OPTIONS} run -v ${MOUNT_PATH}:/run/test ${BB} cat /proc/self/mountinfo | grep '${MOUNT_PATH} /run/test rw,relatime'" - echo $output - [ "$status" -eq 0 ] - run bash -c "${PODMAN_BINARY} ${PODMAN_OPTIONS} run -v ${MOUNT_PATH}:/run/test:ro ${BB} cat /proc/self/mountinfo | grep '${MOUNT_PATH} /run/test ro,relatime'" - echo $output - [ "$status" -eq 0 ] - run bash -c "${PODMAN_BINARY} ${PODMAN_OPTIONS} run -v ${MOUNT_PATH}:/run/test:shared ${BB} cat /proc/self/mountinfo | grep '${MOUNT_PATH} /run/test rw,relatime shared:'" - echo $output - [ "$status" -eq 0 ] -} - -@test "podman run with cidfile" { - run ${PODMAN_BINARY} ${PODMAN_OPTIONS} run --cidfile /tmp/cidfile $BB ls - echo "$output" - [ "$status" -eq 0 ] - run rm /tmp/cidfile - echo "$output" - [ "$status" -eq 0 ] -} - -@test "podman run sysctl test" { - run bash -c "${PODMAN_BINARY} ${PODMAN_OPTIONS} run --rm --sysctl net.core.somaxconn=65535 ${ALPINE} sysctl net.core.somaxconn | tr -d '\r'" - echo "$output" - [ "$status" -eq 0 ] - [ "$output" = "net.core.somaxconn = 65535" ] -} - -@test "podman run blkio-weight test" { - run bash -c "${PODMAN_BINARY} ${PODMAN_OPTIONS} run --rm --blkio-weight=15 ${ALPINE} cat /sys/fs/cgroup/blkio/blkio.weight | tr -d '\r'" - echo "$output" - [ "$status" -eq 0 ] - [ "$output" = 15 ] -} diff --git a/test/podman_wait.bats b/test/podman_wait.bats deleted file mode 100644 index 9d6c22734..000000000 --- a/test/podman_wait.bats +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bats - -load helpers - -function setup() { - copy_images -} - -function teardown() { - cleanup_test -} - -@test "wait on a bogus container" { - run ${PODMAN_BINARY} ${PODMAN_OPTIONS} wait 12343 - echo $output - echo $status - [ "$status" -eq 125 ] -} - -@test "wait on a stopped container" { - run ${PODMAN_BINARY} ${PODMAN_OPTIONS} run -d ${ALPINE} ls - echo $output - [ "$status" -eq 0 ] - ctr_id=${output} - run bash -c ${PODMAN_BINARY} ${PODMAN_OPTIONS} wait $ctr_id - [ "$status" -eq 0 ] -} - -@test "wait on a sleeping container" { - run ${PODMAN_BINARY} ${PODMAN_OPTIONS} run -d ${ALPINE} sleep 10 - echo $output - [ "$status" -eq 0 ] - ctr_id=${output} - run bash -c ${PODMAN_BINARY} ${PODMAN_OPTIONS} wait $ctr_id - [ "$status" -eq 0 ] -} - -@test "wait on the latest container" { - ${PODMAN_BINARY} ${PODMAN_OPTIONS} run -d ${ALPINE} sleep 5 - run bash -c ${PODMAN_BINARY} ${PODMAN_OPTIONS} wait -l - echo "$output" - [ "$status" -eq 0 ] -} |