From f6963cea130133086f739173b55d1af74a680fef Mon Sep 17 00:00:00 2001 From: Chris Evich Date: Thu, 10 Mar 2022 12:51:47 -0500 Subject: Cirrus: Build multi-arch images + manifests Github-actions for large/complex tasks is hard to read and maintain. Reimplement the multi-arch image build workflow into a set of bash scripts that use all native contrainer-org tooling. This requires a special VM image setup with emulation to build foreign architectures. It also requires renaming the `helloimage` directory, because the build script uses the directory name in the image FQIN. Signed-off-by: Chris Evich --- .cirrus.yml | 37 +++++- .github/workflows/multi-arch-build.yaml | 212 -------------------------------- contrib/hello/Containerfile | 11 ++ contrib/hello/README.md | 77 ++++++++++++ contrib/hello/podman_hello_world.c | 26 ++++ contrib/helloimage/Containerfile | 11 -- contrib/helloimage/README.md | 77 ------------ contrib/helloimage/podman_hello_world.c | 26 ---- 8 files changed, 150 insertions(+), 327 deletions(-) delete mode 100644 .github/workflows/multi-arch-build.yaml create mode 100644 contrib/hello/Containerfile create mode 100644 contrib/hello/README.md create mode 100644 contrib/hello/podman_hello_world.c delete mode 100644 contrib/helloimage/Containerfile delete mode 100644 contrib/helloimage/README.md delete mode 100644 contrib/helloimage/podman_hello_world.c diff --git a/.cirrus.yml b/.cirrus.yml index ae5463427..8bf01bb50 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -38,7 +38,7 @@ env: UBUNTU_NAME: "ubuntu-2110" # Google-cloud VM Images - IMAGE_SUFFIX: "c5814666029957120" + IMAGE_SUFFIX: "c6261670816251904" FEDORA_CACHE_IMAGE_NAME: "fedora-${IMAGE_SUFFIX}" PRIOR_FEDORA_CACHE_IMAGE_NAME: "prior-fedora-${IMAGE_SUFFIX}" UBUNTU_CACHE_IMAGE_NAME: "ubuntu-${IMAGE_SUFFIX}" @@ -743,6 +743,39 @@ upgrade_test_task: always: *logs_artifacts +image_build_task: + name: "Build multi-arch $CTXDIR" + alias: image_build + # Some of these container images take > 1h to build, limit + # this task to a specific Cirrus-Cron entry with this name. + only_if: $CIRRUS_CRON == 'multiarch' + depends_on: + - ext_svc_check + timeout_in: 120m # emulation is sssllllooooowwww + gce_instance: + <<: *standardvm + image_name: build-push-${IMAGE_SUFFIX} + # More muscle required for parallel multi-arch build + type: "n2-standard-4" + env: + PODMAN_USERNAME: ENCRYPTED[b9f0f2550029dd2196e086d9dd6c2d1fec7e328630b15990d9bb610f9fcccb5baab8b64a8c3e72b0c1d0f5917cf65aa1] + PODMAN_PASSWORD: ENCRYPTED[e3444f6072853f0c8db7f964ead5e2204116af485469fa0de367f26b9316b460fd842a9882f552b9e9a83bbaf650d8b4] + CONTAINERS_USERNAME: ENCRYPTED[54a372d5f22f424173c114c6fb25c3214956cad323d5b285c7393a71041884ce96471d0ff733774e5dab9fa5a3c8795c] + CONTAINERS_PASSWORD: ENCRYPTED[4ecc3fb534935095a99fb1f2e320ac6bc87f3e7e186746e41cbcc4b5f5379a014b9fc8cc90e1f3d5abdbaf31580a4ab9] + matrix: + - env: + CTXDIR: contrib/podmanimage/upstream + - env: + CTXDIR: contrib/podmanimage/testing + - env: + CTXDIR: contrib/podmanimage/stable + - env: + CTXDIR: contrib/hello + script: + - set -a; source /etc/automation_environment; set +a + - main.sh $CIRRUS_REPO_CLONE_URL $CTXDIR + + # This task is critical. It updates the "last-used by" timestamp stored # in metadata for all VM images. This mechanism functions in tandem with # an out-of-band pruning operation to remove disused VM images. @@ -759,6 +792,7 @@ meta_task: ${FEDORA_CACHE_IMAGE_NAME} ${PRIOR_FEDORA_CACHE_IMAGE_NAME} ${UBUNTU_CACHE_IMAGE_NAME} + build-push-${IMAGE_SUFFIX} BUILDID: "${CIRRUS_BUILD_ID}" REPOREF: "${CIRRUS_REPO_NAME}" GCPJSON: ENCRYPTED[3a198350077849c8df14b723c0f4c9fece9ebe6408d35982e7adf2105a33f8e0e166ed3ed614875a0887e1af2b8775f4] @@ -801,6 +835,7 @@ success_task: - rootless_gitlab_test - upgrade_test - buildah_bud_test + - image_build - meta container: *smallcontainer env: diff --git a/.github/workflows/multi-arch-build.yaml b/.github/workflows/multi-arch-build.yaml deleted file mode 100644 index 1dc485d71..000000000 --- a/.github/workflows/multi-arch-build.yaml +++ /dev/null @@ -1,212 +0,0 @@ ---- - -# Please see contrib/image/README.md for details on the intentions -# of this workflow. -# -# BIG FAT WARNING: This workflow is duplicated across containers/skopeo, -# containers/buildah, and containers/podman. ANY AND -# ALL CHANGES MADE HERE MUST BE MANUALLY DUPLICATED -# TO THE OTHER REPOS. - -name: build multi-arch images - -on: - # Upstream tends to be very active, with many merges per day. - # Only run this daily via cron schedule, or manually, not by branch push. - schedule: - - cron: '0 8 * * *' - # allows to run this workflow manually from the Actions tab - workflow_dispatch: - -permissions: - contents: read - -jobs: - multi: - name: multi-arch image build - env: - REPONAME: podman # No easy way to parse this out of $GITHUB_REPOSITORY - # Server/namespace value used to format FQIN - REPONAME_QUAY_REGISTRY: quay.io/podman - CONTAINERS_QUAY_REGISTRY: quay.io/containers - # list of architectures for build - PLATFORMS: linux/amd64,linux/s390x,linux/ppc64le,linux/arm64 - # Command to execute in container to obtain project version number - VERSION_CMD: "podman --version" - - # build several images (upstream, testing, stable) in parallel - strategy: - # By default, failure of one matrix item cancels all others - fail-fast: false - matrix: - # Builds are located under contrib/image/ directory - source: - - upstream - - testing - - stable - runs-on: ubuntu-latest - # internal registry caches build for inspection before push - services: - registry: - image: quay.io/libpod/registry:2 - ports: - - 5000:5000 - steps: - - name: Checkout - uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2 - - - name: Set up QEMU - uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480 # v1 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25 # v1 - with: - driver-opts: network=host - install: true - - - name: Build and locally push image - uses: docker/build-push-action@ac9327eae2b366085ac7f6a2d02df8aa8ead720a # v2 - with: - context: contrib/${{ env.REPONAME }}image/${{ matrix.source }} - file: ./contrib/${{ env.REPONAME }}image/${{ matrix.source }}/Dockerfile - platforms: ${{ env.PLATFORMS }} - push: true - tags: localhost:5000/${{ env.REPONAME }}/${{ matrix.source }} - - # Simple verification that stable images work, and - # also grab version number use in forming the FQIN. - - name: amd64 container sniff test - if: matrix.source == 'stable' - id: sniff_test - run: | - podman pull --tls-verify=false \ - localhost:5000/$REPONAME/${{ matrix.source }} - VERSION_OUTPUT=$(podman run \ - localhost:5000/$REPONAME/${{ matrix.source }} \ - $VERSION_CMD) - echo "$VERSION_OUTPUT" - VERSION=$(awk -r -e "/^${REPONAME} version /"'{print $3}' <<<"$VERSION_OUTPUT") - test -n "$VERSION" - echo "::set-output name=version::$VERSION" - - - name: Generate image FQIN(s) to push - id: reponame_reg - run: | - if [[ "${{ matrix.source }}" == 'stable' ]]; then - # The command version in image just built - VERSION='v${{ steps.sniff_test.outputs.version }}' - # workaround vim syntax-highlight bug: ' - # Push both new|updated version-tag and latest-tag FQINs - FQIN="$REPONAME_QUAY_REGISTRY/stable:$VERSION,$REPONAME_QUAY_REGISTRY/stable:latest" - elif [[ "${{ matrix.source }}" == 'testing' ]]; then - # Assume some contents changed, always push latest testing. - FQIN="$REPONAME_QUAY_REGISTRY/testing:latest" - elif [[ "${{ matrix.source }}" == 'upstream' ]]; then - # Assume some contents changed, always push latest upstream. - FQIN="$REPONAME_QUAY_REGISTRY/upstream:latest" - else - echo "::error::Unknown matrix item '${{ matrix.source }}'" - exit 1 - fi - echo "::warning::Pushing $FQIN" - echo "::set-output name=fqin::${FQIN}" - echo '::set-output name=push::true' - - # This is substantially similar to the above logic, - # but only handles $CONTAINERS_QUAY_REGISTRY for - # the stable "latest" and named-version tagged images. - - name: Generate containers reg. image FQIN(s) - if: matrix.source == 'stable' - id: containers_reg - run: | - VERSION='v${{ steps.sniff_test.outputs.version }}' - # workaround vim syntax-highlight bug: ' - # Push both new|updated version-tag and latest-tag FQINs - FQIN="$CONTAINERS_QUAY_REGISTRY/$REPONAME:$VERSION,$CONTAINERS_QUAY_REGISTRY/$REPONAME:latest" - echo "::warning::Pushing $FQIN" - echo "::set-output name=fqin::${FQIN}" - echo '::set-output name=push::true' - - - name: Define LABELS multi-line env. var. value - run: | - # This is a really hacky/strange workflow idiom, required - # for setting multi-line $LABELS value for consumption in - # a future step. There is literally no cleaner way to do this :< - # https://docs.github.com/en/actions/reference/workflow-commands-for-github-actions#multiline-strings - function set_labels() { - echo 'LABELS<> "$GITHUB_ENV" - for line; do - echo "$line" | tee -a "$GITHUB_ENV" - done - echo "DELIMITER" >> "$GITHUB_ENV" - } - - declare -a lines - lines=(\ - "org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}.git" - "org.opencontainers.image.revision=${GITHUB_SHA}" - "org.opencontainers.image.created=$(date -u --iso-8601=seconds)" - ) - - # Only the 'stable' matrix source obtains $VERSION - if [[ "${{ matrix.source }}" == "stable" ]]; then - lines+=(\ - "org.opencontainers.image.version=${{ steps.sniff_test.outputs.version }}" - ) - fi - - set_labels "${lines[@]}" - - # Separate steps to login and push for $REPONAME_QUAY_REGISTRY and - # $CONTAINERS_QUAY_REGISTRY are required, because 2 sets of credentials - # are used and namespaced within the registry. At the same time, reuse - # of non-shell steps is not supported by Github Actions nor are YAML - # anchors/aliases, nor composite actions. - - # Push to $REPONAME_QUAY_REGISTRY for stable, testing. and upstream - - name: Login to ${{ env.REPONAME_QUAY_REGISTRY }} - uses: docker/login-action@dd4fa0671be5250ee6f50aedf4cb05514abda2c7 # v1 - if: steps.reponame_reg.outputs.push == 'true' - with: - registry: ${{ env.REPONAME_QUAY_REGISTRY }} - # N/B: Secrets are not passed to workflows that are triggered - # by a pull request from a fork - username: ${{ secrets.REPONAME_QUAY_USERNAME }} - password: ${{ secrets.REPONAME_QUAY_PASSWORD }} - - - name: Push images to ${{ steps.reponame_reg.outputs.fqin }} - uses: docker/build-push-action@ac9327eae2b366085ac7f6a2d02df8aa8ead720a # v2 - if: steps.reponame_reg.outputs.push == 'true' - with: - cache-from: type=registry,ref=localhost:5000/${{ env.REPONAME }}/${{ matrix.source }} - cache-to: type=inline - context: contrib/${{ env.REPONAME }}image/${{ matrix.source }} - file: ./contrib/${{ env.REPONAME }}image/${{ matrix.source }}/Dockerfile - platforms: ${{ env.PLATFORMS }} - push: true - tags: ${{ steps.reponame_reg.outputs.fqin }} - labels: | - ${{ env.LABELS }} - - # Push to $CONTAINERS_QUAY_REGISTRY only stable - - name: Login to ${{ env.CONTAINERS_QUAY_REGISTRY }} - if: steps.containers_reg.outputs.push == 'true' - uses: docker/login-action@dd4fa0671be5250ee6f50aedf4cb05514abda2c7 # v1 - with: - registry: ${{ env.CONTAINERS_QUAY_REGISTRY}} - username: ${{ secrets.CONTAINERS_QUAY_USERNAME }} - password: ${{ secrets.CONTAINERS_QUAY_PASSWORD }} - - - name: Push images to ${{ steps.containers_reg.outputs.fqin }} - if: steps.containers_reg.outputs.push == 'true' - uses: docker/build-push-action@ac9327eae2b366085ac7f6a2d02df8aa8ead720a # v2 - with: - cache-from: type=registry,ref=localhost:5000/${{ env.REPONAME }}/${{ matrix.source }} - cache-to: type=inline - context: contrib/${{ env.REPONAME }}image/${{ matrix.source }} - file: ./contrib/${{ env.REPONAME }}image/${{ matrix.source }}/Dockerfile - platforms: ${{ env.PLATFORMS }} - push: true - tags: ${{ steps.containers_reg.outputs.fqin }} - labels: | - ${{ env.LABELS }} diff --git a/contrib/hello/Containerfile b/contrib/hello/Containerfile new file mode 100644 index 000000000..0cbf6d9a0 --- /dev/null +++ b/contrib/hello/Containerfile @@ -0,0 +1,11 @@ +FROM docker.io/alpine as builder +RUN apk add gcc libc-dev +ADD podman_hello_world.c . +RUN gcc -O2 -static -o podman_hello_world podman_hello_world.c + +FROM scratch +LABEL maintainer="Podman Maintainers" +LABEL artist="Máirín Ní Ḋuḃṫaiġ, Twitter:@mairin" +USER 1000 +COPY --from=builder podman_hello_world /usr/local/bin/podman_hello_world +CMD ["/usr/local/bin/podman_hello_world"] diff --git a/contrib/hello/README.md b/contrib/hello/README.md new file mode 100644 index 000000000..528466f7b --- /dev/null +++ b/contrib/hello/README.md @@ -0,0 +1,77 @@ +![PODMAN logo](../../logo/podman-logo-source.svg) + +# Podman Hello World image + +## Overview + +This directory contains the Containerfile and bash script necessary to create the +"hello" podman image housed on quay.io under the Podman account in a public +repository. The image is public and can be pulled without credentials. + +Using this image is helpful to: + + * Prove that basic Podman operations are working on the host. + * Shows that the image was pulled from the quay.io container registry. + * Container creation was successfully accomplished. (`podman ps -a`) + * The created container was able to stream output to your terminal. + +## Directory Contents + +The contents of this directory contain: + * ./Containerfile + * ./podman_hello_world.c + +## Sample Usage + +To simply run the image: + +``` +podman run quay.io/podman/hello + +!... Hello Podman World ...! + + .--"--. + / - - \ + / (O) (O) \ + ~~~| -=(,Y,)=- | + .---. /` \ |~~ + ~/ o o \~~~~.----. ~~ + | =(X)= |~ / (O (O) \ + ~~~~~~~ ~| =(Y_)=- | + ~~~~ ~~~| U |~~ + +Project: https://github.com/containers/podman +Website: https://podman.io +Documents: https://docs.podman.io +Twitter: @Podman_io +``` +To build the image yourself, copy the files from this directory into +a local directory and issue these commands: + +``` +podman build -t myhello . +podman run myhello +``` + +## Potential Issues: + +The image runs as a rootless user with the UID set to `1000`. +If the /etc/subuid and /etch/subgid values are not set appropriately to run as a +rootless user on the host, an error like this might be raised: + +``` +Copying blob acab339ca1e8 done +ERRO[0002] Error while applying layer: ApplyLayer exit status 1 stdout: stderr: potentially insufficient UIDs or GIDs available in user namespace (requested 0:12 for /var/spool/mail): Check /etc/subuid and /etc/subgid: lchown /var/spool/mail: invalid argument +Error: writing blob: adding layer with blob "sha256:ee0cde9de8a68f171a8c03b0e9954abf18576947e2f3187e84d8c31ccd8f6a09": ApplyLayer exit status 1 stdout: stderr: potentially insufficient UIDs or GIDs available in user namespace (requested 0:12 for /var/spool/mail): Check /etc/subuid and /etc/subgid: lchown /var/spool/mail: invalid argument +``` + +Please refer to this [blog post](https://www.redhat.com/sysadmin/rootless-podman) for further configuration information. + +## THANKS! + +Many Thanks to @afbjorklund for a great discussion during the +first revision of this container image that resulted in moving +from using bash to using C, and the ensuing changes to the +Containerfile. + +Also many thanks to @mairin for the awesome ASCII art! diff --git a/contrib/hello/podman_hello_world.c b/contrib/hello/podman_hello_world.c new file mode 100644 index 000000000..ee574130d --- /dev/null +++ b/contrib/hello/podman_hello_world.c @@ -0,0 +1,26 @@ +//### +// ASCII art by the incomparable Máirín Duffy, +// duffy@redhat.com, Twitter: @mairin +// January 2022 +//### + +#include +int main() { + puts("\ +!... Hello Podman World ...!\n\ +\n\ + .--\"--. \n\ + / - - \\ \n\ + / (O) (O) \\ \n\ + ~~~| -=(,Y,)=- | \n\ + .---. /` \\ |~~ \n\ + ~/ o o \\~~~~.----. ~~ \n\ + | =(X)= |~ / (O (O) \\ \n\ + ~~~~~~~ ~| =(Y_)=- | \n\ + ~~~~ ~~~| U |~~ \n\ +\n\ +Project: https://github.com/containers/podman\n\ +Website: https://podman.io\n\ +Documents: https://docs.podman.io\n\ +Twitter: @Podman_io"); +} diff --git a/contrib/helloimage/Containerfile b/contrib/helloimage/Containerfile deleted file mode 100644 index 0cbf6d9a0..000000000 --- a/contrib/helloimage/Containerfile +++ /dev/null @@ -1,11 +0,0 @@ -FROM docker.io/alpine as builder -RUN apk add gcc libc-dev -ADD podman_hello_world.c . -RUN gcc -O2 -static -o podman_hello_world podman_hello_world.c - -FROM scratch -LABEL maintainer="Podman Maintainers" -LABEL artist="Máirín Ní Ḋuḃṫaiġ, Twitter:@mairin" -USER 1000 -COPY --from=builder podman_hello_world /usr/local/bin/podman_hello_world -CMD ["/usr/local/bin/podman_hello_world"] diff --git a/contrib/helloimage/README.md b/contrib/helloimage/README.md deleted file mode 100644 index 528466f7b..000000000 --- a/contrib/helloimage/README.md +++ /dev/null @@ -1,77 +0,0 @@ -![PODMAN logo](../../logo/podman-logo-source.svg) - -# Podman Hello World image - -## Overview - -This directory contains the Containerfile and bash script necessary to create the -"hello" podman image housed on quay.io under the Podman account in a public -repository. The image is public and can be pulled without credentials. - -Using this image is helpful to: - - * Prove that basic Podman operations are working on the host. - * Shows that the image was pulled from the quay.io container registry. - * Container creation was successfully accomplished. (`podman ps -a`) - * The created container was able to stream output to your terminal. - -## Directory Contents - -The contents of this directory contain: - * ./Containerfile - * ./podman_hello_world.c - -## Sample Usage - -To simply run the image: - -``` -podman run quay.io/podman/hello - -!... Hello Podman World ...! - - .--"--. - / - - \ - / (O) (O) \ - ~~~| -=(,Y,)=- | - .---. /` \ |~~ - ~/ o o \~~~~.----. ~~ - | =(X)= |~ / (O (O) \ - ~~~~~~~ ~| =(Y_)=- | - ~~~~ ~~~| U |~~ - -Project: https://github.com/containers/podman -Website: https://podman.io -Documents: https://docs.podman.io -Twitter: @Podman_io -``` -To build the image yourself, copy the files from this directory into -a local directory and issue these commands: - -``` -podman build -t myhello . -podman run myhello -``` - -## Potential Issues: - -The image runs as a rootless user with the UID set to `1000`. -If the /etc/subuid and /etch/subgid values are not set appropriately to run as a -rootless user on the host, an error like this might be raised: - -``` -Copying blob acab339ca1e8 done -ERRO[0002] Error while applying layer: ApplyLayer exit status 1 stdout: stderr: potentially insufficient UIDs or GIDs available in user namespace (requested 0:12 for /var/spool/mail): Check /etc/subuid and /etc/subgid: lchown /var/spool/mail: invalid argument -Error: writing blob: adding layer with blob "sha256:ee0cde9de8a68f171a8c03b0e9954abf18576947e2f3187e84d8c31ccd8f6a09": ApplyLayer exit status 1 stdout: stderr: potentially insufficient UIDs or GIDs available in user namespace (requested 0:12 for /var/spool/mail): Check /etc/subuid and /etc/subgid: lchown /var/spool/mail: invalid argument -``` - -Please refer to this [blog post](https://www.redhat.com/sysadmin/rootless-podman) for further configuration information. - -## THANKS! - -Many Thanks to @afbjorklund for a great discussion during the -first revision of this container image that resulted in moving -from using bash to using C, and the ensuing changes to the -Containerfile. - -Also many thanks to @mairin for the awesome ASCII art! diff --git a/contrib/helloimage/podman_hello_world.c b/contrib/helloimage/podman_hello_world.c deleted file mode 100644 index ee574130d..000000000 --- a/contrib/helloimage/podman_hello_world.c +++ /dev/null @@ -1,26 +0,0 @@ -//### -// ASCII art by the incomparable Máirín Duffy, -// duffy@redhat.com, Twitter: @mairin -// January 2022 -//### - -#include -int main() { - puts("\ -!... Hello Podman World ...!\n\ -\n\ - .--\"--. \n\ - / - - \\ \n\ - / (O) (O) \\ \n\ - ~~~| -=(,Y,)=- | \n\ - .---. /` \\ |~~ \n\ - ~/ o o \\~~~~.----. ~~ \n\ - | =(X)= |~ / (O (O) \\ \n\ - ~~~~~~~ ~| =(Y_)=- | \n\ - ~~~~ ~~~| U |~~ \n\ -\n\ -Project: https://github.com/containers/podman\n\ -Website: https://podman.io\n\ -Documents: https://docs.podman.io\n\ -Twitter: @Podman_io"); -} -- cgit v1.2.3-54-g00ecf