diff options
79 files changed, 1185 insertions, 2411 deletions
diff --git a/.cirrus.yml b/.cirrus.yml index 67c212c15..9d220c69a 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -46,16 +46,6 @@ env: PRIOR_UBUNTU_CACHE_IMAGE_NAME: "${PRIOR_UBUNTU_NAME}-${_BUILT_IMAGE_SUFFIX}" #### - #### Variables for composing new cache-images (used in PR testing) from - #### base-images (pre-existing in GCE) - #### - BUILT_IMAGE_SUFFIX: "-${CIRRUS_REPO_NAME}-${CIRRUS_BUILD_ID}" - # Special image w/ nested-libvirt + tools for creating new cache and base images - IMAGE_BUILDER_CACHE_IMAGE_NAME: "image-builder-image-1541772081" - # Name where this repositories VM images are stored - GCP_PROJECT_ID: libpod-218412 - - #### #### Default to NOT operating in any special-case testing mode #### SPECIALMODE: "none" # don't do anything special @@ -66,8 +56,8 @@ env: #### #### Credentials and other secret-sauces, decrypted at runtime when authorized. #### - # Freenode IRC credentials for posting status messages - IRCID: ENCRYPTED[0c4a3cc4ecda08bc47cd3d31592be8ae5c2bd0151bf3def00a9afd139ef1ab23a1bd0523319d076c027f9749ddb1f3c8] + # Name where this repositories VM images are stored + GCP_PROJECT_ID: libpod-218412 # Service-account client_email - needed to build images SERVICE_ACCOUNT: ENCRYPTED[702a8e07e27a6faf7988fcddcc068c2ef2bb182a5aa671f5ccb7fbbfb891c823aa4a7856fb17240766845dbd68bd3f90] # Service account username part of client_email - for ssh'ing into VMs @@ -138,38 +128,24 @@ gating_task: # Verify some aspects of ci/related scripts ci_script: - '${GOSRC}/${SCRIPT_BASE}/lib.sh.t |& ${TIMESTAMP}' - - '/usr/local/bin/entrypoint.sh -C ${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/packer test' - '${GOSRC}/${SCRIPT_BASE}/cirrus_yaml_test.py |& ${TIMESTAMP}' # Verify expected bash environment (-o pipefail) pipefail_enabledscript: 'if /bin/false | /bin/true; then echo "pipefail fault" && exit 72; fi' - on_failure: - failed_branch_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_branch_failure.sh' - # Ensure these container images can build container_image_build_task: alias: 'container_image_build' + name: "build gate image $DEST_BRANCH branch" depends_on: - "gating" # Only run for PRs, quay.io will automatically build after branch-push only_if: $CIRRUS_BRANCH != $DEST_BRANCH - matrix: - - name: "build in_podman image ${FEDORA_NAME} " - container: - dockerfile: Dockerfile - - name: "build in_podman image ${UBUNTU_NAME}" - container: - dockerfile: Dockerfile.ubuntu - - name: "build gate image $DEST_BRANCH branch" - container: - dockerfile: contrib/gate/Dockerfile - container: - dockerfile: Dockerfile + dockerfile: contrib/gate/Dockerfile script: make install.remote @@ -179,7 +155,6 @@ container_image_build_task: rpmbuild_task: only_if: >- - $CIRRUS_CHANGE_TITLE !=~ '.*CI:IMG.*' && $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*' && $CIRRUS_BRANCH != $DEST_BRANCH @@ -197,18 +172,13 @@ rpmbuild_task: - 'make -C ${CIRRUS_WORKING_DIR} -f ${CIRRUS_WORKING_DIR}/.copr/Makefile' - 'rpmbuild --rebuild ${CIRRUS_WORKING_DIR}/podman-*.src.rpm' - on_failure: - failed_branch_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_branch_failure.sh |& ${TIMESTAMP}' - # This task runs `make vendor` followed by ./hack/tree_status.sh to check # whether the git tree is clean. The reasoning for that is to make sure # that the vendor.conf, the code and the vendored packages in ./vendor are # in sync at all times. vendor_task: - only_if: >- - $CIRRUS_CHANGE_TITLE !=~ '.*CI:IMG.*' && - $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*' + only_if: $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*' depends_on: - "gating" @@ -230,17 +200,12 @@ vendor_task: - 'cd ${CIRRUS_WORKING_DIR} && make vendor' - 'cd ${CIRRUS_WORKING_DIR} && ./hack/tree_status.sh' - on_failure: - failed_branch_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_branch_failure.sh |& ${TIMESTAMP}' - # This task runs `make varlink_api_generate` followed by ./hack/tree_status.sh to check # whether the git tree is clean. varlink_api_task: - only_if: >- - $CIRRUS_CHANGE_TITLE !=~ '.*CI:IMG.*' && - $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*' + only_if: $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*' depends_on: - "gating" @@ -265,9 +230,6 @@ varlink_api_task: - '/usr/local/bin/entrypoint.sh BUILDTAGS="varlink" varlink_api_generate |& ${TIMESTAMP}' - 'cd ${GOSRC} && ./hack/tree_status.sh |& ${TIMESTAMP}' - on_failure: - failed_branch_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_branch_failure.sh' - build_each_commit_task: @@ -278,7 +240,6 @@ build_each_commit_task: only_if: >- $CIRRUS_BRANCH != $DEST_BRANCH && - $CIRRUS_CHANGE_TITLE !=~ '.*CI:IMG.*' && $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*' gce_instance: @@ -297,9 +258,6 @@ build_each_commit_task: - 'git fetch --depth 50 origin $DEST_BRANCH |& ${TIMESTAMP}' - 'make build-all-new-commits GIT_BASE_BRANCH=origin/$DEST_BRANCH |& ${TIMESTAMP}' - on_failure: - failed_branch_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_branch_failure.sh' - build_without_cgo_task: @@ -310,7 +268,6 @@ build_without_cgo_task: only_if: >- $CIRRUS_BRANCH != $DEST_BRANCH && - $CIRRUS_CHANGE_TITLE !=~ '.*CI:IMG.*' && $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*' gce_instance: @@ -327,9 +284,6 @@ build_without_cgo_task: - 'source $SCRIPT_BASE/lib.sh' - 'make build-no-cgo' - on_failure: - failed_branch_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_branch_failure.sh' - # Update metadata on VM images referenced by this repository state meta_task: @@ -360,32 +314,6 @@ meta_task: script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/update_meta.sh |& ${TIMESTAMP}' -# Remove old and disused images based on labels set by meta_task -image_prune_task: - - # This should ONLY ever run from the master branch, and never - # anywhere else so it's behavior is always consistent, even - # as new branches are created. - only_if: $CIRRUS_BRANCH == "master" - - depends_on: - - "meta" - - container: - image: "quay.io/libpod/imgprune:master" # see contrib/imgprune - cpu: 1 - memory: 1 - - env: - <<: *meta_env_vars - GCPJSON: ENCRYPTED[766916fedf780cbc16ac3152f7f73c5d9dcf64768fc6e80b0858c5badd31e7b41f3c864405c814189fd340e5a056ba18] - GCPNAME: ENCRYPTED[d6869741209b8cf380adb8a3858cbce4542c9cf115452fcd2024a176b08fce10112e8bf0fbcc2f0033e7b87ef4342b3a] - - timeout_in: 10m - - script: '/usr/local/bin/entrypoint.sh |& ${TIMESTAMP}' - - # This task does the unit and integration testing for every platform testing_task: @@ -399,9 +327,7 @@ testing_task: - "container_image_build" # Only test build cache-images, if that's what's requested - only_if: >- - $CIRRUS_CHANGE_TITLE !=~ '.*CI:IMG.*' && - $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*' + only_if: $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*' matrix: - name: "test ${FEDORA_NAME}" @@ -435,9 +361,6 @@ testing_task: path: "*.tar.gz" type: "application/x-tar" - on_failure: - failed_branch_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_branch_failure.sh' - always: &standardlogs package_versions_script: '$SCRIPT_BASE/logcollector.sh packages' ginkgo_node_logs_script: '$SCRIPT_BASE/logcollector.sh ginkgo' @@ -460,9 +383,7 @@ special_testing_rootless_task: - "build_each_commit" - "build_without_cgo" - only_if: >- - $CIRRUS_CHANGE_TITLE !=~ '.*CI:IMG.*' && - $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*' + only_if: $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*' env: ADD_SECOND_PARTITION: 'true' @@ -477,9 +398,6 @@ special_testing_rootless_task: system_test_script: '$SCRIPT_BASE/system_test.sh |& ${TIMESTAMP} | ${LOGFORMAT} system_test' apiv2_test_script: '$SCRIPT_BASE/apiv2_test.sh |& ${TIMESTAMP} | ${LOGFORMAT} apiv2_test' - on_failure: - failed_branch_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_branch_failure.sh' - always: <<: *standardlogs @@ -494,9 +412,7 @@ special_testing_in_podman_task: - "build_each_commit" - "build_without_cgo" - only_if: >- - $CIRRUS_CHANGE_TITLE !=~ '.*CI:IMG.*' && - $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*' + only_if: $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*' matrix: - name: "in-podman ${PRIOR_FEDORA_NAME}" @@ -515,9 +431,6 @@ special_testing_in_podman_task: setup_environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}' integration_test_script: '$SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP} | ${LOGFORMAT} integration_test' - on_failure: - failed_branch_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_branch_failure.sh' - always: <<: *standardlogs @@ -530,9 +443,7 @@ special_testing_cross_task: - "varlink_api" - "vendor" - only_if: >- - $CIRRUS_CHANGE_TITLE !=~ '.*CI:IMG.*' && - $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*' + only_if: $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*' matrix: - name: 'cross-platform: windows' @@ -548,9 +459,6 @@ special_testing_cross_task: setup_environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}' build_release_script: '$SCRIPT_BASE/build_release.sh |& ${TIMESTAMP}' - on_failure: - failed_branch_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_branch_failure.sh' - # When examining a particular run, provide convenient access to release files. zip_artifacts: path: "*.zip" @@ -568,9 +476,7 @@ special_testing_bindings_task: - "varlink_api" - "vendor" - only_if: >- - $CIRRUS_CHANGE_TITLE !=~ '.*CI:IMG.*' && - $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*' + only_if: $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*' env: SPECIALMODE: 'bindings' # See docs @@ -581,9 +487,6 @@ special_testing_bindings_task: setup_environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}' integration_test_script: '$SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP} | ${LOGFORMAT} integration_test' - on_failure: - failed_branch_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_branch_failure.sh' - always: <<: *standardlogs @@ -595,9 +498,7 @@ special_testing_endpoint_task: - "varlink_api" - "vendor" - only_if: >- - $CIRRUS_CHANGE_TITLE !=~ '.*CI:IMG.*' && - $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*' + only_if: $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*' env: SPECIALMODE: 'endpoint' # See docs @@ -606,96 +507,12 @@ special_testing_endpoint_task: setup_environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}' integration_test_script: '$SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP} | ${LOGFORMAT} integration_test' - - on_failure: - failed_branch_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_branch_failure.sh' - - always: - <<: *standardlogs - - -# Test building of new cache-images for future PR testing, in this PR. -test_build_cache_images_task: - - only_if: >- - $CIRRUS_BRANCH != $DEST_BRANCH && - $CIRRUS_CHANGE_TITLE =~ '.*CI:IMG.*' && - $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*' - - depends_on: - - "gating" - - 'container_image_build' - - # VMs created by packer are not cleaned up by cirrus, must allow task to complete - auto_cancellation: $CI != "true" - - gce_instance: - image_project: $GCP_PROJECT_ID - zone: "us-central1-a" - cpu: 4 - memory: "4Gb" - disk: 200 - image_name: "${IMAGE_BUILDER_CACHE_IMAGE_NAME}" - scopes: # required for image building - - compute - - devstorage.full_control - - networking_script: '${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/networking.sh' - build_vm_images_script: '$SCRIPT_BASE/build_vm_images.sh |& ${TIMESTAMP}' - - on_failure: - failed_df_script: '${DFCMD}' - failed_journalctl_b_script: 'journalctl -b || echo "Uh oh, journalctl -b failed"' - - -# Test building of new cache-images for future PR testing, in this PR. -verify_test_built_images_task: - - only_if: >- - $CIRRUS_BRANCH != $DEST_BRANCH && - $CIRRUS_CHANGE_TITLE =~ '.*CI:IMG.*' && - $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*' - - - depends_on: - - "gating" - - "test_build_cache_images" - - gce_instance: - # Images generated by test_build_cache_images_task (above) - image_name: "${PACKER_BUILDER_NAME}${BUILT_IMAGE_SUFFIX}" - - env: - ADD_SECOND_PARTITION: 'true' - matrix: - - RCLI: 'true' - - RCLI: 'false' - matrix: - PACKER_BUILDER_NAME: "${FEDORA_NAME}" - PACKER_BUILDER_NAME: "${PRIOR_FEDORA_NAME}" - PACKER_BUILDER_NAME: "${UBUNTU_NAME}" - PACKER_BUILDER_NAME: "${PRIOR_UBUNTU_NAME}" - - networking_script: '${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/networking.sh' - installed_packages_script: '$SCRIPT_BASE/logcollector.sh packages' - environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}' - # Verify expectations of built images - check_image_script: '$SCRIPT_BASE/check_image.sh |& ${TIMESTAMP}' - # Note: A truncated form of normal testing. It only needs to confirm new images - # "probably" work. A full round of testing will happen again after $*_CACHE_IMAGE_NAME - # are updated in this or another PR (w/o '***CIRRUS: TEST IMAGES***'). - integration_test_script: '$SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP}' - system_test_script: '$SCRIPT_BASE/system_test.sh |& ${TIMESTAMP}' - always: <<: *standardlogs docs_task: - # Don't run this when building/testing new VM images - only_if: $CIRRUS_CHANGE_TITLE !=~ '.*CI:IMG.*' - depends_on: - "gating" env: @@ -724,7 +541,6 @@ success_task: - "build_without_cgo" - "container_image_build" - "meta" - - "image_prune" - "testing" - "rpmbuild" - "special_testing_rootless" @@ -732,8 +548,6 @@ success_task: - "special_testing_cross" - "special_testing_endpoint" - "special_testing_bindings" - - "test_build_cache_images" - - "verify_test_built_images" - "docs" - "static_build" - "darwin_build" @@ -749,7 +563,7 @@ success_task: cpu: 1 memory: 1 - success_script: '/usr/local/bin/entrypoint.sh ./$SCRIPT_BASE/success.sh |& ${TIMESTAMP}' + success_script: /bin/true # Build the static binary static_build_task: diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index c5120e440..000000000 --- a/Dockerfile +++ /dev/null @@ -1,21 +0,0 @@ -FROM registry.fedoraproject.org/fedora:latest - -# This container image is utilized by the containers CI automation system -# for building and testing libpod inside a container environment. -# It is assumed that the source to be tested will overwrite $GOSRC (below) -# at runtime. -ENV GOPATH=/var/tmp/go -ENV GOSRC=$GOPATH/src/github.com/containers/podman -ENV SCRIPT_BASE=./contrib/cirrus -ENV PACKER_BASE=$SCRIPT_BASE/packer - -ADD / $GOSRC -WORKDIR $GOSRC - -# Re-use repositories and package setup as in VMs under CI -RUN bash $PACKER_BASE/fedora_packaging.sh && \ - dnf clean all && \ - rm -rf /var/cache/dnf - -# Mirror steps taken under CI -RUN bash -c 'source $GOSRC/$SCRIPT_BASE/lib.sh && install_test_configs' diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu deleted file mode 100644 index df4ebf3d4..000000000 --- a/Dockerfile.ubuntu +++ /dev/null @@ -1,28 +0,0 @@ -# Must resemble $UBUNTU_BASE_IMAGE in ./contrib/cirrus/lib.sh -FROM ubuntu:20.04 - -# This container image is intended for building and testing libpod -# from inside a container environment. It is assumed that the source -# to be tested will overwrite $GOSRC (below) at runtime. -ENV GOPATH=/var/tmp/go -ENV GOSRC=$GOPATH/src/github.com/containers/podman -ENV SCRIPT_BASE=./contrib/cirrus -ENV PACKER_BASE=$SCRIPT_BASE/packer - -RUN export DEBIAN_FRONTEND="noninteractive" && \ - apt-get -qq update --yes && \ - apt-get -qq upgrade --yes && \ - apt-get -qq install curl git && \ - apt-get -qq autoremove --yes && \ - rm -rf /var/cache/apt - -ADD / $GOSRC -WORKDIR $GOSRC - -# Re-use repositories and package setup as in VMs under CI -RUN bash $PACKER_BASE/ubuntu_packaging.sh && \ - apt-get -qq autoremove --yes && \ - rm -rf /var/cache/apt - -# Mirror steps taken under CI -RUN bash -c 'source $GOSRC/$SCRIPT_BASE/lib.sh && install_test_configs' @@ -177,8 +177,8 @@ familiar container cli commands. For more details, see the [Container Tools Guide](https://github.com/containers/buildah/tree/master/docs/containertools). ## Podman Legacy API (Varlink) -Podman offers a Varlink-based API for remote management of containers. -However, this API has been deprecated by the REST API. +Podman offers a [Varlink-based API](https://github.com/containers/podman/blob/master/docs/tutorials/varlink_remote_client.md) +for remote management of containers. However, this API has been deprecated by the REST API. Varlink support is in maintenance mode, and will be removed in a future release. For more details, you can see [this blog](https://podman.io/blogs/2020/01/17/podman-new-api.html). diff --git a/cmd/podman/manifest/add.go b/cmd/podman/manifest/add.go index ca633263d..128bf66a7 100644 --- a/cmd/podman/manifest/add.go +++ b/cmd/podman/manifest/add.go @@ -4,14 +4,26 @@ import ( "context" "fmt" + "github.com/containers/common/pkg/auth" + "github.com/containers/image/v5/types" "github.com/containers/podman/v2/cmd/podman/registry" "github.com/containers/podman/v2/pkg/domain/entities" + "github.com/containers/podman/v2/pkg/util" "github.com/pkg/errors" "github.com/spf13/cobra" ) +// manifestAddOptsWrapper wraps entities.ManifestAddOptions and prevents leaking +// CLI-only fields into the API types. +type manifestAddOptsWrapper struct { + entities.ManifestAddOptions + + TLSVerifyCLI bool // CLI only + CredentialsCLI string +} + var ( - manifestAddOpts = entities.ManifestAddOptions{} + manifestAddOpts = manifestAddOptsWrapper{} addCmd = &cobra.Command{ Use: "add [flags] LIST LIST", Short: "Add images to a manifest list or image index", @@ -33,15 +45,48 @@ func init() { flags.BoolVar(&manifestAddOpts.All, "all", false, "add all of the list's images if the image is a list") flags.StringSliceVar(&manifestAddOpts.Annotation, "annotation", nil, "set an `annotation` for the specified image") flags.StringVar(&manifestAddOpts.Arch, "arch", "", "override the `architecture` of the specified image") + flags.StringVar(&manifestAddOpts.Authfile, "authfile", auth.GetDefaultAuthFile(), "path of the authentication file. Use REGISTRY_AUTH_FILE environment variable to override") + flags.StringVar(&manifestAddOpts.CertDir, "cert-dir", "", "use certificates at the specified path to access the registry") + flags.StringVar(&manifestAddOpts.CredentialsCLI, "creds", "", "use `[username[:password]]` for accessing the registry") + flags.StringSliceVar(&manifestAddOpts.Features, "features", nil, "override the `features` of the specified image") flags.StringVar(&manifestAddOpts.OS, "os", "", "override the `OS` of the specified image") flags.StringVar(&manifestAddOpts.OSVersion, "os-version", "", "override the OS `version` of the specified image") + flags.BoolVar(&manifestAddOpts.TLSVerifyCLI, "tls-verify", true, "require HTTPS and verify certificates when accessing the registry") flags.StringVar(&manifestAddOpts.Variant, "variant", "", "override the `Variant` of the specified image") + + if registry.IsRemote() { + _ = flags.MarkHidden("authfile") + _ = flags.MarkHidden("cert-dir") + _ = flags.MarkHidden("tls-verify") + } } func add(cmd *cobra.Command, args []string) error { + if err := auth.CheckAuthFile(manifestPushOpts.Authfile); err != nil { + return err + } + manifestAddOpts.Images = []string{args[1], args[0]} - listID, err := registry.ImageEngine().ManifestAdd(context.Background(), manifestAddOpts) + + if manifestAddOpts.CredentialsCLI != "" { + creds, err := util.ParseRegistryCreds(manifestAddOpts.CredentialsCLI) + if err != nil { + return err + } + manifestAddOpts.Username = creds.Username + manifestAddOpts.Password = creds.Password + } + + // TLS verification in c/image is controlled via a `types.OptionalBool` + // which allows for distinguishing among set-true, set-false, unspecified + // which is important to implement a sane way of dealing with defaults of + // boolean CLI flags. + if cmd.Flags().Changed("tls-verify") { + manifestAddOpts.SkipTLSVerify = types.NewOptionalBool(!manifestAddOpts.TLSVerifyCLI) + } + + listID, err := registry.ImageEngine().ManifestAdd(context.Background(), manifestAddOpts.ManifestAddOptions) if err != nil { return errors.Wrapf(err, "error adding to manifest list %s", args[0]) } diff --git a/cmd/podman/networks/create.go b/cmd/podman/networks/create.go index dabf6f0d2..68a577ae1 100644 --- a/cmd/podman/networks/create.go +++ b/cmd/podman/networks/create.go @@ -21,9 +21,6 @@ var ( RunE: networkCreate, Args: cobra.MaximumNArgs(1), Example: `podman network create podman1`, - Annotations: map[string]string{ - registry.ParentNSRequired: "", - }, } ) diff --git a/cmd/podman/networks/inspect.go b/cmd/podman/networks/inspect.go index f00d6b63c..c5872def7 100644 --- a/cmd/podman/networks/inspect.go +++ b/cmd/podman/networks/inspect.go @@ -22,9 +22,6 @@ var ( RunE: networkInspect, Example: `podman network inspect podman`, Args: cobra.MinimumNArgs(1), - Annotations: map[string]string{ - registry.ParentNSRequired: "", - }, } ) diff --git a/cmd/podman/networks/list.go b/cmd/podman/networks/list.go index 3a2651cbc..b6fb2bb80 100644 --- a/cmd/podman/networks/list.go +++ b/cmd/podman/networks/list.go @@ -25,9 +25,6 @@ var ( Long: networklistDescription, RunE: networkList, Example: `podman network list`, - Annotations: map[string]string{ - registry.ParentNSRequired: "", - }, } ) diff --git a/cmd/podman/networks/rm.go b/cmd/podman/networks/rm.go index dfbb5d081..ac49993b7 100644 --- a/cmd/podman/networks/rm.go +++ b/cmd/podman/networks/rm.go @@ -19,9 +19,6 @@ var ( RunE: networkRm, Example: `podman network rm podman`, Args: cobra.MinimumNArgs(1), - Annotations: map[string]string{ - registry.ParentNSRequired: "", - }, } ) diff --git a/completions/bash/podman b/completions/bash/podman index e250f344b..e8185235b 100644 --- a/completions/bash/podman +++ b/completions/bash/podman @@ -1846,6 +1846,9 @@ _podman_manifest() { _podman_manifest_add() { local options_with_args=" --annotation + --authfile + --cert-dir + --creds --arch --features --os @@ -1857,6 +1860,7 @@ _podman_manifest_add() { --all --help -h + --tls-verify " _complete_ "$options_with_args" "$boolean_options" diff --git a/contrib/cirrus/README.md b/contrib/cirrus/README.md index 977762293..f66560cc8 100644 --- a/contrib/cirrus/README.md +++ b/contrib/cirrus/README.md @@ -76,95 +76,6 @@ exercising cgroups v2 with Podman integration tests. Also depends on having `SPECIALMODE` set to 'cgroupv2` -### ``test_build_cache_images_task`` Task - -Modifying the contents of cache-images is tested by making changes to -one or more of the ``./contrib/cirrus/packer/*_setup.sh`` files. Then -in the PR description, add the magic string: ``[CI:IMG]`` - -***N/B: Steps below are performed by automation*** - -1. ``setup_environment.sh``: Same as for other tasks. - -2. ``build_vm_images.sh``: Utilize [the packer tool](http://packer.io/docs/) - to produce new VM images. Create a new VM from each base-image, connect - to them with ``ssh``, and perform the steps as defined by the - ``$PACKER_BASE/libpod_images.yml`` file: - - 1. On a base-image VM, as root, copy the current state of the repository - into ``/tmp/libpod``. - 2. Execute distribution-specific scripts to prepare the image for - use. For example, ``fedora_setup.sh``. - 3. If successful, shut down each VM and record the names, and dates - into a json manifest file. - 4. Move the manifest file, into a google storage bucket object. - This is a retained as a secondary method for tracking/auditing - creation of VM images, should it ever be needed. - -### ``verify_test_built_images`` Task - -Only runs following successful ``test_build_cache_images_task`` task. Uses -images following the standard naming format; ***however, only runs a limited -sub-set of automated tests***. Validating newly built images fully, requires -updating ``.cirrus.yml``. - -***N/B: Steps below are performed by automation*** - -1. Using the just build VM images, launch VMs and wait for them to boot. - -2. Execute the `setup_environment.sh` as in the `testing` task. - -2. Execute the `integration_test.sh` as in the `testing` task. - - -***Manual Steps:*** Assuming the automated steps pass, then -you'll find the new image names displayed at the end of the -`test_build_cache_images`. For example: - - -``` -...cut... - -[+0747s] ==> Builds finished. The artifacts of successful builds are: -[+0747s] --> ubuntu-18: A disk image was created: ubuntu-18-libpod-5664838702858240 -[+0747s] --> fedora-29: A disk image was created: fedora-29-libpod-5664838702858240 -[+0747s] --> fedora-30: A disk image was created: fedora-30-libpod-5664838702858240 -[+0747s] --> ubuntu-19: A disk image was created: ubuntu-19-libpod-5664838702858240 -``` - -Notice the suffix on all the image names comes from the env. var. set in -*.cirrus.yml*: `BUILT_IMAGE_SUFFIX: "-${CIRRUS_REPO_NAME}-${CIRRUS_BUILD_ID}"`. -Edit `.cirrus.yml`, in the top-level `env` section, update the suffix variable -used at runtime to launch VMs for testing: - - -```yaml -env: - ...cut... - #### - #### Cache-image names to test with (double-quotes around names are critical) - ### - _BUILT_IMAGE_SUFFIX: "libpod-5664838702858240" - FEDORA_CACHE_IMAGE_NAME: "fedora-30-${_BUILT_IMAGE_SUFFIX}" - PRIOR_FEDORA_CACHE_IMAGE_NAME: "fedora-29-${_BUILT_IMAGE_SUFFIX}" - ...cut... -``` - -***NOTES:*** -* If re-using the same PR with new images in `.cirrus.yml`, - take care to also *update the PR description* to remove - the magic ``[CI:IMG]`` string. Keeping it and - `--force` pushing would needlessly cause Cirrus-CI to build - and test images again. -* In the future, if you need to review the log from the build that produced - the referenced image: - - * Note the Build ID from the image name (for example `5664838702858240`). - * Go to that build in the Cirrus-CI WebUI, using the build ID in the URL. - (For example `https://cirrus-ci.com/build/5664838702858240`. - * Choose the *test_build_cache_images* task. - * Open the *build_vm_images* script section. - ### `docs` Task Builds swagger API documentation YAML and uploads to google storage (an online @@ -226,99 +137,6 @@ gsutil cors set /path/to/file.json gs://libpod-master-releases file. Therefore, if it is not functioning or misconfigured, a person must have altered it or changes were made to the referring site (e.g. `docs.podman.io`). -## Base-images - -Base-images are VM disk-images specially prepared for executing as GCE VMs. -In particular, they run services on startup similar in purpose/function -as the standard 'cloud-init' services. - -* The google services are required for full support of ssh-key management - and GCE OAuth capabilities. Google provides native images in GCE - with services pre-installed, for many platforms. For example, - RHEL, CentOS, and Ubuntu. - -* Google does ***not*** provide any images for Fedora (as of 5/2019), nor do - they provide a base-image prepared to run packer for creating other images - in the ``test_build_vm_images`` Task (above). - -* Base images do not need to be produced often, but doing so completely - manually would be time-consuming and error-prone. Therefore a special - semi-automatic *Makefile* target is provided to assist with producing - all the base-images: ``libpod_base_images`` - -To produce new base-images, including an `image-builder-image` (used by -the ``cache_images`` Task) some input parameters are required: - -* ``GCP_PROJECT_ID``: The complete GCP project ID string e.g. foobar-12345 - identifying where the images will be stored. - -* ``GOOGLE_APPLICATION_CREDENTIALS``: A *JSON* file containing - credentials for a GCE service account. This can be [a service - account](https://cloud.google.com/docs/authentication/production#obtaining_and_providing_service_account_credentials_manually) - or [end-user - credentials](https://cloud.google.com/docs/authentication/end-user#creating_your_client_credentials) - -* Optionally, CSV's may be specified to ``PACKER_BUILDS`` - to limit the base-images produced. For example, - ``PACKER_BUILDS=fedora,image-builder-image``. - -If there is no existing 'image-builder-image' within GCE, a new -one may be bootstrapped by creating a CentOS 7 VM with support for -nested-virtualization, and with elevated cloud privileges (to access -GCE, from within the GCE VM). For example: - -``` -$ alias pgcloud='sudo podman run -it --rm -e AS_ID=$UID - -e AS_USER=$USER -v $HOME:$HOME:z quay.io/cevich/gcloud_centos:latest' - -$ URL=https://www.googleapis.com/auth -$ SCOPES=$URL/userinfo.email,$URL/compute,$URL/devstorage.full_control - -# The --min-cpu-platform is critical for nested-virt. -$ pgcloud compute instances create $USER-image-builder \ - --image-family centos-7 \ - --boot-disk-size "200GB" \ - --min-cpu-platform "Intel Haswell" \ - --machine-type n1-standard-2 \ - --scopes $SCOPES -``` - -Then from that VM, execute the -``contrib/cirrus/packer/image-builder-image_base_setup.sh`` script. -Shutdown the VM, and convert it into a new image-builder-image. - -Building new base images is done by first creating a VM from an -image-builder-image and copying the credentials json file to it. - -``` -$ hack/get_ci_vm.sh image-builder-image-1541772081 -...in another terminal... -$ pgcloud compute scp /path/to/gac.json $USER-image-builder-image-1541772081:. -``` - -Then, on the VM, change to the ``packer`` sub-directory, and build the images: - -``` -$ cd libpod/contrib/cirrus/packer -$ make libpod_base_images GCP_PROJECT_ID=<VALUE> \ - GOOGLE_APPLICATION_CREDENTIALS=/path/to/gac.json \ - PACKER_BUILDS=<OPTIONAL> -``` - -Assuming this is successful (hence the semi-automatic part), packer will -produce a ``packer-manifest.json`` output file. This contains the base-image -names suitable for updating in ``.cirrus.yml``, `env` keys ``*_BASE_IMAGE``. - -On failure, it should be possible to determine the problem from the packer -output. Sometimes that means setting `PACKER_LOG=1` and troubleshooting -the nested virt calls. It's also possible to observe the (nested) qemu-kvm -console output. Simply set the ``TTYDEV`` parameter, for example: - -``` -$ make libpod_base_images ... TTYDEV=$(tty) - ... -``` - ## `$SPECIALMODE` Some tasks alter their behavior based on this value. A summary of supported diff --git a/contrib/cirrus/add_second_partition.sh b/contrib/cirrus/add_second_partition.sh index 3c2f9f056..d0407be86 100644 --- a/contrib/cirrus/add_second_partition.sh +++ b/contrib/cirrus/add_second_partition.sh @@ -7,8 +7,7 @@ SLASH_DEVICE="/dev/sda" # Always the case on GCP # The unallocated space results from the difference in disk-size between VM Image -# and runtime request. The check_image.sh test includes a minimum-space check, -# with the Image size set initially lower by contrib/cirrus/packer/libpod_images.yml +# and runtime request. NEW_PART_START="50%" NEW_PART_END="100%" diff --git a/contrib/cirrus/build_vm_images.sh b/contrib/cirrus/build_vm_images.sh deleted file mode 100755 index be1c82185..000000000 --- a/contrib/cirrus/build_vm_images.sh +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env bash - -set -e -source $(dirname $0)/lib.sh - -BASE_IMAGE_VARS='FEDORA_BASE_IMAGE PRIOR_FEDORA_BASE_IMAGE UBUNTU_BASE_IMAGE PRIOR_UBUNTU_BASE_IMAGE' -ENV_VARS="PACKER_BUILDS BUILT_IMAGE_SUFFIX $BASE_IMAGE_VARS SERVICE_ACCOUNT GCE_SSH_USERNAME GCP_PROJECT_ID PACKER_VER SCRIPT_BASE PACKER_BASE CIRRUS_BUILD_ID CIRRUS_CHANGE_IN_REPO" -req_env_var $ENV_VARS -# Must also be made available through make, into packer process -export $ENV_VARS - -# Everything here is running on the 'image-builder-image' GCE image -# Assume basic dependencies are all met, but there could be a newer version -# of the packer binary -PACKER_FILENAME="packer_${PACKER_VER}_linux_amd64.zip" -if [[ -d "$HOME/packer" ]] -then - cd "$HOME/packer" - # image_builder_image has packer pre-installed, check if same version requested - if [[ -r "$PACKER_FILENAME" ]] - then - cp $PACKER_FILENAME "$GOSRC/$PACKER_BASE/" - cp packer "$GOSRC/$PACKER_BASE/" - fi -fi - -cd "$GOSRC/$PACKER_BASE" -# Add/update labels on base-images used in this build to prevent premature deletion -ARGS=" -" -for base_image_var in $BASE_IMAGE_VARS -do - # See entrypoint.sh in contrib/imgts and contrib/imgprune - # These updates can take a while, run them in the background, check later - gcloud compute images update \ - --update-labels=last-used=$(date +%s) \ - --update-labels=build-id=$CIRRUS_BUILD_ID \ - --update-labels=repo-ref=$CIRRUS_CHANGE_IN_REPO \ - --update-labels=project=$GCP_PROJECT_ID \ - ${!base_image_var} & -done - -make libpod_images \ - PACKER_BUILDS=$PACKER_BUILDS \ - PACKER_VER=$PACKER_VER \ - GOSRC=$GOSRC \ - SCRIPT_BASE=$SCRIPT_BASE \ - PACKER_BASE=$PACKER_BASE \ - BUILT_IMAGE_SUFFIX=$BUILT_IMAGE_SUFFIX - -# Separate PR-produced images from those produced on master. -if [[ "${CIRRUS_BRANCH:-}" == "master" ]] -then - POST_MERGE_BUCKET_SUFFIX="-master" -else - POST_MERGE_BUCKET_SUFFIX="" -fi - -# When successful, upload manifest of produced images using a filename unique -# to this build. -URI="gs://packer-import${POST_MERGE_BUCKET_SUFFIX}/manifest${BUILT_IMAGE_SUFFIX}.json" -gsutil cp packer-manifest.json "$URI" - -# Ensure any background 'gcloud compute images update' processes finish -wait # No -n option in CentOS, this is the best that can be done :( - -echo "Finished. A JSON manifest of produced images is available at $URI" diff --git a/contrib/cirrus/check_image.sh b/contrib/cirrus/check_image.sh deleted file mode 100755 index 04867ca64..000000000 --- a/contrib/cirrus/check_image.sh +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env bash - -set -eo pipefail - -source $(dirname $0)/lib.sh - -EVIL_UNITS="$($CIRRUS_WORKING_DIR/$PACKER_BASE/systemd_banish.sh --list)" - -req_env_var PACKER_BUILDER_NAME RCLI EVIL_UNITS OS_RELEASE_ID CG_FS_TYPE - -NFAILS=0 -echo "Validating VM image" - -MIN_SLASH_GIGS=30 -read SLASH_DEVICE SLASH_FSTYPE SLASH_SIZE JUNK <<<$(findmnt --df --first-only --noheadings / | cut -d '.' -f 1) -SLASH_SIZE_GIGS=$(echo "$SLASH_SIZE" | sed -r -e 's/G|g//') -item_test "Minimum available disk space" $SLASH_SIZE_GIGS -gt $MIN_SLASH_GIGS || let "NFAILS+=1" - -MIN_MEM_MB=2000 -read JUNK TOTAL USED MEM_FREE JUNK <<<$(free -tm | tail -1) -item_test 'Minimum available memory' $MEM_FREE -ge $MIN_MEM_MB || let "NFAILS+=1" - -# We're testing a custom-built podman; make sure there isn't a distro-provided -# binary anywhere; that could potentially taint our results. -remove_packaged_podman_files -item_test "remove_packaged_podman_files() does it's job" -z "$(type -P podman)" || let "NFAILS+=1" - -MIN_ZIP_VER='3.0' -VER_RE='.+([[:digit:]]+\.[[:digit:]]+).+' -ACTUAL_VER=$(zip --version 2>&1 | egrep -m 1 "Zip$VER_RE" | sed -r -e "s/$VER_RE/\\1/") -item_test "minimum zip version" "$MIN_ZIP_VER" = $(echo -e "$MIN_ZIP_VER\n$ACTUAL_VER" | sort -V | head -1) || let "NFAILS+=1" - -for REQ_UNIT in google-accounts-daemon.service \ - google-clock-skew-daemon.service \ - google-instance-setup.service \ - google-network-daemon.service \ - google-shutdown-scripts.service \ - google-startup-scripts.service -do - # enabled/disabled appears at the end of the line, on some Ubuntu's it appears twice - service_status=$(systemctl list-unit-files --no-legend $REQ_UNIT | tac -s ' ' | head -1) - item_test "required $REQ_UNIT status is enabled" \ - "$service_status" = "enabled" || let "NFAILS+=1" -done - -for evil_unit in $EVIL_UNITS -do - # Exits zero if any unit matching pattern is running - unit_status=$(systemctl is-active $evil_unit &> /dev/null; echo $?) - item_test "No $evil_unit unit is present or active:" "$unit_status" -ne "0" || let "NFAILS+=1" -done - -echo "Checking items specific to ${PACKER_BUILDER_NAME}${BUILT_IMAGE_SUFFIX}" -case "$PACKER_BUILDER_NAME" in - ubuntu*) - item_test "On ubuntu, no periodic apt crap is enabled" -z "$(egrep $PERIODIC_APT_RE /etc/apt/apt.conf.d/*)" - ;; - fedora*) - # Only runc -OR- crun should be installed, never both - case "$CG_FS_TYPE" in - tmpfs) - HAS=runc - HAS_NOT=crun - ;; - cgroup2fs) - HAS=crun - HAS_NOT=runc - ;; - esac - HAS_RC=$(rpm -qV $HAS &> /dev/null; echo $?) - HAS_NOT_RC=$(rpm -qV $HAS_NOT &> /dev/null; echo $?) - item_test "With a cgroups-fs type $CG_FS_TYPE, the $HAS package is installed" $HAS_RC -eq 0 - item_test "With a cgroups-fs type $CG_FS_TYPE, the $HAS_NOT package is not installed" $HAS_NOT_RC -ne 0 - ;; - xfedora*) - echo "Kernel Command-line: $(cat /proc/cmdline)" - item_test \ - "On ${PACKER_BUILDER_NAME} images, the /sys/fs/cgroup/unified directory does NOT exist" \ - "!" "-d" "/sys/fs/cgroup/unified" || let "NFAILS+=1" - ;; - *) echo "No vm-image specific items to check" -esac - -echo "Total failed tests: $NFAILS" -exit $NFAILS diff --git a/contrib/cirrus/git_authors_to_irc_nicks.csv b/contrib/cirrus/git_authors_to_irc_nicks.csv deleted file mode 100644 index a584cc76a..000000000 --- a/contrib/cirrus/git_authors_to_irc_nicks.csv +++ /dev/null @@ -1,12 +0,0 @@ -# Comma separated mapping of author e-mail, to Freenode IRC nick. -# When no match is found here, the username portion of the e-mail is used. -# Sorting is done at runtime - first-found e-mail match wins. -# Comments (like this) and blank lines are ignored. - -bbaude@redhat.com,baude -matthew.heon@pm.me,mheon -matthew.heon@gmail.com,mheon -emilien@redhat.com,EmilienM -rothberg@redhat.com,vrothberg -santiago@redhat.com,edsantiago -gscrivan@redhat.com,giuseppe diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh index 3292e9d14..f125dd76d 100644 --- a/contrib/cirrus/lib.sh +++ b/contrib/cirrus/lib.sh @@ -35,10 +35,8 @@ export PATH="$HOME/bin:$GOPATH/bin:/usr/local/bin:$PATH" export LD_LIBRARY_PATH="/usr/local/lib${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}" # Saves typing / in case location ever moves SCRIPT_BASE=${SCRIPT_BASE:-./contrib/cirrus} -PACKER_BASE=${PACKER_BASE:-./contrib/cirrus/packer} # Important filepaths SETUP_MARKER_FILEPATH="${SETUP_MARKER_FILEPATH:-/var/tmp/.setup_environment_sh_complete}" -AUTHOR_NICKS_FILEPATH="${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/git_authors_to_irc_nicks.csv" # Downloaded, but not installed packages. PACKAGE_DOWNLOAD_DIR=/var/cache/download @@ -61,22 +59,15 @@ CONTINUOUS_INTEGRATION="${CONTINUOUS_INTEGRATION:-false}" CIRRUS_REPO_NAME=${CIRRUS_REPO_NAME:-libpod} CIRRUS_BASE_SHA=${CIRRUS_BASE_SHA:-unknown$(date +%s)} # difficult to reliably discover CIRRUS_BUILD_ID=${CIRRUS_BUILD_ID:-$RANDOM$(date +%s)} # must be short and unique -# Vars. for image-building -PACKER_VER="1.4.2" -# CSV of cache-image names to build (see $PACKER_BASE/libpod_images.json) - -# List of cache imaes to build for 'CI:IMG' mode via build_vm_images.sh -# Exists to support manual single-image building in case of emergency -export PACKER_BUILDS="${PACKER_BUILDS:-ubuntu-20,ubuntu-19,fedora-32,fedora-31}" -# Google cloud provides these, we just make copies (see $SCRIPT_BASE/README.md) for use -export UBUNTU_BASE_IMAGE="ubuntu-2004-focal-v20200506" -export PRIOR_UBUNTU_BASE_IMAGE="ubuntu-1910-eoan-v20200211" -# Manually produced base-image names (see $SCRIPT_BASE/README.md) -export FEDORA_BASE_IMAGE="fedora-cloud-base-32-1-6-1588257430" -export PRIOR_FEDORA_BASE_IMAGE="fedora-cloud-base-31-1-9-1588257430" -export BUILT_IMAGE_SUFFIX="${BUILT_IMAGE_SUFFIX:--$CIRRUS_REPO_NAME-${CIRRUS_BUILD_ID}}" + +OS_RELEASE_ID="$(source /etc/os-release; echo $ID)" +# GCE image-name compatible string representation of distribution _major_ version +OS_RELEASE_VER="$(source /etc/os-release; echo $VERSION_ID | cut -d '.' -f 1)" +# Combined to ease soe usage +OS_REL_VER="${OS_RELEASE_ID}-${OS_RELEASE_VER}" + # IN_PODMAN container image -IN_PODMAN_IMAGE="quay.io/libpod/in_podman:$DEST_BRANCH" +IN_PODMAN_IMAGE="quay.io/libpod/${OS_RELEASE_ID}_podman:$_BUILT_IMAGE_SUFFIX" # Image for uploading releases UPLDREL_IMAGE="quay.io/libpod/upldrel:master" @@ -98,7 +89,7 @@ BIGTO="timeout_attempt_delay_command 300s 5 60s" # Safe env. vars. to transfer from root -> $ROOTLESS_USER (go env handled separately) ROOTLESS_ENV_RE='(CIRRUS_.+)|(ROOTLESS_.+)|(.+_IMAGE.*)|(.+_BASE)|(.*DIRPATH)|(.*FILEPATH)|(SOURCE.*)|(DEPEND.*)|(.+_DEPS_.+)|(OS_REL.*)|(.+_ENV_RE)|(TRAVIS)|(CI.+)|(REMOTE.*)' # Unsafe env. vars for display -SECRET_ENV_RE='(IRCID)|(ACCOUNT)|(GC[EP]..+)|(SSH)' +SECRET_ENV_RE='(ACCOUNT)|(GC[EP]..+)|(SSH)' SPECIALMODE="${SPECIALMODE:-none}" RCLI="${RCLI:-false}" @@ -111,22 +102,9 @@ then else ROOTLESS_USER="${ROOTLESS_USER:-$USER}" fi - -# GCE image-name compatible string representation of distribution name -OS_RELEASE_ID="$(source /etc/os-release; echo $ID)" -# GCE image-name compatible string representation of distribution _major_ version -OS_RELEASE_VER="$(source /etc/os-release; echo $VERSION_ID | cut -d '.' -f 1)" -# Combined to ease soe usage -OS_REL_VER="${OS_RELEASE_ID}-${OS_RELEASE_VER}" # Type of filesystem used for cgroups CG_FS_TYPE="$(stat -f -c %T /sys/fs/cgroup)" -# When building images, the version of automation tooling to install -INSTALL_AUTOMATION_VERSION=1.1.3 - -# Installed into cache-images, supports overrides -# by user-data in case of breakage or for debugging. -CUSTOM_CLOUD_CONFIG_DEFAULTS="$GOSRC/$PACKER_BASE/cloud-init/$OS_RELEASE_ID/cloud.cfg.d" # Pass in a list of one or more envariable names; exit non-zero with # helpful error message if any value is empty req_env_var() { @@ -237,67 +215,6 @@ timeout_attempt_delay_command() { fi } -ircmsg() { - req_env_var CIRRUS_TASK_ID IRCID - [[ -n "$*" ]] || die 9 "ircmsg() invoked without message text argument" - # Sometimes setup_environment.sh didn't run - SCRIPT="$(dirname $0)/podbot.py" - NICK="podbot_$CIRRUS_TASK_ID" - NICK="${NICK:0:15}" # Any longer will break things - set +e - $SCRIPT $NICK $@ - echo "Ignoring exit($?)" - set -e -} - -# This covers all possible human & CI workflow parallel & serial combinations -# where at least one caller must definitively discover if within a commit range -# there is at least one release tag not having any '-' characters (return 0) -# or otherwise (return non-0). -is_release() { - unset RELVER - local ret - req_env_var CIRRUS_CHANGE_IN_REPO - if [[ -n "$CIRRUS_TAG" ]]; then - RELVER="$CIRRUS_TAG" - elif [[ ! "$CIRRUS_BASE_SHA" =~ "unknown" ]] - then - # Normally not possible for this to be empty, except when unittesting. - req_env_var CIRRUS_BASE_SHA - local range="${CIRRUS_BASE_SHA}..${CIRRUS_CHANGE_IN_REPO}" - if echo "${range}$CIRRUS_TAG" | grep -iq 'unknown'; then - die 11 "is_release() unusable range ${range} or tag $CIRRUS_TAG" - fi - - if type -P git &> /dev/null - then - git fetch --all --tags &> /dev/null|| \ - die 12 "is_release() failed to fetch tags" - RELVER=$(git log --pretty='format:%d' $range | \ - grep '(tag:' | sed -r -e 's/\s+[(]tag:\s+(v[0-9].*)[)]/\1/' | \ - sort -uV | tail -1) - ret=$? - else - warn -1 "Git command not found while checking for release" - ret="-1" - fi - [[ "$ret" -eq "0" ]] || \ - die 13 "is_release() failed to parse tags" - else # Not testing a PR, but neither CIRRUS_BASE_SHA or CIRRUS_TAG are set - return 1 - fi - if [[ -n "$RELVER" ]]; then - echo "Found \$RELVER $RELVER" - if echo "$RELVER" | grep -q '-'; then - return 2 # development tag - else - return 0 - fi - else - return 1 # not a release - fi -} - setup_rootless() { req_env_var ROOTLESS_USER GOPATH GOSRC SECRET_ENV_RE ROOTLESS_ENV_RE @@ -369,20 +286,6 @@ setup_rootless() { die 11 "Timeout exceeded waiting for localhost ssh capability" } -# Grab a newer version of git from software collections -# https://www.softwarecollections.org/en/ -# and use it with a wrapper -install_scl_git() { - echo "Installing SoftwareCollections updated 'git' version." - ooe.sh $SUDO yum -y install rh-git29 - cat << "EOF" | $SUDO tee /usr/bin/git -#!/usr/bin/env bash - -scl enable rh-git29 -- git $@ -EOF - $SUDO chmod 755 /usr/bin/git -} - install_test_configs() { echo "Installing cni config, policy and registry config" req_env_var GOSRC SCRIPT_BASE @@ -457,66 +360,3 @@ $FEDORA_BASE_IMAGE $PRIOR_FEDORA_BASE_IMAGE " } - -systemd_banish() { - $GOSRC/$PACKER_BASE/systemd_banish.sh -} - -# This can be removed when the kernel bug fix is included in Fedora -workaround_bfq_bug() { - if [[ "$OS_RELEASE_ID" == "fedora" ]] && [[ $OS_RELEASE_VER -le 32 ]]; then - warn "Switching io scheduler to 'deadline' to avoid RHBZ 1767539" - warn "aka https://bugzilla.kernel.org/show_bug.cgi?id=205447" - echo "mq-deadline" | sudo tee /sys/block/sda/queue/scheduler > /dev/null - echo -n "IO Scheduler set to: " - $SUDO cat /sys/block/sda/queue/scheduler - fi -} - -# Warning: DO NOT USE. -# This is called by other functions as the very last step during the VM Image build -# process. It's purpose is to "reset" the image, so all the first-boot operations -# happen at test runtime (like generating new ssh host keys, resizing partitions, etc.) -_finalize() { - set +e # Don't fail at the very end - if [[ -d "$CUSTOM_CLOUD_CONFIG_DEFAULTS" ]] - then - echo "Installing custom cloud-init defaults" - $SUDO cp -v "$CUSTOM_CLOUD_CONFIG_DEFAULTS"/* /etc/cloud/cloud.cfg.d/ - else - echo "Could not find any files in $CUSTOM_CLOUD_CONFIG_DEFAULTS" - fi - echo "Re-initializing so next boot does 'first-boot' setup again." - cd / - $SUDO rm -rf $GOPATH/src # Actual source will be cloned at runtime - $SUDO rm -rf /var/lib/cloud/instanc* - $SUDO rm -rf /root/.ssh/* - $SUDO rm -rf /etc/ssh/*key* - $SUDO rm -rf /etc/ssh/moduli - $SUDO rm -rf /home/* - $SUDO rm -rf /tmp/* - $SUDO rm -rf /tmp/.??* - $SUDO sync - $SUDO fstrim -av -} - -# Called during VM Image setup, not intended for general use. -rh_finalize() { - set +e # Don't fail at the very end - echo "Resetting to fresh-state for usage as cloud-image." - PKG=$(type -P dnf || type -P yum || echo "") - $SUDO $PKG clean all - $SUDO rm -rf /var/cache/{yum,dnf} - $SUDO rm -f /etc/udev/rules.d/*-persistent-*.rules - $SUDO touch /.unconfigured # force firstboot to run - _finalize -} - -# Called during VM Image setup, not intended for general use. -ubuntu_finalize() { - set +e # Don't fail at the very end - echo "Resetting to fresh-state for usage as cloud-image." - $LILTO $SUDOAPTGET autoremove - $SUDO rm -rf /var/cache/apt - _finalize -} diff --git a/contrib/cirrus/lib.sh.t b/contrib/cirrus/lib.sh.t index 204af1245..643b5513d 100755 --- a/contrib/cirrus/lib.sh.t +++ b/contrib/cirrus/lib.sh.t @@ -84,7 +84,7 @@ BAR=1 test_rev "FOO BAR" 0 '' ############################################################################### -# tests for test_okay() +# tests for item_test() function test_item_test { local exp_msg=$1 @@ -118,46 +118,4 @@ test_item_test "ok okay enough" 0 "okay enough" "line 1 line2" "=" "line 1 line2" -############################################################################### -# tests for is_release() - -# N/B: Assuming tests run in their own process, so wiping out the local -# CIRRUS_BASE_SHA CIRRUS_CHANGE_IN_REPO and CIRRUS_TAG will be okay. -function test_is_release() { - CIRRUS_BASE_SHA="$1" - CIRRUS_CHANGE_IN_REPO="$2" - CIRRUS_TAG="$3" - local exp_status=$4 - local exp_msg=$5 - local msg - msg=$(is_release) - local status=$? - - check_result "$msg" "$exp_msg" "is_release(CIRRUS_BASE_SHA='$1' CIRRUS_CHANGE_IN_REPO='$2' CIRRUS_TAG='$3')" - check_result "$status" "$exp_status" "is_release(...) returned $status" -} - -# FROM TO TAG RET MSG -test_is_release "" "" "" "9" "FATAL: is_release() requires \$CIRRUS_CHANGE_IN_REPO to be non-empty" -test_is_release "x" "" "" "9" "FATAL: is_release() requires \$CIRRUS_CHANGE_IN_REPO to be non-empty" - -# post-merge / tag-push testing, FROM will be set 'unknown' by (lib.sh default) -test_is_release "unknown" "x" "" "1" "" -# post-merge / tag-push testing, oddball tag is set, FROM will be set 'unknown' -test_is_release "unknown" "unknown" "test-tag" "2" "Found \$RELVER test-tag" -# post-merge / tag-push testing, sane tag is set, FROM will be set 'unknown' -test_is_release "unknown" "unknown" "0.0.0" "0" "Found \$RELVER 0.0.0" -# hack/get_ci_vm or PR testing, FROM and TO are set, no tag is set -test_is_release "x" "x" "" "1" "" - -# Negative-testing git with this function is very difficult, assume git works -# test_is_release ... "is_release() failed to fetch tags" -# test_is_release ... "is_release() failed to parse tags" - -BF_V1=$(git rev-parse v1.0.0^) -AT_V1=$(git rev-parse v1.0.0) -test_is_release "$BF_V1" "$BF_V1" "v9.8.7-dev" "2" "Found \$RELVER v9.8.7-dev" -test_is_release "$BF_V1" "$AT_V1" "v9.8.7-dev" "2" "Found \$RELVER v9.8.7-dev" -test_is_release "$BF_V1" "$AT_V1" "" "0" "Found \$RELVER v1.0.0" - exit $rc diff --git a/contrib/cirrus/notice_branch_failure.sh b/contrib/cirrus/notice_branch_failure.sh deleted file mode 100755 index b810bd266..000000000 --- a/contrib/cirrus/notice_branch_failure.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash - -set -e - -source $(dirname $0)/lib.sh - -# mIRC "escape" codes are the most standard, for a non-standard client-side interpretation. -ETX="$(echo -n -e '\x03')" -RED="${ETX}4" -NOR="$(echo -n -e '\x0f')" - -if [[ "$CIRRUS_BRANCH" = "$DEST_BRANCH" ]] -then - BURL="https://cirrus-ci.com/build/$CIRRUS_BUILD_ID" - ircmsg "${RED}[Action Recommended]: ${NOR}Post-merge testing on ${RED}$CIRRUS_BRANCH failed${NOR} in $CIRRUS_TASK_NAME on ${OS_RELEASE_ID}-${OS_RELEASE_VER}: $BURL. Please investigate, and re-run if appropriate." -fi - -# This script assumed to be executed on failure -die 1 "Testing Failed" diff --git a/contrib/cirrus/packer/.gitignore b/contrib/cirrus/packer/.gitignore deleted file mode 100644 index 8f7bdeaf7..000000000 --- a/contrib/cirrus/packer/.gitignore +++ /dev/null @@ -1,7 +0,0 @@ -*json -packer -packer*zip -packer_cache -cidata* -meta-data -user-data diff --git a/contrib/cirrus/packer/Makefile b/contrib/cirrus/packer/Makefile deleted file mode 100644 index c5a8e4cac..000000000 --- a/contrib/cirrus/packer/Makefile +++ /dev/null @@ -1,94 +0,0 @@ -PACKER_VER ?= 1.4.3 -GOARCH=$(shell go env GOARCH) -ARCH=$(uname -m) -PACKER_DIST_FILENAME := packer_${PACKER_VER}_linux_${GOARCH}.zip - -# Only needed for libpod_base_images target -TIMESTAMP := $(shell date +%s) -GOPATH ?= /var/tmp/go -GOSRC ?= $(GOPATH)/src/github.com/containers/libpod -PACKER_BASE ?= contrib/cirrus/packer -SCRIPT_BASE ?= contrib/cirrus -POST_MERGE_BUCKET_SUFFIX ?= - -UBUNTU_BASE_IMAGE = $(shell source ../lib.sh && echo "$$UBUNTU_BASE_IMAGE") -PRIOR_UBUNTU_BASE_IMAGE = $(shell source ../lib.sh && echo "$$PRIOR_UBUNTU_BASE_IMAGE") - -# For debugging nested-virt, use -#TTYDEV := $(shell tty) -TTYDEV := /dev/null - -.PHONY: all -all: libpod_images - -# Utility target for checking required parameters -.PHONY: guard-% -guard-%: - @if [[ -z "$($*)" ]]; then \ - echo "Missing or empty required make variable '$*'."; \ - exit 1; \ - fi; - -%.json: %.yml - @python3 -c 'import json,yaml; json.dump( yaml.safe_load(open("$<").read()), open("$@","w"), indent=2);' - -${PACKER_DIST_FILENAME}: - @curl -L --silent --show-error \ - -O https://releases.hashicorp.com/packer/${PACKER_VER}/${PACKER_DIST_FILENAME} - -packer: ${PACKER_DIST_FILENAME} - @curl -L --silent --show-error \ - https://releases.hashicorp.com/packer/${PACKER_VER}/packer_${PACKER_VER}_SHA256SUMS \ - | grep linux_${GOARCH} > /tmp/packer_sha256sums - @sha256sum --check /tmp/packer_sha256sums - @unzip -o ${PACKER_DIST_FILENAME} - @touch --reference=Makefile ${PACKER_DIST_FILENAME} - -.PHONY: test -test: libpod_base_images.json libpod_images.json packer - ./packer inspect libpod_base_images.json > /dev/null - ./packer inspect libpod_images.json > /dev/null - @echo "All good" - -.PHONY: libpod_images -libpod_images: guard-PACKER_BUILDS libpod_images.json packer - ./packer build \ - -force \ - $(shell test -z "${PACKER_BUILDS}" || echo "-only=${PACKER_BUILDS}") \ - -var GOPATH=$(GOPATH) \ - -var GOSRC=$(GOSRC) \ - -var PACKER_BASE=$(PACKER_BASE) \ - -var SCRIPT_BASE=$(SCRIPT_BASE) \ - libpod_images.json - -cidata.ssh: - ssh-keygen -f $@ -P "" -q - -cidata.ssh.pub: cidata.ssh - touch $@ - -meta-data: - echo "local-hostname: localhost.localdomain" > $@ - -user-data: cidata.ssh.pub - bash make-user-data.sh - -cidata.iso: user-data meta-data - genisoimage -output cidata.iso -volid cidata -input-charset utf-8 -joliet -rock user-data meta-data - -# This is intended to be run by a human, with admin access to the libpod GCE project. -.PHONY: libpod_base_images -libpod_base_images: guard-GCP_PROJECT_ID guard-GOOGLE_APPLICATION_CREDENTIALS libpod_base_images.json cidata.iso cidata.ssh packer - PACKER_CACHE_DIR=/tmp ./packer build \ - $(shell test -z "${PACKER_BUILDS}" || echo "-only=${PACKER_BUILDS}") \ - -force \ - -var TIMESTAMP=$(TIMESTAMP) \ - -var TTYDEV=$(TTYDEV) \ - -var GCP_PROJECT_ID=$(GCP_PROJECT_ID) \ - -var GOOGLE_APPLICATION_CREDENTIALS=$(GOOGLE_APPLICATION_CREDENTIALS) \ - -var GOSRC=$(GOSRC) \ - -var PACKER_BASE=$(PACKER_BASE) \ - -var SCRIPT_BASE=$(SCRIPT_BASE) \ - -var UBUNTU_BASE_IMAGE=$(UBUNTU_BASE_IMAGE) \ - -var PRIOR_UBUNTU_BASE_IMAGE=$(PRIOR_UBUNTU_BASE_IMAGE) \ - libpod_base_images.json diff --git a/contrib/cirrus/packer/README.how-to-update-cirrus-vms b/contrib/cirrus/packer/README.how-to-update-cirrus-vms deleted file mode 100644 index ac2902ffb..000000000 --- a/contrib/cirrus/packer/README.how-to-update-cirrus-vms +++ /dev/null @@ -1,89 +0,0 @@ -This document briefly describes how to update VMs on Cirrus. - -Examples of when you need to do this: - - - to update crun, conmon, or some other package(s) - - to add and/or remove an OS (eg drop f31, add f33) - - to change system config (eg containers.conf or other /etc files) - - to change kernel command-line (boot time) options - -This is a TWO-STEP process: you need to submit a PR with a magic [CI:IMG] -description string, wait for it to finish, grab a magic string from the -results, then resubmit without [CI:IMG]. - -Procedure, Part One of Two: - - 1) Create a working branch: - - $ git co -b my_branch_name - - 2) Make your changes. Typically, zero or more of the following files: - - .cirrus.yml - contrib/cirrus/packer/*_packaging.sh - - I said zero because sometimes you just want to update VMs - with the latest in dnf or ubuntu repos. That doesn't require - changing anything here, simply running new dnf/apt installs. - - 3) Commit your changes. Be sure to include the magic [CI:IMG] string: - - $ git commit -asm'[CI:IMG] this is my commit message' - - 4) Submit your PR: - - $ gh pr create --fill --web - - - -------------------------- INTERMISSION -------------------------- - ...in which we wait for CI to turn green. In particular, although - we only really need 'test_build_cache_images' (45 minutes or so) - to get the required magic number strings, please be a decent - human being and wait for 'verify_test_built_images' (another hour) - so we can all have confidence in our process. Thank you. - -------------------------- INTERMISSION -------------------------- - - -Procedure, Part Two of Two: - - 1) When 'test_build_cache_images' completes, click it, then click - 'View more details on Cirrus CI', then expand the 'Run build_vm_image' - accordion. This gives you a garishly colorful display of lines. - Each color is a different VM. - - 2) Verify that each VM has the packages you require. (The garish log - doesn't actually list this for all packages, so you may need to - look in the 'verify_test_built_images' log for each individual - VM. Click the 'package_versions' accordion.) - - 3) At the bottom of this log you will see a block like: - - Builds finished. The artifacts of successful builds are: - ubuntu-19: A disk image was created: ubuntu-19-podman-6439450735542272 - fedora-31: A disk image was created: fedora-31-podman-6439450735542272 - ..... - - The long numbers at the end should (MUST!) be all identical. - - 4) Edit .cirrus.yml locally. Find '_BUILT_IMAGE_SUFFIX' near the - top. Copy that long number ("6439450735542272", above) and paste - it here, replacing the previous long number. - - 5) Wait for CI to turn green. I know you might have skipped that, - because 'test_build_cache_images' finishes long before 'verify', - and maybe you're in a hurry, but come on. Be responsible. - - 6) Edit the PR description in github: remove '[CI:IMG]' from the - title. Again, *in github*, in the web UI, use the 'Edit' button - at top right next to the PR title. Remove the '[CI:IMG]' string - from the PR title, press Save. If you forget to do this, the - VM-building steps will run again (taking a long time) but it - will be a waste of time. - - 7) Update your PR: - - $ git add .cirrus.yml (to get the new magic IMAGE_SUFFIX string) - $ git commit --amend (remove [CI:IMG] for consistency with 6) - $ git push --force - -You can probably take it from here. diff --git a/contrib/cirrus/packer/README.md b/contrib/cirrus/packer/README.md deleted file mode 100644 index 9a07ed960..000000000 --- a/contrib/cirrus/packer/README.md +++ /dev/null @@ -1,3 +0,0 @@ -These are definitions and scripts consumed by packer to produce the -various distribution images used for CI testing. For more details -see the [Cirrus CI documentation](../README.md) diff --git a/contrib/cirrus/packer/cloud-init/fedora/cloud-init.service b/contrib/cirrus/packer/cloud-init/fedora/cloud-init.service deleted file mode 100644 index 4d2197d87..000000000 --- a/contrib/cirrus/packer/cloud-init/fedora/cloud-init.service +++ /dev/null @@ -1,20 +0,0 @@ -[Unit] -Description=Initial cloud-init job (metadata service crawler) -DefaultDependencies=no -Wants=cloud-init-local.service -After=cloud-init-local.service -Wants=google-network-daemon.service -After=google-network-daemon.service -Before=systemd-user-sessions.service - -[Service] -Type=oneshot -ExecStart=/usr/bin/cloud-init init -RemainAfterExit=yes -TimeoutSec=0 - -# Output needs to appear in instance console output -StandardOutput=journal+console - -[Install] -WantedBy=cloud-init.target diff --git a/contrib/cirrus/packer/cloud-init/fedora/cloud.cfg.d/40_enable_root.cfg b/contrib/cirrus/packer/cloud-init/fedora/cloud.cfg.d/40_enable_root.cfg deleted file mode 100644 index 672d1907b..000000000 --- a/contrib/cirrus/packer/cloud-init/fedora/cloud.cfg.d/40_enable_root.cfg +++ /dev/null @@ -1 +0,0 @@ -disable_root: 0 diff --git a/contrib/cirrus/packer/cloud-init/fedora/cloud.cfg.d/50_custom_disk_setup.cfg b/contrib/cirrus/packer/cloud-init/fedora/cloud.cfg.d/50_custom_disk_setup.cfg deleted file mode 100644 index c0fdf0e23..000000000 --- a/contrib/cirrus/packer/cloud-init/fedora/cloud.cfg.d/50_custom_disk_setup.cfg +++ /dev/null @@ -1,4 +0,0 @@ -#cloud-config -growpart: - mode: false -resize_rootfs: false diff --git a/contrib/cirrus/packer/cloud-init/ubuntu/cloud.cfg.d/40_enable_root.cfg b/contrib/cirrus/packer/cloud-init/ubuntu/cloud.cfg.d/40_enable_root.cfg deleted file mode 100644 index 672d1907b..000000000 --- a/contrib/cirrus/packer/cloud-init/ubuntu/cloud.cfg.d/40_enable_root.cfg +++ /dev/null @@ -1 +0,0 @@ -disable_root: 0 diff --git a/contrib/cirrus/packer/fedora_base-setup.sh b/contrib/cirrus/packer/fedora_base-setup.sh deleted file mode 100644 index bf29a1aec..000000000 --- a/contrib/cirrus/packer/fedora_base-setup.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env bash - -# N/B: This script is not intended to be run by humans. It is used to configure the -# fedora base image for importing, so that it will boot in GCE - -set -e - -# Load in library (copied by packer, before this script was run) -source $GOSRC/$SCRIPT_BASE/lib.sh - -echo "Updating packages" -dnf -y update - -echo "Installing necessary packages and google services" -dnf -y install rng-tools google-compute-engine-tools google-compute-engine-oslogin ethtool - -echo "Enabling services" -systemctl enable rngd - -# There is a race that can happen on boot between the GCE services configuring -# the VM, and cloud-init trying to do similar activities. Use a customized -# unit file to make sure cloud-init starts after the google-compute-* services. -echo "Setting cloud-init service to start after google-network-daemon.service" -cp -v $GOSRC/$PACKER_BASE/cloud-init/fedora/cloud-init.service /etc/systemd/system/ - -# ref: https://cloud.google.com/compute/docs/startupscript -# The mechanism used by Cirrus-CI to execute tasks on the system is through an -# "agent" process launched as a GCP startup-script (from the metadata service). -# This agent is responsible for cloning the repository and executing all task -# scripts and other operations. Therefor, on SELinux-enforcing systems, the -# service must be labeled properly to ensure it's child processes can -# run with the proper contexts. -METADATA_SERVICE_CTX=unconfined_u:unconfined_r:unconfined_t:s0 -METADATA_SERVICE_PATH=systemd/system/google-startup-scripts.service -sed -r -e \ - "s/Type=oneshot/Type=oneshot\nSELinuxContext=$METADATA_SERVICE_CTX/" \ - /lib/$METADATA_SERVICE_PATH > /etc/$METADATA_SERVICE_PATH - -# Ensure there are no disruptive periodic services enabled by default in image -systemd_banish - -rh_finalize - -echo "SUCCESS!" diff --git a/contrib/cirrus/packer/fedora_packaging.sh b/contrib/cirrus/packer/fedora_packaging.sh deleted file mode 100644 index fcf9eb93f..000000000 --- a/contrib/cirrus/packer/fedora_packaging.sh +++ /dev/null @@ -1,194 +0,0 @@ -#!/usr/bin/env bash - -# This script is called from fedora_setup.sh and various Dockerfiles. -# It's not intended to be used outside of those contexts. It assumes the lib.sh -# library has already been sourced, and that all "ground-up" package-related activity -# needs to be done, including repository setup and initial update. - -set -e - -echo "Updating/Installing repos and packages for $OS_REL_VER" - -source $GOSRC/$SCRIPT_BASE/lib.sh - -req_env_var GOSRC SCRIPT_BASE BIGTO INSTALL_AUTOMATION_VERSION FEDORA_BASE_IMAGE PRIOR_FEDORA_BASE_IMAGE - -# Pre-req. to install automation tooing -$LILTO $SUDO dnf install -y git - -# Install common automation tooling (i.e. ooe.sh) -curl --silent --show-error --location \ - --url "https://raw.githubusercontent.com/containers/automation/master/bin/install_automation.sh" | \ - $SUDO env INSTALL_PREFIX=/usr/share /bin/bash -s - "$INSTALL_AUTOMATION_VERSION" -# Reload installed environment right now (happens automatically in a new process) -source /usr/share/automation/environment - -# Set this to 1 to NOT enable updates-testing repository -DISABLE_UPDATES_TESTING=${DISABLE_UPDATES_TESTING:0} - -# Do not enable updates-testing on the previous Fedora release -if ((DISABLE_UPDATES_TESTING!=0)); then - warn "Enabling updates-testing repository for image based on $FEDORA_BASE_IMAGE" - $LILTO $SUDO ooe.sh dnf install -y 'dnf-command(config-manager)' - $LILTO $SUDO ooe.sh dnf config-manager --set-enabled updates-testing -else - warn "NOT enabling updates-testing repository for image based on $PRIOR_FEDORA_BASE_IMAGE" -fi - -$BIGTO ooe.sh $SUDO dnf update -y - -# Fedora, as of 31, uses cgroups v2 by default. runc does not support -# cgroups v2, only crun does. (As of 2020-07-30 runc support is -# forthcoming but not even close to ready yet). To ensure a reliable -# runtime environment, force-remove runc if it is present. -# However, because a few other repos. which use these images still need -# it, ensure the runc package is cached in $PACKAGE_DOWNLOAD_DIR so -# it may be swap it in when required. -REMOVE_PACKAGES=(runc) - -INSTALL_PACKAGES=(\ - autoconf - automake - bash-completion - bats - bridge-utils - btrfs-progs-devel - buildah - bzip2 - conmon - container-selinux - containernetworking-plugins - containers-common - criu - crun - curl - device-mapper-devel - dnsmasq - e2fsprogs-devel - emacs-nox - file - findutils - fuse3 - fuse3-devel - gcc - git - glib2-devel - glibc-devel - glibc-static - gnupg - go-md2man - golang - gpgme - gpgme-devel - grubby - hostname - httpd-tools - iproute - iptables - jq - krb5-workstation - libassuan - libassuan-devel - libblkid-devel - libcap-devel - libffi-devel - libgpg-error-devel - libguestfs-tools - libmsi1 - libnet - libnet-devel - libnl3-devel - libseccomp - libseccomp-devel - libselinux-devel - libtool - libvarlink-util - libxml2-devel - libxslt-devel - lsof - make - mlocate - msitools - nfs-utils - nmap-ncat - openssl - openssl-devel - ostree-devel - pandoc - pkgconfig - podman - policycoreutils - procps-ng - protobuf - protobuf-c - protobuf-c-devel - protobuf-devel - python2 - python3-PyYAML - python3-dateutil - python3-libselinux - python3-libsemanage - python3-libvirt - python3-psutil - python3-pytoml - python3-requests - redhat-rpm-config - rpcbind - rsync - sed - selinux-policy-devel - skopeo - skopeo-containers - slirp4netns - socat - tar - unzip - vim - wget - which - xz - zip - zlib-devel -) -DOWNLOAD_PACKAGES=(\ - "cri-o-$(get_kubernetes_version)*" - cri-tools - "kubernetes-$(get_kubernetes_version)*" - runc - oci-umount - parallel -) - -echo "Installing general build/test dependencies for Fedora '$OS_RELEASE_VER'" -$BIGTO ooe.sh $SUDO dnf install -y ${INSTALL_PACKAGES[@]} - -# AD-HOC CODE FOR SPECIAL-CASE SITUATIONS! -# On 2020-07-23 we needed this code to upgrade crun on f31, a build -# that is not yet in stable. Since CI:IMG PRs are a two-step process, -# the key part is that we UN-COMMENT-THIS-OUT during the first step, -# then re-comment it on the second (once we have the built images). -# That way this will be dead code in future CI:IMG PRs but will -# serve as an example for anyone in a similar future situation. -# $BIGTO ooe.sh $SUDO dnf --enablerepo=updates-testing -y upgrade crun - -[[ ${#REMOVE_PACKAGES[@]} -eq 0 ]] || \ - $LILTO ooe.sh $SUDO dnf erase -y "${REMOVE_PACKAGES[@]}" - -if [[ ${#DOWNLOAD_PACKAGES[@]} -gt 0 ]]; then - echo "Downloading packages for optional installation at runtime, as needed." - # Required for cri-o - ooe.sh $SUDO dnf -y module enable cri-o:$(get_kubernetes_version) - $SUDO mkdir -p "$PACKAGE_DOWNLOAD_DIR" - cd "$PACKAGE_DOWNLOAD_DIR" - $LILTO ooe.sh $SUDO dnf download -y --resolve "${DOWNLOAD_PACKAGES[@]}" -fi - -echo "Installing runtime tooling" -# Save some runtime by having these already available -cd $GOSRC -# Required since initially go was not installed -source $GOSRC/$SCRIPT_BASE/lib.sh -echo "Go environment has been setup:" -go env -$SUDO make install.tools -$SUDO $GOSRC/hack/install_catatonit.sh diff --git a/contrib/cirrus/packer/fedora_setup.sh b/contrib/cirrus/packer/fedora_setup.sh deleted file mode 100644 index 16ae87d8a..000000000 --- a/contrib/cirrus/packer/fedora_setup.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash - -# This script is called by packer on the subject fedora VM, to setup the podman -# build/test environment. It's not intended to be used outside of this context. - -set -e - -# Load in library (copied by packer, before this script was run) -source $GOSRC/$SCRIPT_BASE/lib.sh - -req_env_var SCRIPT_BASE PACKER_BASE INSTALL_AUTOMATION_VERSION PACKER_BUILDER_NAME GOSRC FEDORA_BASE_IMAGE OS_RELEASE_ID OS_RELEASE_VER - -workaround_bfq_bug - -# Do not enable updates-testing on the previous Fedora release -if [[ "$PRIOR_FEDORA_BASE_IMAGE" =~ "${OS_RELEASE_ID}-cloud-base-${OS_RELEASE_VER}" ]]; then - DISABLE_UPDATES_TESTING=1 -else - DISABLE_UPDATES_TESTING=0 -fi - -bash $PACKER_BASE/fedora_packaging.sh -# Load installed environment right now (happens automatically in a new process) -source /usr/share/automation/environment - -echo "Enabling cgroup management from containers" -ooe.sh sudo setsebool container_manage_cgroup true - -# Ensure there are no disruptive periodic services enabled by default in image -systemd_banish - -rh_finalize - -echo "SUCCESS!" diff --git a/contrib/cirrus/packer/image-builder-image_base-setup.sh b/contrib/cirrus/packer/image-builder-image_base-setup.sh deleted file mode 100644 index 26fbe2903..000000000 --- a/contrib/cirrus/packer/image-builder-image_base-setup.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env bash - -# This script is called by packer on a vanilla CentOS VM, to setup the image -# used for building images FROM base images. It's not intended to be used -# outside of this context. - -set -e - -[[ "$1" == "post" ]] || exit 0 # pre stage not needed - -# Load in library (copied by packer, before this script was run) -source $GOSRC/$SCRIPT_BASE/lib.sh - -req_env_var TIMESTAMP GOSRC SCRIPT_BASE PACKER_BASE - -install_ooe - -echo "Updating packages" -ooe.sh sudo yum -y update - -echo "Configuring repositories" -ooe.sh sudo yum -y install centos-release-scl epel-release - -echo "Installing packages" -ooe.sh sudo yum -y install \ - genisoimage \ - golang \ - google-cloud-sdk \ - libvirt \ - libvirt-admin \ - libvirt-client \ - libvirt-daemon \ - make \ - python36 \ - python36-PyYAML \ - qemu-img \ - qemu-kvm \ - qemu-kvm-tools \ - qemu-user \ - rsync \ - rng-tools \ - unzip \ - util-linux \ - vim - -sudo systemctl enable rngd - -sudo ln -s /usr/libexec/qemu-kvm /usr/bin/ - -sudo tee /etc/modprobe.d/kvm-nested.conf <<EOF -options kvm-intel nested=1 -options kvm-intel enable_shadow_vmcs=1 -options kvm-intel enable_apicv=1 -options kvm-intel ept=1 -EOF - -echo "Installing packer" -sudo mkdir -p /root/$(basename $PACKER_BASE) -sudo cp $GOSRC/$PACKER_BASE/*packer* /root/$(basename $PACKER_BASE) -sudo mkdir -p /root/$(basename $SCRIPT_BASE) -sudo cp $GOSRC/$SCRIPT_BASE/*.sh /root/$(basename $SCRIPT_BASE) - -install_scl_git - -echo "Cleaning up" -cd / -rm -rf $GOSRC - -rh_finalize - -echo "SUCCESS!" diff --git a/contrib/cirrus/packer/libpod_base_images.yml b/contrib/cirrus/packer/libpod_base_images.yml deleted file mode 100644 index f53bfafc5..000000000 --- a/contrib/cirrus/packer/libpod_base_images.yml +++ /dev/null @@ -1,164 +0,0 @@ ---- - -variables: - # Complete local path to this repository (Required) - GOSRC: - # Relative path to this (packer) subdirectory (Required) - PACKER_BASE: - # Relative path to cirrus scripts subdirectory (Required) - SCRIPT_BASE: - # Unique ID for naming new base-images (required) - TIMESTAMP: - # Required for output from qemu builders - TTYDEV: - - # Ubuntu releases are merely copied to this project for control purposes - UBUNTU_BASE_IMAGE: - PRIOR_UBUNTU_BASE_IMAGE: - - # Latest Fedora release - FEDORA_IMAGE_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/32/Cloud/x86_64/images/Fedora-Cloud-Base-32-1.6.x86_64.qcow2" - FEDORA_CSUM_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/32/Cloud/x86_64/images/Fedora-Cloud-32-1.6-x86_64-CHECKSUM" - FEDORA_BASE_IMAGE_NAME: 'fedora-cloud-base-32-1-6' - - # Prior Fedora release - PRIOR_FEDORA_IMAGE_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/31/Cloud/x86_64/images/Fedora-Cloud-Base-31-1.9.x86_64.qcow2" - PRIOR_FEDORA_CSUM_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/31/Cloud/x86_64/images/Fedora-Cloud-31-1.9-x86_64-CHECKSUM" - PRIOR_FEDORA_BASE_IMAGE_NAME: 'fedora-cloud-base-31-1-9' - - # The name of the image in GCE used for packer build libpod_images.yml - IBI_BASE_NAME: 'image-builder-image' - CIDATA_ISO: 'cidata.iso' # produced by Makefile - - # Path to json file (required, likely ~/.config/gcloud/legacy_credentials/*/adc.json) - GOOGLE_APPLICATION_CREDENTIALS: - # The complete project ID (required, not the short name) - GCP_PROJECT_ID: - # Pre-existing storage bucket w/ lifecycle-enabled - XFERBUCKET: "packer-import" # pre-created, globally unique, lifecycle-enabled - -# Don't leak sensitive values in error messages / output -sensitive-variables: - - 'GOOGLE_APPLICATION_CREDENTIALS' - - 'GCP_PROJECT_ID' - -# What images to produce in which cloud -builders: - - &nested_virt - name: 'fedora' - type: 'qemu' - accelerator: "kvm" - iso_url: '{{user `FEDORA_IMAGE_URL`}}' - disk_image: true - format: "raw" - disk_size: 5120 - iso_checksum_url: '{{user `FEDORA_CSUM_URL`}}' - iso_checksum_type: "sha256" - output_directory: '/tmp/{{build_name}}' - vm_name: "disk.raw" # actually qcow2, name required for post-processing - boot_wait: '5s' - shutdown_command: 'shutdown -h now' - headless: true - qemu_binary: "/usr/libexec/qemu-kvm" - qemuargs: # List-of-list format required to override packer-generated args - - - "-m" - - "1024" - - - "-cpu" - - "host" - - - "-device" - - "virtio-rng-pci" - - - "-chardev" - - "tty,id=pts,path={{user `TTYDEV`}}" - - - "-device" - - "isa-serial,chardev=pts" - - - "-cdrom" - - "{{user `CIDATA_ISO`}}" - - - "-netdev" - - "user,id=net0,hostfwd=tcp::{{ .SSHHostPort }}-:22" - - - "-device" - - "virtio-net,netdev=net0" - communicator: 'ssh' - ssh_private_key_file: 'cidata.ssh' - ssh_username: 'root' - - - <<: *nested_virt - name: 'prior-fedora' - iso_url: '{{user `PRIOR_FEDORA_IMAGE_URL`}}' - iso_checksum_url: '{{user `PRIOR_FEDORA_CSUM_URL`}}' - - - &imgcopy - name: 'ubuntu' - type: 'googlecompute' - image_name: '{{user `UBUNTU_BASE_IMAGE`}}' - image_family: '{{build_name}}-base' - source_image: '{{user `UBUNTU_BASE_IMAGE`}}' - source_image_project_id: 'ubuntu-os-cloud' - project_id: '{{user `GCP_PROJECT_ID`}}' - account_file: '{{user `GOOGLE_APPLICATION_CREDENTIALS`}}' - startup_script_file: "systemd_banish.sh" - zone: 'us-central1-a' - disk_size: 20 - communicator: 'none' - - - <<: *imgcopy - name: 'prior-ubuntu' - image_name: '{{user `PRIOR_UBUNTU_BASE_IMAGE`}}' - source_image: '{{user `PRIOR_UBUNTU_BASE_IMAGE`}}' - -provisioners: - - type: 'shell' - only: ['fedora', 'prior-fedora'] - inline: - - 'mkdir -p /tmp/libpod/{{user `SCRIPT_BASE`}}' - - 'mkdir -p /tmp/libpod/{{user `PACKER_BASE`}}' - - - type: 'file' - only: ['fedora', 'prior-fedora'] - source: '{{user `GOSRC`}}/.cirrus.yml' - destination: '/tmp/libpod/.cirrus.yml' - - - type: 'file' - only: ['fedora', 'prior-fedora'] - source: '{{user `GOSRC`}}/{{user `SCRIPT_BASE`}}/' - destination: '/tmp/libpod/{{user `SCRIPT_BASE`}}/' - - - type: 'file' - only: ['fedora', 'prior-fedora'] - source: '{{user `GOSRC`}}/{{user `PACKER_BASE`}}/' - destination: '/tmp/libpod/{{user `PACKER_BASE`}}/' - - - &shell_script - only: ['fedora', 'prior-fedora'] - type: 'shell' - inline: - - 'chmod +x /tmp/libpod/{{user `PACKER_BASE`}}/*.sh' - - '/tmp/libpod/{{user `PACKER_BASE`}}/{{build_name}}_base-setup.sh' - expect_disconnect: true # Allow this to reboot the VM if needed - environment_vars: - - 'TIMESTAMP={{user `TIMESTAMP`}}' - - 'GOSRC=/tmp/libpod' - - 'SCRIPT_BASE={{user `SCRIPT_BASE`}}' - - 'PACKER_BASE={{user `PACKER_BASE`}}' - -post-processors: - - - type: "compress" - only: ['fedora', 'prior-fedora'] - output: '/tmp/{{build_name}}/disk.raw.tar.gz' - format: '.tar.gz' - compression_level: 9 - - &gcp_import - only: ['fedora'] - type: "googlecompute-import" - project_id: '{{user `GCP_PROJECT_ID`}}' - account_file: '{{user `GOOGLE_APPLICATION_CREDENTIALS`}}' - bucket: '{{user `XFERBUCKET`}}' - gcs_object_name: '{{build_name}}-{{user `TIMESTAMP`}}.tar.gz' - image_name: "{{user `FEDORA_BASE_IMAGE_NAME`}}-{{user `TIMESTAMP`}}" - image_description: 'Based on {{user `FEDORA_IMAGE_URL`}}' - image_family: '{{build_name}}-base' - - <<: *gcp_import - only: ['prior-fedora'] - image_name: "{{user `PRIOR_FEDORA_BASE_IMAGE_NAME`}}-{{user `TIMESTAMP`}}" - image_description: 'Based on {{user `PRIOR_FEDORA_IMAGE_URL`}}' - image_family: '{{build_name}}-base' - - type: 'manifest' diff --git a/contrib/cirrus/packer/libpod_images.yml b/contrib/cirrus/packer/libpod_images.yml deleted file mode 100644 index 38f5a8250..000000000 --- a/contrib/cirrus/packer/libpod_images.yml +++ /dev/null @@ -1,86 +0,0 @@ ---- - -# All of these are required -variables: - BUILT_IMAGE_SUFFIX: '{{env `BUILT_IMAGE_SUFFIX`}}' - GOPATH: '{{env `GOPATH`}}' - GOSRC: '{{env `GOSRC`}}' - PACKER_BASE: '{{env `PACKER_BASE`}}' - SCRIPT_BASE: '{{env `SCRIPT_BASE`}}' - - # Base-image names are required. Using image family-names breaks parallelism - UBUNTU_BASE_IMAGE: '{{env `UBUNTU_BASE_IMAGE`}}' - PRIOR_UBUNTU_BASE_IMAGE: '{{env `PRIOR_UBUNTU_BASE_IMAGE`}}' - FEDORA_BASE_IMAGE: '{{env `FEDORA_BASE_IMAGE`}}' - PRIOR_FEDORA_BASE_IMAGE: '{{env `PRIOR_FEDORA_BASE_IMAGE`}}' - - # Protected credentials, decrypted by Cirrus at runtime - GCE_SSH_USERNAME: '{{env `GCE_SSH_USERNAME`}}' - GCP_PROJECT_ID: '{{env `GCP_PROJECT_ID`}}' - SERVICE_ACCOUNT: '{{env `SERVICE_ACCOUNT`}}' - GOOGLE_APPLICATION_CREDENTIALS: '{{env `GOOGLE_APPLICATION_CREDENTIALS`}}' - -# Don't leak sensitive values in error messages / output -sensitive-variables: - - 'GCE_SSH_USERNAME' - - 'GCP_PROJECT_ID' - - 'SERVICE_ACCOUNT' - -# What images to produce in which cloud -builders: - # v----- is a YAML anchor, allows referencing this object by name (below) - - &gce_hosted_image - name: 'ubuntu-20' - type: 'googlecompute' - image_name: '{{build_name}}{{user `BUILT_IMAGE_SUFFIX`}}' - image_family: '{{build_name}}-cache' - source_image: '{{user `UBUNTU_BASE_IMAGE`}}' # precedence over family - source_image_family: 'ubuntu-base' # for ref. only - disk_size: 20 # REQUIRED: Runtime allocation > this value - project_id: '{{user `GCP_PROJECT_ID`}}' - service_account_email: '{{user `SERVICE_ACCOUNT`}}' - communicator: 'ssh' - ssh_username: '{{user `GCE_SSH_USERNAME`}}' - ssh_pty: 'true' - # The only supported zone in Cirrus-CI, as of addition of this comment - zone: 'us-central1-a' - - # v----- is a YAML alias, allows partial re-use of the anchor object - - <<: *gce_hosted_image - name: 'ubuntu-19' - source_image: '{{user `PRIOR_UBUNTU_BASE_IMAGE`}}' - source_image_family: 'prior-ubuntu-base' - - - <<: *gce_hosted_image - name: 'fedora-32' - source_image: '{{user `FEDORA_BASE_IMAGE`}}' - source_image_family: 'fedora-base' - - - <<: *gce_hosted_image - name: 'fedora-31' - source_image: '{{user `PRIOR_FEDORA_BASE_IMAGE`}}' - source_image_family: 'prior-fedora-base' - -# The brains of the operation, making actual modifications to the base-image. -provisioners: - - type: 'shell' - inline: - - 'set -ex' - # The 'file' provisioner item (below) will create the final component - - 'mkdir -vp $(dirname {{user `GOSRC`}})' - - - type: 'file' - source: '{{user `GOSRC`}}' - destination: '{{user `GOSRC`}}' - - - type: 'shell' - script: '{{user `GOSRC`}}/{{user `PACKER_BASE`}}/{{split build_name "-" 0}}_setup.sh' - environment_vars: - - 'PACKER_BUILDER_NAME={{build_name}}' - - 'GOPATH={{user `GOPATH`}}' - - 'GOSRC={{user `GOSRC`}}' - - 'PACKER_BASE={{user `PACKER_BASE`}}' - - 'SCRIPT_BASE={{user `SCRIPT_BASE`}}' - -post-processors: - - type: 'manifest' # writes packer-manifest.json diff --git a/contrib/cirrus/packer/make-user-data.sh b/contrib/cirrus/packer/make-user-data.sh deleted file mode 100644 index 676a50f5c..000000000 --- a/contrib/cirrus/packer/make-user-data.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash - -# This script is utilized by Makefile, it's not intended to be run by humans - -cat <<EOF > user-data -#cloud-config -timezone: US/Eastern -growpart: - mode: auto -disable_root: false -ssh_pwauth: True -ssh_import_id: [root] -ssh_authorized_keys: - - $(cat cidata.ssh.pub) -users: - - name: root - primary-group: root - homedir: /root - system: true -EOF diff --git a/contrib/cirrus/packer/prior-fedora_base-setup.sh b/contrib/cirrus/packer/prior-fedora_base-setup.sh deleted file mode 100644 index bf29a1aec..000000000 --- a/contrib/cirrus/packer/prior-fedora_base-setup.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env bash - -# N/B: This script is not intended to be run by humans. It is used to configure the -# fedora base image for importing, so that it will boot in GCE - -set -e - -# Load in library (copied by packer, before this script was run) -source $GOSRC/$SCRIPT_BASE/lib.sh - -echo "Updating packages" -dnf -y update - -echo "Installing necessary packages and google services" -dnf -y install rng-tools google-compute-engine-tools google-compute-engine-oslogin ethtool - -echo "Enabling services" -systemctl enable rngd - -# There is a race that can happen on boot between the GCE services configuring -# the VM, and cloud-init trying to do similar activities. Use a customized -# unit file to make sure cloud-init starts after the google-compute-* services. -echo "Setting cloud-init service to start after google-network-daemon.service" -cp -v $GOSRC/$PACKER_BASE/cloud-init/fedora/cloud-init.service /etc/systemd/system/ - -# ref: https://cloud.google.com/compute/docs/startupscript -# The mechanism used by Cirrus-CI to execute tasks on the system is through an -# "agent" process launched as a GCP startup-script (from the metadata service). -# This agent is responsible for cloning the repository and executing all task -# scripts and other operations. Therefor, on SELinux-enforcing systems, the -# service must be labeled properly to ensure it's child processes can -# run with the proper contexts. -METADATA_SERVICE_CTX=unconfined_u:unconfined_r:unconfined_t:s0 -METADATA_SERVICE_PATH=systemd/system/google-startup-scripts.service -sed -r -e \ - "s/Type=oneshot/Type=oneshot\nSELinuxContext=$METADATA_SERVICE_CTX/" \ - /lib/$METADATA_SERVICE_PATH > /etc/$METADATA_SERVICE_PATH - -# Ensure there are no disruptive periodic services enabled by default in image -systemd_banish - -rh_finalize - -echo "SUCCESS!" diff --git a/contrib/cirrus/packer/systemd_banish.sh b/contrib/cirrus/packer/systemd_banish.sh deleted file mode 100755 index 2219f2a4f..000000000 --- a/contrib/cirrus/packer/systemd_banish.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash - -set +e # Not all of these exist on every platform - -# This is intended to be executed on VMs as a startup script on initial-boot. -# Alternatively, it may be executed with the '--list' option to return the list -# of systemd units defined for disablement (useful for testing). - -EVIL_UNITS="cron crond atd apt-daily-upgrade apt-daily fstrim motd-news systemd-tmpfiles-clean" - -if [[ "$1" == "--list" ]] -then - echo "$EVIL_UNITS" - exit 0 -fi - -echo "Disabling periodic services that could destabilize testing:" -for unit in $EVIL_UNITS -do - echo "Banishing $unit (ignoring errors)" - ( - sudo systemctl stop $unit - sudo systemctl disable $unit - sudo systemctl disable $unit.timer - sudo systemctl mask $unit - sudo systemctl mask $unit.timer - ) &> /dev/null -done diff --git a/contrib/cirrus/packer/ubuntu_packaging.sh b/contrib/cirrus/packer/ubuntu_packaging.sh deleted file mode 100644 index c478028b5..000000000 --- a/contrib/cirrus/packer/ubuntu_packaging.sh +++ /dev/null @@ -1,175 +0,0 @@ -#!/usr/bin/env bash - -# This script is called from ubuntu_setup.sh and various Dockerfiles. -# It's not intended to be used outside of those contexts. It assumes the lib.sh -# library has already been sourced, and that all "ground-up" package-related activity -# needs to be done, including repository setup and initial update. - -set -e - -echo "Updating/Installing repos and packages for $OS_REL_VER" - -source $GOSRC/$SCRIPT_BASE/lib.sh - -req_env_var GOSRC SCRIPT_BASE BIGTO SUDOAPTGET INSTALL_AUTOMATION_VERSION - -echo "Updating/configuring package repositories." -$BIGTO $SUDOAPTGET update - -echo "Installing deps to add third-party repositories and automation tooling" -$LILTO $SUDOAPTGET install software-properties-common git curl - -# Install common automation tooling (i.e. ooe.sh) -curl --silent --show-error --location \ - --url "https://raw.githubusercontent.com/containers/automation/master/bin/install_automation.sh" | \ - $SUDO env INSTALL_PREFIX=/usr/share /bin/bash -s - "$INSTALL_AUTOMATION_VERSION" -# Reload installed environment right now (happens automatically in a new process) -source /usr/share/automation/environment - -$LILTO ooe.sh $SUDOAPTADD ppa:criu/ppa - -echo "Configuring/Instaling deps from Open build server" -VERSION_ID=$(source /etc/os-release; echo $VERSION_ID) -echo "deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_$VERSION_ID/ /" \ - | ooe.sh $SUDO tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list -ooe.sh curl -L -o /tmp/Release.key "https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/xUbuntu_${VERSION_ID}/Release.key" -ooe.sh $SUDO apt-key add - < /tmp/Release.key - -INSTALL_PACKAGES=(\ - apache2-utils - apparmor - aufs-tools - autoconf - automake - bash-completion - bats - bison - btrfs-progs - build-essential - buildah - bzip2 - conmon - containernetworking-plugins - containers-common - coreutils - cri-o-runc - criu - curl - dnsmasq - e2fslibs-dev - emacs-nox - file - fuse3 - gawk - gcc - gettext - git - go-md2man - golang-1.14 - iproute2 - iptables - jq - libaio-dev - libapparmor-dev - libbtrfs-dev - libcap-dev - libdevmapper-dev - libdevmapper1.02.1 - libfuse-dev - libfuse2 - libfuse3-dev - libglib2.0-dev - libgpgme11-dev - liblzma-dev - libnet1 - libnet1-dev - libnl-3-dev - libprotobuf-c-dev - libprotobuf-dev - libseccomp-dev - libseccomp2 - libselinux-dev - libsystemd-dev - libtool - libudev-dev - libvarlink - lsof - make - netcat - openssl - pkg-config - podman - protobuf-c-compiler - protobuf-compiler - python-dateutil - python-protobuf - python2 - python3-dateutil - python3-pip - python3-psutil - python3-pytoml - python3-requests - python3-setuptools - rsync - runc - scons - skopeo - slirp4netns - socat - sudo - unzip - vim - wget - xz-utils - zip - zlib1g-dev -) -DOWNLOAD_PACKAGES=(\ - cri-o-$(get_kubernetes_version) - cri-tools - parallel -) - -# These aren't resolvable on Ubuntu 20 -if [[ "$OS_RELEASE_VER" -le 19 ]]; then - INSTALL_PACKAGES+=(\ - python-future - python-minimal - yum-utils - ) -else - INSTALL_PACKAGES+=(\ - python-is-python3 - ) -fi - -# Do this at the last possible moment to avoid dpkg lock conflicts -echo "Upgrading all packages" -$BIGTO ooe.sh $SUDOAPTGET upgrade - -echo "Installing general testing and system dependencies" -# Necessary to update cache of newly added repos -$LILTO ooe.sh $SUDOAPTGET update -$BIGTO ooe.sh $SUDOAPTGET install "${INSTALL_PACKAGES[@]}" - -if [[ ${#DOWNLOAD_PACKAGES[@]} -gt 0 ]]; then - echo "Downloading packages for optional installation at runtime, as needed." - $SUDO ln -s /var/cache/apt/archives "$PACKAGE_DOWNLOAD_DIR" - $LILTO ooe.sh $SUDOAPTGET install --download-only "${DOWNLOAD_PACKAGES[@]}" -fi - -echo "Configuring Go environment" -# There are multiple (otherwise conflicting) versions of golang available -# on Ubuntu. Being primarily localized by env. vars and defaults, dropping -# a symlink is the appropriate way to "install" a specific version system-wide. -$SUDO ln -sf /usr/lib/go-1.14/bin/go /usr/bin/go -# Initially go was not installed -cd $GOSRC -source $SCRIPT_BASE/lib.sh -echo "Go environment has been setup:" -go env - -echo "Building/Installing runtime tooling" -$SUDO hack/install_catatonit.sh -$SUDO make install.libseccomp.sudo -$SUDO make install.tools GO_BUILD='go build' # -mod=vendor breaks this diff --git a/contrib/cirrus/packer/ubuntu_setup.sh b/contrib/cirrus/packer/ubuntu_setup.sh deleted file mode 100644 index d650e6c76..000000000 --- a/contrib/cirrus/packer/ubuntu_setup.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env bash - -# This script is called by packer on the subject Ubuntu VM, to setup the podman -# build/test environment. It's not intended to be used outside of this context. - -set -e - -# Load in library (copied by packer, before this script was run) -source $GOSRC/$SCRIPT_BASE/lib.sh - -req_env_var SCRIPT_BASE PACKER_BASE INSTALL_AUTOMATION_VERSION PACKER_BUILDER_NAME GOSRC UBUNTU_BASE_IMAGE OS_RELEASE_ID OS_RELEASE_VER - -# Ensure there are no disruptive periodic services enabled by default in image -systemd_banish - -# Stop disruption upon boot ASAP after booting -echo "Disabling all packaging activity on boot" -for filename in $(sudo ls -1 /etc/apt/apt.conf.d); do \ - echo "Checking/Patching $filename" - sudo sed -i -r -e "s/$PERIODIC_APT_RE/"'\10"\;/' "/etc/apt/apt.conf.d/$filename"; done - -bash $PACKER_BASE/ubuntu_packaging.sh - -# Load installed environment right now (happens automatically in a new process) -source /usr/share/automation/environment - -echo "Making Ubuntu kernel to enable cgroup swap accounting as it is not the default." -SEDCMD='s/^GRUB_CMDLINE_LINUX="(.*)"/GRUB_CMDLINE_LINUX="\1 cgroup_enable=memory swapaccount=1"/g' -ooe.sh sudo sed -re "$SEDCMD" -i /etc/default/grub.d/* -ooe.sh sudo sed -re "$SEDCMD" -i /etc/default/grub -ooe.sh sudo update-grub - -ubuntu_finalize - -echo "SUCCESS!" diff --git a/contrib/cirrus/packer/xfedora_setup.sh b/contrib/cirrus/packer/xfedora_setup.sh deleted file mode 100644 index 16ae87d8a..000000000 --- a/contrib/cirrus/packer/xfedora_setup.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash - -# This script is called by packer on the subject fedora VM, to setup the podman -# build/test environment. It's not intended to be used outside of this context. - -set -e - -# Load in library (copied by packer, before this script was run) -source $GOSRC/$SCRIPT_BASE/lib.sh - -req_env_var SCRIPT_BASE PACKER_BASE INSTALL_AUTOMATION_VERSION PACKER_BUILDER_NAME GOSRC FEDORA_BASE_IMAGE OS_RELEASE_ID OS_RELEASE_VER - -workaround_bfq_bug - -# Do not enable updates-testing on the previous Fedora release -if [[ "$PRIOR_FEDORA_BASE_IMAGE" =~ "${OS_RELEASE_ID}-cloud-base-${OS_RELEASE_VER}" ]]; then - DISABLE_UPDATES_TESTING=1 -else - DISABLE_UPDATES_TESTING=0 -fi - -bash $PACKER_BASE/fedora_packaging.sh -# Load installed environment right now (happens automatically in a new process) -source /usr/share/automation/environment - -echo "Enabling cgroup management from containers" -ooe.sh sudo setsebool container_manage_cgroup true - -# Ensure there are no disruptive periodic services enabled by default in image -systemd_banish - -rh_finalize - -echo "SUCCESS!" diff --git a/contrib/cirrus/podbot.py b/contrib/cirrus/podbot.py deleted file mode 100755 index 9ca4915a7..000000000 --- a/contrib/cirrus/podbot.py +++ /dev/null @@ -1,105 +0,0 @@ -#!/usr/bin/env python3 - -# Simple and dumb script to send a message to the #podman IRC channel on frenode -# Based on example from: https://pythonspot.com/building-an-irc-bot/ - -import os -import time -import random -import errno -import socket -import sys - -class IRC: - - response_timeout = 30 # seconds - irc = socket.socket() - - def __init__(self, server, nickname, channel): - self.server = server - self.nickname = nickname - self.channel = channel - self.irc = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - - def _send(self, cmdstr): - self.irc.send(bytes(cmdstr + '\r\n', 'utf-8')) - - def message(self, msg): - data = 'PRIVMSG {0} :{1}\r\n'.format(self.channel, msg) - print(data) - self._send(data) - - @staticmethod - def fix_newlines(bufr): - return bufr.replace('\\r\\n', '\n') - - def _required_response(self, needle, haystack): - start = time.time() - end = start + self.response_timeout - while time.time() < end: - if haystack.find(needle) != -1: - return (False, haystack) - time.sleep(0.1) - try: - haystack += str(self.irc.recv(4096, socket.MSG_DONTWAIT)) - except socket.error as serr: - if serr.errno == errno.EWOULDBLOCK: - continue - raise # can't handle this - return (True, haystack) # Error - - def connect(self, username, password): - # This is ugly as sin, but seems to be a working send/expect sequence - - print("connecting to: {0}".format(self.server)) - self.irc.connect((self.server, 6667)) #connects to the server - self._send("USER {0} {0} {0} :I am {0}".format(self.nickname)) - self._send("NICK {0}".format(self.nickname)) - - err, haystack = self._required_response('End of /MOTD command.' - ''.format(self.nickname), "") - if err: - print(self.fix_newlines(haystack)) - print("Error connecting to {0}".format(self.server)) - return True - - print("Logging in as {0}".format(username)) - self._send("PRIVMSG NickServ :IDENTIFY {0} {1}".format(username, password)) - err, _ = self._required_response("You are now identified for", "") - if err: - print("Error logging in to {0} as {1}".format(self.server, username)) - return True - - print("Joining {0}".format(self.channel)) - self._send("JOIN {0}".format(self.channel)) - err, haystack = self._required_response("{0} {1} :End of /NAMES list." - "".format(self.nickname, self.channel), - haystack) - print(self.fix_newlines(haystack)) - if err: - print("Error joining {0}".format(self.channel)) - return True - return False - - def quit(self): - print("Quitting") - self._send("QUIT :my work is done here") - self.irc.close() - - -if len(sys.argv) < 3: - print("Error: Must pass desired nick and message as parameters") -else: - for try_again in (True,False): - irc = IRC("irc.freenode.net", sys.argv[1], "#podman") - err = irc.connect(*os.environ.get('IRCID', 'Big Bug').split(" ", 2)) - if err and try_again: - print("Trying again in 5 seconds...") - time.sleep(5) - continue - elif err: - break - irc.message(" ".join(sys.argv[2:])) - time.sleep(5.0) # avoid join/quit spam - irc.quit() - break diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh index 855866477..b406d7b5c 100755 --- a/contrib/cirrus/setup_environment.sh +++ b/contrib/cirrus/setup_environment.sh @@ -33,9 +33,6 @@ done # Sometimes environment setup needs to vary between distros # Note: This should only be used for environment variables, and temporary workarounds. -# Anything externally dependent, should be made fixed-in-time by adding to -# contrib/cirrus/packer/*_setup.sh to be incorporated into VM cache-images -# (see docs). cd "${GOSRC}/" case "${OS_RELEASE_ID}" in ubuntu) @@ -44,8 +41,6 @@ case "${OS_RELEASE_ID}" in # All SELinux distros need this for systemd-in-a-container setsebool container_manage_cgroup true - workaround_bfq_bug - if [[ "$ADD_SECOND_PARTITION" == "true" ]]; then bash "$SCRIPT_BASE/add_second_partition.sh" fi @@ -67,14 +62,14 @@ source "$SCRIPT_BASE/lib.sh" case "$CG_FS_TYPE" in tmpfs) warn "Forcing testing with runc instead of crun" - # On ubuntu, the default runc is usually not new enough. - if [[ "${OS_RELEASE_ID}" == "ubuntu" ]]; then - X=$(echo "export OCI_RUNTIME=/usr/lib/cri-o-runc/sbin/runc" | \ - tee -a /etc/environment) && eval "$X" && echo "$X" - else - X=$(echo "export OCI_RUNTIME=/usr/bin/runc" | \ - tee -a /etc/environment) && eval "$X" && echo "$X" - fi + # On ubuntu, the default runc is usually not new enough. + if [[ "$OS_RELEASE_ID" == "ubuntu" ]]; then + X=$(echo "export OCI_RUNTIME=/usr/lib/cri-o-runc/sbin/runc" | \ + tee -a /etc/environment) && eval "$X" && echo "$X" + else + X=$(echo "export OCI_RUNTIME=/usr/bin/runc" | \ + tee -a /etc/environment) && eval "$X" && echo "$X" + fi ;; cgroup2fs) # This is necessary since we've built/installed from source, which uses runc as the default. diff --git a/contrib/cirrus/success.sh b/contrib/cirrus/success.sh deleted file mode 100755 index 8783f6b81..000000000 --- a/contrib/cirrus/success.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env bash - -set -e - -source $(dirname $0)/lib.sh - -req_env_var CIRRUS_BRANCH CIRRUS_REPO_FULL_NAME CIRRUS_BASE_SHA CIRRUS_CHANGE_IN_REPO CIRRUS_CHANGE_MESSAGE - -cd $CIRRUS_WORKING_DIR - -if [[ "$CIRRUS_BRANCH" =~ "pull" ]] -then - echo "Retrieving latest HEADS and tags" - git fetch --all --tags - echo "Finding commit authors for PR $CIRRUS_PR" - unset NICKS - if [[ -r "$AUTHOR_NICKS_FILEPATH" ]] - then - SHARANGE="${CIRRUS_BASE_SHA}..${CIRRUS_CHANGE_IN_REPO}" - EXCLUDE_RE='merge-robot' - EMAILCSET='[:alnum:]-+_@.' - AUTHOR_NICKS=$(egrep -v '(^[[:space:]]*$)|(^[[:space:]]*#)' "$AUTHOR_NICKS_FILEPATH" | sort -u) - # Depending on branch-state, it's possible SHARANGE could be _WAY_ too big - MAX_NICKS=10 - # newline separated - GITLOG="git log --format='%ae'" - COMMIT_AUTHORS=$($GITLOG $SHARANGE || $GITLOG -1 HEAD | \ - tr --delete --complement "$EMAILCSET[:space:]" | \ - egrep -v "$EXCLUDE_RE" | \ - sort -u | \ - tail -$MAX_NICKS) - - for c_email in $COMMIT_AUTHORS - do - c_email=$(echo "$c_email" | tr --delete --complement "$EMAILCSET") - echo -e "\tExamining $c_email" - NICK=$(echo "$AUTHOR_NICKS" | grep -m 1 "$c_email" | \ - awk --field-separator ',' '{print $2}' | tr -d '[[:blank:]]') - if [[ -n "$NICK" ]] - then - echo -e "\t\tFound $c_email -> $NICK in $(basename $AUTHOR_NICKS_FILEPATH)" - else - echo -e "\t\tNot found in $(basename $AUTHOR_NICKS_FILEPATH), using e-mail username." - NICK=$(echo "$c_email" | cut -d '@' -f 1) - fi - if ! echo "$NICKS" | grep -q "$NICK" - then - echo -e "\tUsing nick $NICK" - NICKS="${NICKS:+$NICKS, }$NICK" - else - echo -e "\tNot re-adding duplicate nick $NICK" - fi - done - fi - - unset MENTION_PREFIX - [[ -z "$NICKS" ]] || \ - MENTION_PREFIX="$NICKS: " - - URL="https://github.com/$CIRRUS_REPO_FULL_NAME/pull/$CIRRUS_PR" - PR_SUBJECT=$(echo "$CIRRUS_CHANGE_MESSAGE" | head -1) - ircmsg "${MENTION_PREFIX}Cirrus-CI testing successful for PR '$PR_SUBJECT': $URL" -else - URL="https://cirrus-ci.com/github/containers/libpod/$CIRRUS_BRANCH" - ircmsg "Cirrus-CI testing branch $(basename $CIRRUS_BRANCH) successful: $URL" -fi diff --git a/contrib/rootless-cni-infra/Containerfile b/contrib/rootless-cni-infra/Containerfile new file mode 100644 index 000000000..c5d812a6e --- /dev/null +++ b/contrib/rootless-cni-infra/Containerfile @@ -0,0 +1,35 @@ +ARG GOLANG_VERSION=1.15 +ARG ALPINE_VERSION=3.12 +ARG CNI_VERSION=v0.8.0 +ARG CNI_PLUGINS_VERSION=v0.8.7 +# Aug 20, 2020 +ARG DNSNAME_VESION=78b4da7bbfc51c27366da630e1df1c4f2e8b1b5b + +FROM golang:${GOLANG_VERSION}-alpine${ALPINE_VERSION} AS golang-base +RUN apk add --no-cache git + +FROM golang-base AS cnitool +RUN git clone https://github.com/containernetworking/cni /go/src/github.com/containernetworking/cni +WORKDIR /go/src/github.com/containernetworking/cni +ARG CNI_VERSION +RUN git checkout ${CNI_VERSION} +RUN go build -o /cnitool ./cnitool + +FROM golang-base AS dnsname +RUN git clone https://github.com/containers/dnsname /go/src/github.com/containers/dnsname +WORKDIR /go/src/github.com/containers/dnsname +ARG DNSNAME_VERSION +RUN git checkout ${DNSNAME_VERSION} +RUN go build -o /dnsname ./plugins/meta/dnsname + +FROM alpine:${ALPINE_VERSION} +RUN apk add --no-cache curl dnsmasq iptables ip6tables iproute2 +ARG TARGETARCH +ARG CNI_PLUGINS_VERSION +RUN mkdir -p /opt/cni/bin && \ + curl -fsSL https://github.com/containernetworking/plugins/releases/download/${CNI_PLUGINS_VERSION}/cni-plugins-linux-${TARGETARCH}-${CNI_PLUGINS_VERSION}.tgz | tar xz -C /opt/cni/bin +COPY --from=cnitool /cnitool /usr/local/bin +COPY --from=dnsname /dnsname /opt/cni/bin +COPY rootless-cni-infra /usr/local/bin +ENV CNI_PATH=/opt/cni/bin +CMD ["sleep", "infinity"] diff --git a/contrib/rootless-cni-infra/README.md b/contrib/rootless-cni-infra/README.md new file mode 100644 index 000000000..937e057fb --- /dev/null +++ b/contrib/rootless-cni-infra/README.md @@ -0,0 +1,22 @@ +# rootless-cni-infra + +Infra container for CNI-in-slirp4netns. + +## How it works + +When a CNI network is specified for `podman run` in rootless mode, Podman launches the `rootless-cni-infra` container to execute CNI plugins inside slirp4netns. + +The infra container is created per user, by executing an equivalent of: +`podman run -d --name rootless-cni-infra --pid=host --privileged -v $HOME/.config/cni/net.d:/etc/cni/net.d rootless-cni-infra`. +The infra container is automatically deleted when no CNI network is in use. + +Podman then allocates a CNI netns in the infra container, by executing an equivalent of: +`podman exec rootless-cni-infra rootless-cni-infra alloc $CONTAINER_ID $NETWORK_NAME $POD_NAME`. + +The allocated netns is deallocated when the container is being removed, by executing an equivalent of: +`podman exec rootless-cni-infra rootless-cni-infra dealloc $CONTAINER_ID $NETWORK_NAME`. + +## Directory layout + +* `/run/rootless-cni-infra/${CONTAINER_ID}/pid`: PID of the `sleep infinity` process that corresponds to the allocated netns +* `/run/rootless-cni-infra/${CONTAINER_ID}/attached/${NETWORK_NAME}`: CNI result diff --git a/contrib/rootless-cni-infra/rootless-cni-infra b/contrib/rootless-cni-infra/rootless-cni-infra new file mode 100755 index 000000000..5a574d2eb --- /dev/null +++ b/contrib/rootless-cni-infra/rootless-cni-infra @@ -0,0 +1,147 @@ +#!/bin/sh +set -eu + +ARG0="$0" +VERSION="0.1.0" +BASE="/run/rootless-cni-infra" + +# CLI subcommand: "alloc $CONTAINER_ID $NETWORK_NAME $POD_NAME" +cmd_entrypoint_alloc() { + if [ "$#" -ne 3 ]; then + echo >&2 "Usage: $ARG0 alloc CONTAINER_ID NETWORK_NAME POD_NAME" + exit 1 + fi + + ID="$1" + NET="$2" + K8S_POD_NAME="$3" + + dir="${BASE}/${ID}" + mkdir -p "${dir}/attached" + + pid="" + if [ -f "${dir}/pid" ]; then + pid=$(cat "${dir}/pid") + else + unshare -n sleep infinity & + pid="$!" + echo "${pid}" >"${dir}/pid" + nsenter -t "${pid}" -n ip link set lo up + fi + CNI_ARGS="IgnoreUnknown=1;K8S_POD_NAME=${K8S_POD_NAME}" + nwcount=$(find "${dir}/attached" -type f | wc -l) + CNI_IFNAME="eth${nwcount}" + export CNI_ARGS CNI_IFNAME + cnitool add "${NET}" "/proc/${pid}/ns/net" >"${dir}/attached/${NET}" + + # return the result + ns="/proc/${pid}/ns/net" + echo "{\"ns\":\"${ns}\"}" +} + +# CLI subcommand: "dealloc $CONTAINER_ID $NETWORK_NAME" +cmd_entrypoint_dealloc() { + if [ "$#" -ne 2 ]; then + echo >&2 "Usage: $ARG0 dealloc CONTAINER_ID NETWORK_NAME" + exit 1 + fi + + ID=$1 + NET=$2 + + dir="${BASE}/${ID}" + if [ ! -f "${dir}/pid" ]; then + exit 0 + fi + pid=$(cat "${dir}/pid") + cnitool del "${NET}" "/proc/${pid}/ns/net" + rm -f "${dir}/attached/${NET}" + + nwcount=$(find "${dir}/attached" -type f | wc -l) + if [ "${nwcount}" = 0 ]; then + kill -9 "${pid}" + rm -rf "${dir}" + fi + + # return empty json + echo "{}" +} + +# CLI subcommand: "is-idle" +cmd_entrypoint_is_idle() { + if [ ! -d ${BASE} ]; then + echo '{"idle": true}' + elif [ -z "$(ls -1 ${BASE})" ]; then + echo '{"idle": true}' + else + echo '{"idle": false}' + fi +} + +# CLI subcommand: "print-cni-result $CONTAINER_ID $NETWORK_NAME" +cmd_entrypoint_print_cni_result() { + if [ "$#" -ne 2 ]; then + echo >&2 "Usage: $ARG0 print-cni-result CONTAINER_ID NETWORK_NAME" + exit 1 + fi + + ID=$1 + NET=$2 + + # the result shall be CNI JSON + cat "${BASE}/${ID}/attached/${NET}" +} + +# CLI subcommand: "print-netns-path $CONTAINER_ID" +cmd_entrypoint_print_netns_path() { + if [ "$#" -ne 1 ]; then + echo >&2 "Usage: $ARG0 print-netns-path CONTAINER_ID" + exit 1 + fi + + ID=$1 + + pid=$(cat "${BASE}/${ID}/pid") + path="/proc/${pid}/ns/net" + + # return the result + echo "{\"path\":\"${path}\"}" +} + +# CLI subcommand: "help" +cmd_entrypoint_help() { + echo "Usage: ${ARG0} COMMAND" + echo + echo "Rootless CNI Infra container" + echo + echo "Commands:" + echo " alloc Allocate a netns" + echo " dealloc Deallocate a netns" + echo " is-idle Print whether the infra container is idle" + echo " print-cni-result Print CNI result" + echo " print-netns-path Print netns path" + echo " help Print help" + echo " version Print version" +} + +# CLI subcommand: "version" +cmd_entrypoint_version() { + echo "{\"version\": \"${VERSION}\"}" +} + +# parse args +command="${1:-}" +if [ -z "$command" ]; then + echo >&2 "No command was specified. Run \`${ARG0} help\` to see the usage." + exit 1 +fi + +command_func=$(echo "cmd_entrypoint_${command}" | sed -e "s/-/_/g") +if ! command -v "${command_func}" >/dev/null 2>&1; then + echo >&2 "Unknown command: ${command}. Run \`${ARG0} help\` to see the usage." + exit 1 +fi + +# start the command func +shift +"${command_func}" "$@" diff --git a/docs/source/Tutorials.rst b/docs/source/Tutorials.rst index 33e4ae3d3..83818e3ae 100644 --- a/docs/source/Tutorials.rst +++ b/docs/source/Tutorials.rst @@ -6,7 +6,7 @@ Here are a number of useful tutorials to get you up and running with Podman. If * `Basic Setup and Use of Podman <https://github.com/containers/podman/blob/master/docs/tutorials/podman_tutorial.md>`_: Learn how to setup Podman and perform some basic commands with the utility. * `Basic Setup and Use of Podman in a Rootless environment <https://github.com/containers/podman/blob/master/docs/tutorials/rootless_tutorial.md>`_: The steps required to setup rootless Podman are enumerated. -* `Podman Mac Client tutorial <https://github.com/containers/podman/blob/master/docs/tutorials/mac_client.md>`_: Special setup for running the Podman remote client on a Mac and connecting to Podman running on a Linux VM are documented. +* `Podman Mac/Windows tutorial <https://github.com/containers/podman/blob/master/docs/tutorials/mac_win_client.md>`_: Special setup for running the Podman remote client on a Mac or Windows PC and connecting to Podman running on a Linux VM are documented. * `How to sign and distribute container images using Podman <https://github.com/containers/podman/blob/master/docs/tutorials/image_signing.md>`_: Learn how to setup and use image signing with Podman. * `Podman remote-client tutorial <https://github.com/containers/podman/blob/master/docs/tutorials/remote_client.md>`_: A brief how-to on using the Podman remote-client. * `How to use libpod for custom/derivative projects <https://github.com/containers/podman/blob/master/docs/tutorials/podman-derivative-api.md>`_: How the libpod API can be used within your own project. diff --git a/docs/source/markdown/podman-manifest-add.1.md b/docs/source/markdown/podman-manifest-add.1.md index 44815def5..c4d4417c4 100644 --- a/docs/source/markdown/podman-manifest-add.1.md +++ b/docs/source/markdown/podman-manifest-add.1.md @@ -33,6 +33,25 @@ the image. If *imageName* refers to a manifest list or image index, the architecture information will be retrieved from it. Otherwise, it will be retrieved from the image's configuration information. +**--authfile**=*path* + +Path of the authentication file. Default is ${XDG\_RUNTIME\_DIR}/containers/auth.json, which is set using `podman login`. +If the authorization state is not found there, $HOME/.docker/config.json is checked, which is set using `docker login`. (Not available for remote commands) + +Note: You can also override the default path of the authentication file by setting the REGISTRY\_AUTH\_FILE +environment variable. `export REGISTRY_AUTH_FILE=path` + +**--cert-dir**=*path* + +Use certificates at *path* (\*.crt, \*.cert, \*.key) to connect to the registry. +Default certificates directory is _/etc/containers/certs.d_. (Not available for remote commands) + +**--creds**=*creds* + +The [username[:password]] to use to authenticate with the registry if required. +If one or both values are not supplied, a command line prompt will appear and the +value can be entered. The password is entered without echo. + **--features** Specify the features list which the list or index records as requirements for @@ -50,6 +69,10 @@ configuration information. Specify the OS version which the list or index records as a requirement for the image. This option is rarely used. +**--tls-verify** + +Require HTTPS and verify certificates when talking to container registries (defaults to true). (Not available for remote commands) + **--variant** Specify the variant which the list or index records for the image. This option diff --git a/docs/tutorials/README.md b/docs/tutorials/README.md index 4beb069ac..7f7b4853d 100644 --- a/docs/tutorials/README.md +++ b/docs/tutorials/README.md @@ -12,9 +12,9 @@ Learn how to setup Podman and perform some basic commands with the utility. The steps required to setup rootless Podman are enumerated. -**[Setup on OS X](mac_client.md)** +**[Setup Mac/Windows](mac_win_client.md) -Special setup for running the Podman remote client on a Mac and connecting to Podman running on a Linux VM are documented. +Special setup for running the Podman remote client on a Mac or Windows PC and connecting to Podman running on a Linux VM are documented. **[Remote Client](remote_client.md)** diff --git a/docs/tutorials/mac_client.md b/docs/tutorials/mac_client.md index f6c9160a8..f406ca54d 100644 --- a/docs/tutorials/mac_client.md +++ b/docs/tutorials/mac_client.md @@ -1,99 +1,2 @@ -# Podman Mac Client tutorial - -## What is the Podman Mac Client - -First and foremost, the Mac Client is under heavy development. We are working on getting the -Mac client to be packaged and run for a native-like experience. This is the setup tutorial -for the Mac client at its current stage of development and packaging. - -The purpose of the Mac client for Podman is to allow users to run Podman on a Mac. Since Podman is a Linux -container engine, The Mac client is actually a version of the [Podman-remote client](remote_client.md), -edited to that the client side works on a Mac machine, and connects to a Podman "backend" on a Linux -machine, virtual or physical. The goal is to have a native-like experience when working with the Mac -client, so the command line interface of the remote client is exactly the same as the regular Podman -commands with the exception of some flags and commands that do not apply to the Mac client. - -## What you need - -To use the Mac client, you will need a binary built for MacOS and a Podman "backend" on a Linux machine; -hereafter referred to as the Podman node. In this context, a Podman node is a Linux system with Podman -installed on it and the varlink service activated. You will also need to be able to ssh into this -system as a user with privileges to the varlink socket (more on this later). - -For best results, use the most recent version of MacOS - -## Getting the Mac client -The Mac client is available through [Homebrew](https://brew.sh/). -``` -$ brew cask install podman -``` - -## Setting up the client and Podman node connection - -To use the Mac client, you must perform some setup on both the Mac and Podman nodes. In this case, -the Mac node refers to the Mac on which Podman is being run; and the Podman node refers to where -Podman and its storage reside. - -### Connection settings -Your Linux box must have ssh enabled, and you must copy your Mac's public key from `~/.sconf sh/id.pub` to -`/root/.ssh/authorized_keys` on your Linux box using `ssh-copy-id` This allows for the use of SSH keys -for remote access. - -You may need to edit your `/etc/ssh/sshd_config` in your Linux machine as follows: -``` -PermitRootLogin yes -``` - -Use of SSH keys are strongly encouraged to ensure a secure login. However, if you wish to avoid ‘logging in’ every -time you run a Podman command, you may edit your `/etc/ssh/sshd_config` on your Linux machine as follows: -``` -PasswordAuthentication no -PermitRootLogin without-password -``` - -### Podman node setup -The Podman node must be running a Linux distribution that supports Podman and must have Podman (not the Mac -client) installed. You must also have root access to the node. Check if your system uses systemd: -``` -$cat /proc/1/comm -systemd -``` -If it does, then simply start the Podman varlink socket: -``` -$ sudo systemctl start io.podman.socket -$ sudo systemctl enable io.podman.socket -``` - -If your system cannot use systemd, then you can manually establish the varlink socket with the Podman -command: -``` -$ sudo podman --log-level debug varlink --timeout 0 unix://run/podman/io.podman -``` - -### Required permissions -For now, the Mac client requires that you be able to run a privileged Podman and have privileged ssh -access to the remote system. This limitation is being worked on. - -#### Running the remote client -There are three different ways to pass connection information into the client: flags, conf file, and -environment variables. All three require information on username and a remote host ip address. Most often, -your username should be root and you can obtain your remote-host-ip using `ip addr` - -To connect using flags, you can use -``` -$ podman --remote-host remote-host-ip --username root images -REPOSITORY TAG IMAGE ID CREATED SIZE -quay.io/podman/stable latest 9c1e323be87f 10 days ago 414 MB -localhost/test latest 4b8c27c343e1 4 weeks ago 253 MB -k8s.gcr.io/pause 3.1 da86e6ba6ca1 20 months ago 747 kB -``` -If the conf file is set up, you may simply use Podman as you would on the linux machine. Take a look at -[podman-remote.conf.5.md](https://github.com/containers/podman/blob/master/docs/podman-remote.conf.5.md) on how to use the conf file: - -``` -$ podman images -REPOSITORY TAG IMAGE ID CREATED SIZE -quay.io/podman/stable latest 9c1e323be87f 10 days ago 414 MB -localhost/test latest 4b8c27c343e1 4 weeks ago 253 MB -k8s.gcr.io/pause 3.1 da86e6ba6ca1 20 months ago 747 kB -``` +# [Podman Mac Client tutorial](https://github.com/containers/podman/blob/master/docs/tutorials/mac_win_client.md) +This tutorial has moved! You can find out how to set up Podman on MacOS (as well as Windows) [here](https://github.com/containers/podman/blob/master/docs/tutorials/mac_win_client.md) diff --git a/docs/tutorials/mac_win_client.md b/docs/tutorials/mac_win_client.md new file mode 100644 index 000000000..63830a5b1 --- /dev/null +++ b/docs/tutorials/mac_win_client.md @@ -0,0 +1,111 @@ +# Podman Remote clients for MacOS and Windows + +## Introduction + +The core Podman runtime environment can only run on Linux operating systems. But other operating systems can use the “remote client” to manage their containers to a Linux backend. This remote client is nearly identical to the standard Podman program. Certain functions that do not make sense for remote clients have been removed. For example, the “--latest” switch for container commands has been removed. + +### Brief architecture + +The remote client uses a client-server model. You need Podman installed on a Linux machine or VM that also has the SSH daemon running. On the local operating system, when you execute a Podman command, Podman connects to the server via SSH. It then connects to the Podman service by using systemd socket activation. The Podman commands are executed on the server. From the client's point of view, it seems like Podman runs locally. + +## Obtaining and installing Podman + +### Windows + +Installing the Windows Podman client begins by downloading the Podman windows installer. The windows installer is built with each Podman release and is downloadable from its [release description page](https://github.com/containers/podman/releases/latest). You can also build the installer from source using the `podman.msi` Makefile endpoint. + +Once you have downloaded the installer, simply double click the installer and Podman will be installed. The path is also set to put `podman` in the default user path. + +Podman must be run at a command prompt using the Windows ‘cmd” or powershell applications. + +### MacOS + +The Mac Client is available through [Homebrew](https://brew.sh/). You can download homebrew via the instructions on their site. Install podman using: +``` +$ brew install podman +``` + +## Creating the first connection + +### Enable the Podman service on the server machine. + +Before performing any Podman client commands, you must enable the podman.sock SystemD service on the Linux server. In these examples, we are running Podman as a normal, unprivileged user, also known as a rootless user. By default, the rootless socket listens at `/run/user/${UID}/podman/podman.sock`. You can enable this socket, permanently using the following command: +``` +$ systemctl --user enable podman.socket +``` +You will need to enable linger for this user in order for the socket to work when the user is not logged in. + +``` +$ sudo loginctl enable-linger $USER +``` + +You can verify that the socket is listening with a simple Podman command. + +``` +$ podman --remote info +host: + arch: amd64 + buildahVersion: 1.16.0-dev + cgroupVersion: v2 + conmon: + package: conmon-2.0.19-1.fc32.x86_64 +``` + +#### Enable sshd + +In order for the client to communicate with the server you need to enable and start the SSH daemon on your Linux machine, if it is not currently enabled. +``` +$ sudo systemctl enable -s sshd +``` + +#### Setting up SSH +Remote podman uses SSH to communicate between the client and server. The remote client works considerably smoother using SSH keys. To set up your ssh connection, you need to generate an ssh key pair from your client machine. +``` +$ ssh-keygen +``` +Your public key by default should be in your home directory under .ssh\id_rsa.pub. You then need to copy the contents of id_rsa.pub and append it into ~/.ssh/authorized_keys on the Linux server. On a Mac, you can automate this using ssh-copy-id. + +If you do not wish to use SSH keys, you will be prompted with each Podman command for your login password. + +## Using the client + +The first step in using the Podman remote client is to configure a connection.. + +You can add a connection by using the `podman system connection add` command. + +``` +C:\Users\baude> podman system connection add baude --identity c:\Users\baude\.ssh\id_rsa ssh://192.168.122.1/run/user/1000/podman/podman.sock +``` + +This will add a remote connection to Podman and if it is the first connection added, it will mark the connection as the default. You can observe your connections with `podman system connection list` + +``` +C:\Users\baude> podman system connection list +Name Identity URI +baude* id_rsa ssh://baude@192.168.122.1/run/user/1000/podman/podman.sock +``` + +Now we can test the connection with `podman info`. + +``` +C:\Users\baude> podman info +host: + arch: amd64 + buildahVersion: 1.16.0-dev + cgroupVersion: v2 + conmon: + package: conmon-2.0.19-1.fc32.x86_64 +``` + +Podman has also introduced a “--connection” flag where you can use other connections you have defined. If no connection is provided, the default connection will be used. + +``` +C:\Users\baude> podman system connection --help +``` + +## Wrap up + +You can use the podman remote clients to manage your containers running on a Linux server. The communication between client and server relies heavily on SSH connections and the use of SSH keys are encouraged. Once you have Podman installed on your remote client, you should set up a connection using `podman system connection add` which will then be used by subsequent Podman commands. + +## History +Originally published on [Red Hat Enable Sysadmin](https://www.redhat.com/sysadmin/podman-clients-macos-windows) diff --git a/docs/tutorials/podman_tutorial.md b/docs/tutorials/podman_tutorial.md index 97268fc41..85b95af04 100644 --- a/docs/tutorials/podman_tutorial.md +++ b/docs/tutorials/podman_tutorial.md @@ -5,7 +5,7 @@ Podman is a utility provided as part of the libpod library. It can be used to c containers. The following tutorial will teach you how to set up Podman and perform some basic commands with Podman. -If you are running on a Mac, you should instead follow the [Mac tutorial](https://github.com/containers/podman/blob/master/docs/tutorials/mac_client.md) +If you are running on a Mac or Windows PC, you should instead follow the [Mac and Windows tutorial](https://github.com/containers/podman/blob/master/docs/tutorials/mac_win_client.md) to set up the remote Podman client. **NOTE**: the code samples are intended to be run as a non-root user, and use `sudo` where diff --git a/docs/tutorials/remote_client.md b/docs/tutorials/remote_client.md index d4c43dda2..ad506d19a 100644 --- a/docs/tutorials/remote_client.md +++ b/docs/tutorials/remote_client.md @@ -1,88 +1,112 @@ # Podman remote-client tutorial -## What is the remote-client +## Introduction +The purpose of the Podman remote-client is to allow users to interact with a Podman "backend" while on a separate client. The command line interface of the remote client is exactly the same as the regular Podman commands with the exception of some flags being removed as they do not apply to the remote-client. -First and foremost, the remote-client is under heavy development. We are adding new -commands and functions frequently. We also are working on a rootless implementation that -does not require privileged users. +The remote client takes advantage of a client-server model. You need Podman installed on a Linux machine or VM that also has the SSH daemon running. On the local operating system, when you execute a Podman command, Podman connects to the server via SSH. It then connects to the Podman service by using systemd socket activation, and hitting our [Rest API](https://docs.podman.io/en/latest/_static/api.html). The Podman commands are executed on the server. From the client's point of view, it seems like Podman runs locally. -The purpose of the Podman remote-client is to allow users to interact with a Podman "backend" -while on a separate client. The command line interface of the remote client is exactly the -same as the regular Podman commands with the exception of some flags being removed as they -do not apply to the remote-client. +This tutorial is for running Podman remotely on Linux. If you are using a Mac or a Windows PC, please follow the [Mac and Windows tutorial](https://github.com/containers/podman/blob/master/docs/tutorials/mac_win_client.md) -## What you need -To use the remote-client, you will need a binary for your client and a Podman "backend"; hereafter -referred to as the Podman node. In this context, a Podman node is a Linux system with Podman -installed on it and the varlink service activated. You will also need to be able to ssh into this -system as a user with privileges to the varlink socket (more on this later). +## Obtaining and installing Podman -## Building the remote client -At this time, the Podman remote-client is not being packaged for any distribution. It must be built from -source. To set up your build environment, see [Installation notes](https://github.com/containers/podman/blob/master/install.md) and follow the -section [Building from scratch](https://github.com/containers/podman/blob/master/install.md#building-from-scratch). Once you can successfully -build the regular Podman binary, you can now build the remote-client. +### Client machine +You will need either Podman or the podman-remote client. The difference between the two is that the compiled podman-remote client can only act as a remote client connecting to a backend, while Podman can run local, standard Podman commands, as well as act as a remote client (using `podman --remote`) + +If you already have Podman installed, you do not need to install podman-remote. + +You can find out how to [install Podman here](https://podman.io/getting-started/installation) + +If you would like to install only the podman-remote client, it is downloadable from its [release description page](https://github.com/containers/podman/releases/latest). You can also build it from source using the `make podman-remote` + + +### Server Machine +You will need to [install Podman](https://podman.io/getting-started/installation) on your server machine. + + +## Creating the first connection + +### Enable the Podman service on the server machine. + +Before performing any Podman client commands, you must enable the podman.sock SystemD service on the Linux server. In these examples, we are running Podman as a normal, unprivileged user, also known as a rootless user. By default, the rootless socket listens at `/run/user/${UID}/podman/podman.sock`. You can enable this socket permanently using the following command: ``` -$ make podman-remote +$ systemctl --user enable podman.socket ``` -Like building the regular Podman, the resulting binary will be in the *bin* directory. This is the binary -you will run on the remote node later in the instructions. +You will need to enable linger for this user in order for the socket to work when the user is not logged in: -## Setting up the remote and Podman nodes +``` +$ sudo loginctl enable-linger $USER +``` +This is only required if you are not running Podman as root. -To use the remote-client, you must perform some setup on both the remote and Podman nodes. In this case, -the remote node refers to where the remote-client is being run; and the Podman node refers to where -Podman and its storage reside. +You can verify that the socket is listening with a simple Podman command. +``` +$ podman --remote info +host: + arch: amd64 + buildahVersion: 1.16.0-dev + cgroupVersion: v2 + conmon: + package: conmon-2.0.19-1.fc32.x86_64 +``` -### Podman node setup +#### Enable sshd -Varlink bridge support is provided by the varlink cli command and installed using: +In order for the Podman client to communicate with the server you need to enable and start the SSH daemon on your Linux machine, if it is not currently enabled. ``` -$ sudo dnf install varlink-cli +$ sudo systemctl enable -s sshd ``` -The Podman node must have Podman (not the remote-client) installed as normal. If your system uses systemd, -then simply start the Podman varlink socket. +#### Setting up SSH +Remote Podman uses SSH to communicate between the client and server. The remote client works considerably smoother using SSH keys. To set up your ssh connection, you need to generate an ssh key pair from your client machine. ``` -$ sudo systemctl start io.podman.socket +$ ssh-keygen ``` +Your public key by default should be in your home directory under ~/.ssh/id_rsa.pub. You then need to copy the contents of id_rsa.pub and append it into ~/.ssh/authorized_keys on the Linux server. You can automate this using ssh-copy-id. + +If you do not wish to use SSH keys, you will be prompted with each Podman command for your login password. + +## Using the client + +Note: `podman-remote` is equivalent to `podman --remote` here, depending on what you have chosen to install. + +The first step in using the Podman remote client is to configure a connection. + +You can add a connection by using the `podman-remote system connection add` command. -If your system cannot use systemd, then you can manually establish the varlink socket with the Podman -command: ``` -$ sudo podman --log-level debug varlink --timeout 0 unix://run/podman/io.podman +$ podman-remote system connection add myuser --identity ~/.ssh/id_rsa ssh://192.168.122.1/run/user/1000/podman/podman.sock ``` -### Required permissions -For now, the remote-client requires that you be able to run a privileged Podman and have privileged ssh -access to the remote system. This limitation is being worked on. +This will add a remote connection to Podman and if it is the first connection added, it will mark the connection as the default. You can observe your connections with `podman-remote system connection list`: -### Remote node setup - -#### Initiate an ssh session to the Podman node -To use the remote client, an ssh connection to the Podman server must be established. +``` +$ podman-remote system connection list +Name Identity URI +myuser* id_rsa ssh://myuser@192.168.122.1/run/user/1000/podman/podman.sock +``` -Using the varlink bridge, an ssh tunnel must be initiated to connect to the server. Podman must then be informed of the location of the sshd server on the targeted server +Now we can test the connection with `podman info`: ``` -$ export PODMAN_VARLINK_BRIDGE=$'ssh -T -p22 root@remotehost -- "varlink -A \'podman varlink \$VARLINK_ADDRESS\' bridge"' -$ bin/podman-remote images -REPOSITORY TAG IMAGE ID CREATED SIZE -docker.io/library/ubuntu latest 47b19964fb50 2 weeks ago 90.7 MB -docker.io/library/alpine latest caf27325b298 3 weeks ago 5.8 MB -quay.io/cevich/gcloud_centos latest 641dad61989a 5 weeks ago 489 MB -k8s.gcr.io/pause 3.1 da86e6ba6ca1 14 months ago 747 kB +$ podman-remote info +host: + arch: amd64 + buildahVersion: 1.16.0-dev + cgroupVersion: v2 + conmon: + package: conmon-2.0.19-1.fc32.x86_64 ``` -The PODMAN_VARLINK_BRIDGE variable may be added to your log in settings. It does not change per connection. +Podman-remote has also introduced a “--connection” flag where you can use other connections you have defined. If no connection is provided, the default connection will be used. -If coming from a Windows machine, the PODMAN_VARLINK_BRIDGE is formatted as: ``` -set PODMAN_VARLINK_BRIDGE=C:\Windows\System32\OpenSSH\ssh.exe -T -p22 root@remotehost -- varlink -A "podman varlink $VARLINK_ADDRESS" bridge +$ podman-remote system connection --help ``` -The arguments before the `--` are presented to ssh while the arguments after are for the varlink cli. The varlink arguments should be copied verbatim. - - `-p` is the port on the remote host for the ssh tunnel. `22` is the default. - - `root` is the currently supported user, while `remotehost` is the name or IP address of the host providing the Podman service. - - `-i` may be added to select an identity file. +## Wrap up + +You can use the Podman remote clients to manage your containers running on a Linux server. The communication between client and server relies heavily on SSH connections and the use of SSH keys are encouraged. Once you have Podman installed on your remote client, you should set up a connection using `podman-remote system connection add` which will then be used by subsequent Podman commands. + +## History +Adapted from the [Mac and Windows tutorial](https://github.com/containers/podman/blob/master/docs/tutorials/mac_win_client.md) diff --git a/docs/tutorials/varlink_remote_client.md b/docs/tutorials/varlink_remote_client.md new file mode 100644 index 000000000..54c648a48 --- /dev/null +++ b/docs/tutorials/varlink_remote_client.md @@ -0,0 +1,89 @@ +# Podman varlink remote-client tutorial [DEPRECATED] + +## What is the varlink client + +This API has been deprecated by the [REST API](https://docs.podman.io/en/latest/_static/api.html). +For usage on Windows and Mac, please reference the [Podman Mac/Windows tutorial](https://github.com/containers/podman/blob/master/docs/tutorials/mac_win_client.md) +Varlink support is in maintenance mode, and will be removed in a future release. +For more details, you can see [this blog](https://podman.io/blogs/2020/01/17/podman-new-api.html). + +The purpose of the Podman remote-client is to allow users to interact with a Podman "backend" +while on a separate client. The command line interface of the remote client is exactly the +same as the regular Podman commands with the exception of some flags being removed as they +do not apply to the remote-client. + +## What you need +To use the remote-client, you will need a binary for your client and a Podman "backend"; hereafter +referred to as the Podman node. In this context, a Podman node is a Linux system with Podman +installed on it and the varlink service activated. You will also need to be able to ssh into this +system as a user with privileges to the varlink socket (more on this later). + +## Building the remote client +At this time, the Podman remote-client is not being packaged for any distribution. It must be built from +source. To set up your build environment, see [Installation notes](https://github.com/containers/podman/blob/master/install.md) and follow the +section [Building from scratch](https://github.com/containers/podman/blob/master/install.md#building-from-scratch). Once you can successfully +build the regular Podman binary, you can now build the remote-client. +``` +$ make podman-remote +``` +Like building the regular Podman, the resulting binary will be in the *bin* directory. This is the binary +you will run on the remote node later in the instructions. + +## Setting up the remote and Podman nodes + +To use the remote-client, you must perform some setup on both the remote and Podman nodes. In this case, +the remote node refers to where the remote-client is being run; and the Podman node refers to where +Podman and its storage reside. + + +### Podman node setup + +Varlink bridge support is provided by the varlink cli command and installed using: +``` +$ sudo dnf install varlink-cli +``` + +The Podman node must have Podman (not the remote-client) installed as normal. If your system uses systemd, +then simply start the Podman varlink socket. +``` +$ sudo systemctl start io.podman.socket +``` + +If your system cannot use systemd, then you can manually establish the varlink socket with the Podman +command: +``` +$ sudo podman --log-level debug varlink --timeout 0 unix://run/podman/io.podman +``` + +### Required permissions +For now, the remote-client requires that you be able to run a privileged Podman and have privileged ssh +access to the remote system. This limitation is being worked on. + +### Remote node setup + +#### Initiate an ssh session to the Podman node +To use the remote client, an ssh connection to the Podman server must be established. + +Using the varlink bridge, an ssh tunnel must be initiated to connect to the server. Podman must then be informed of the location of the sshd server on the targeted server + +``` +$ export PODMAN_VARLINK_BRIDGE=$'ssh -T -p22 root@remotehost -- "varlink -A \'podman varlink \$VARLINK_ADDRESS\' bridge"' +$ bin/podman-remote images +REPOSITORY TAG IMAGE ID CREATED SIZE +docker.io/library/ubuntu latest 47b19964fb50 2 weeks ago 90.7 MB +docker.io/library/alpine latest caf27325b298 3 weeks ago 5.8 MB +quay.io/cevich/gcloud_centos latest 641dad61989a 5 weeks ago 489 MB +k8s.gcr.io/pause 3.1 da86e6ba6ca1 14 months ago 747 kB +``` + +The PODMAN_VARLINK_BRIDGE variable may be added to your log in settings. It does not change per connection. + +If coming from a Windows machine, the PODMAN_VARLINK_BRIDGE is formatted as: +``` +set PODMAN_VARLINK_BRIDGE=C:\Windows\System32\OpenSSH\ssh.exe -T -p22 root@remotehost -- varlink -A "podman varlink $VARLINK_ADDRESS" bridge +``` + +The arguments before the `--` are presented to ssh while the arguments after are for the varlink cli. The varlink arguments should be copied verbatim. + - `-p` is the port on the remote host for the ssh tunnel. `22` is the default. + - `root` is the currently supported user, while `remotehost` is the name or IP address of the host providing the Podman service. + - `-i` may be added to select an identity file. @@ -62,7 +62,7 @@ require ( golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a golang.org/x/sys v0.0.0-20200810151505-1b9f1253b3ed k8s.io/api v0.0.0-20190620084959-7cf5895f2711 - k8s.io/apimachinery v0.19.0 + k8s.io/apimachinery v0.19.1 k8s.io/client-go v0.0.0-20190620085101-78d2af792bab ) @@ -645,8 +645,8 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh k8s.io/api v0.0.0-20190620084959-7cf5895f2711 h1:BblVYz/wE5WtBsD/Gvu54KyBUTJMflolzc5I2DTvh50= k8s.io/api v0.0.0-20190620084959-7cf5895f2711/go.mod h1:TBhBqb1AWbBQbW3XRusr7n7E4v2+5ZY8r8sAMnyFC5A= k8s.io/apimachinery v0.0.0-20190612205821-1799e75a0719/go.mod h1:I4A+glKBHiTgiEjQiCCQfCAIcIMFGt291SmsvcrFzJA= -k8s.io/apimachinery v0.19.0 h1:gjKnAda/HZp5k4xQYjL0K/Yb66IvNqjthCb03QlKpaQ= -k8s.io/apimachinery v0.19.0/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA= +k8s.io/apimachinery v0.19.1 h1:cwsxZazM/LA9aUsBaL4bRS5ygoM6bYp8dFk22DSYQa4= +k8s.io/apimachinery v0.19.1/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA= k8s.io/client-go v0.0.0-20190620085101-78d2af792bab h1:E8Fecph0qbNsAbijJJQryKu4Oi9QTp5cVpjTE+nqg6g= k8s.io/client-go v0.0.0-20190620085101-78d2af792bab/go.mod h1:E95RaSlHr79aHaX0aGSwcPNfygDiPKOVXdmivCIZT0k= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= diff --git a/libpod/container_internal.go b/libpod/container_internal.go index c41d81a2b..c3f07a48b 100644 --- a/libpod/container_internal.go +++ b/libpod/container_internal.go @@ -957,8 +957,10 @@ func (c *Container) completeNetworkSetup() error { if err := c.syncContainer(); err != nil { return err } - if c.config.NetMode.IsSlirp4netns() { + if rootless.IsRootless() { return c.runtime.setupRootlessNetNS(c) + } else if c.config.NetMode.IsSlirp4netns() { + return c.runtime.setupSlirp4netns(c) } if err := c.runtime.setupNetNS(c); err != nil { return err diff --git a/libpod/container_internal_linux.go b/libpod/container_internal_linux.go index f789b0069..605b526a4 100644 --- a/libpod/container_internal_linux.go +++ b/libpod/container_internal_linux.go @@ -84,7 +84,11 @@ func (c *Container) prepare() error { // Set up network namespace if not already set up noNetNS := c.state.NetNS == nil if c.config.CreateNetNS && noNetNS && !c.config.PostConfigureNetNS { - netNS, networkStatus, createNetNSErr = c.runtime.createNetNS(c) + if rootless.IsRootless() && len(c.config.Networks) > 0 { + netNS, networkStatus, createNetNSErr = AllocRootlessCNI(context.Background(), c) + } else { + netNS, networkStatus, createNetNSErr = c.runtime.createNetNS(c) + } if createNetNSErr != nil { return } @@ -98,8 +102,12 @@ func (c *Container) prepare() error { } // handle rootless network namespace setup - if noNetNS && c.config.NetMode.IsSlirp4netns() && !c.config.PostConfigureNetNS { - createNetNSErr = c.runtime.setupRootlessNetNS(c) + if noNetNS && !c.config.PostConfigureNetNS { + if rootless.IsRootless() { + createNetNSErr = c.runtime.setupRootlessNetNS(c) + } else if c.config.NetMode.IsSlirp4netns() { + createNetNSErr = c.runtime.setupSlirp4netns(c) + } } }() // Mount storage if not mounted diff --git a/libpod/container_validate.go b/libpod/container_validate.go index d657e3549..b78168cd1 100644 --- a/libpod/container_validate.go +++ b/libpod/container_validate.go @@ -2,7 +2,6 @@ package libpod import ( "github.com/containers/podman/v2/libpod/define" - "github.com/containers/podman/v2/pkg/rootless" spec "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" ) @@ -68,16 +67,6 @@ func (c *Container) validate() error { } } - // Rootless has some requirements, compared to networks. - if rootless.IsRootless() { - if len(c.config.Networks) > 0 { - return errors.Wrapf(define.ErrInvalidArg, "cannot join CNI networks if running rootless") - } - - // TODO: Should we make sure network mode is set to Slirp if set - // at all? - } - // Can only set static IP or MAC is creating a network namespace. if !c.config.CreateNetNS && (c.config.StaticIP != nil || c.config.StaticMAC != nil) { return errors.Wrapf(define.ErrInvalidArg, "cannot set static IP or MAC address if not creating a network namespace") diff --git a/libpod/networking_linux.go b/libpod/networking_linux.go index 6f266e5d6..c0508ce39 100644 --- a/libpod/networking_linux.go +++ b/libpod/networking_linux.go @@ -4,6 +4,7 @@ package libpod import ( "bytes" + "context" "crypto/rand" "fmt" "io" @@ -208,6 +209,20 @@ func checkSlirpFlags(path string) (*slirpFeatures, error) { // Configure the network namespace for a rootless container func (r *Runtime) setupRootlessNetNS(ctr *Container) error { + if ctr.config.NetMode.IsSlirp4netns() { + return r.setupSlirp4netns(ctr) + } + if len(ctr.config.Networks) > 0 { + // set up port forwarder for CNI-in-slirp4netns + netnsPath := ctr.state.NetNS.Path() + // TODO: support slirp4netns port forwarder as well + return r.setupRootlessPortMappingViaRLK(ctr, netnsPath) + } + return nil +} + +// setupSlirp4netns can be called in rootful as well as in rootless +func (r *Runtime) setupSlirp4netns(ctr *Container) error { path := r.config.Engine.NetworkCmdPath if path == "" { @@ -711,7 +726,7 @@ func (r *Runtime) teardownNetNS(ctr *Container) error { logrus.Debugf("Tearing down network namespace at %s for container %s", ctr.state.NetNS.Path(), ctr.ID()) - // rootless containers do not use the CNI plugin + // rootless containers do not use the CNI plugin directly if !rootless.IsRootless() && !ctr.config.NetMode.IsSlirp4netns() { var requestedIP net.IP if ctr.requestedIP != nil { @@ -738,6 +753,13 @@ func (r *Runtime) teardownNetNS(ctr *Container) error { } } + // CNI-in-slirp4netns + if rootless.IsRootless() && len(ctr.config.Networks) != 0 { + if err := DeallocRootlessCNI(context.Background(), ctr); err != nil { + return errors.Wrapf(err, "error tearing down CNI-in-slirp4netns for container %s", ctr.ID()) + } + } + // First unmount the namespace if err := netns.UnmountNS(ctr.state.NetNS); err != nil { return errors.Wrapf(err, "error unmounting network namespace for container %s", ctr.ID()) diff --git a/libpod/networking_unsupported.go b/libpod/networking_unsupported.go index dd72a3fd8..76bb01424 100644 --- a/libpod/networking_unsupported.go +++ b/libpod/networking_unsupported.go @@ -8,6 +8,10 @@ func (r *Runtime) setupRootlessNetNS(ctr *Container) error { return define.ErrNotImplemented } +func (r *Runtime) setupSlirp4netns(ctr *Container) error { + return define.ErrNotImplemented +} + func (r *Runtime) setupNetNS(ctr *Container) error { return define.ErrNotImplemented } diff --git a/libpod/oci_conmon_linux.go b/libpod/oci_conmon_linux.go index f66835771..bb138ca14 100644 --- a/libpod/oci_conmon_linux.go +++ b/libpod/oci_conmon_linux.go @@ -1086,7 +1086,7 @@ func (r *ConmonOCIRuntime) createOCIContainer(ctr *Container, restoreOptions *Co cmd.ExtraFiles = append(cmd.ExtraFiles, childSyncPipe, childStartPipe) cmd.ExtraFiles = append(cmd.ExtraFiles, envFiles...) - if r.reservePorts && !ctr.config.NetMode.IsSlirp4netns() { + if r.reservePorts && !rootless.IsRootless() && !ctr.config.NetMode.IsSlirp4netns() { ports, err := bindPorts(ctr.config.PortMappings) if err != nil { return err @@ -1098,7 +1098,7 @@ func (r *ConmonOCIRuntime) createOCIContainer(ctr *Container, restoreOptions *Co cmd.ExtraFiles = append(cmd.ExtraFiles, ports...) } - if ctr.config.NetMode.IsSlirp4netns() { + if ctr.config.NetMode.IsSlirp4netns() || rootless.IsRootless() { if ctr.config.PostConfigureNetNS { havePortMapping := len(ctr.Config().PortMappings) > 0 if havePortMapping { diff --git a/libpod/rootless_cni_linux.go b/libpod/rootless_cni_linux.go new file mode 100644 index 000000000..76dbfdcae --- /dev/null +++ b/libpod/rootless_cni_linux.go @@ -0,0 +1,320 @@ +// +build linux + +package libpod + +import ( + "bytes" + "context" + "io" + "path/filepath" + "runtime" + + cnitypes "github.com/containernetworking/cni/pkg/types/current" + "github.com/containernetworking/plugins/pkg/ns" + "github.com/containers/podman/v2/libpod/define" + "github.com/containers/podman/v2/libpod/image" + "github.com/containers/podman/v2/pkg/util" + "github.com/containers/storage/pkg/lockfile" + "github.com/hashicorp/go-multierror" + spec "github.com/opencontainers/runtime-spec/specs-go" + "github.com/opencontainers/runtime-tools/generate" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +var rootlessCNIInfraImage = map[string]string{ + // Built from ../contrib/rootless-cni-infra + // TODO: move to Podman's official quay + "amd64": "ghcr.io/akihirosuda/podman-rootless-cni-infra:gd34868a13-amd64", +} + +const ( + rootlessCNIInfraContainerNamespace = "podman-system" + rootlessCNIInfraContainerName = "rootless-cni-infra" +) + +// AllocRootlessCNI allocates a CNI netns inside the rootless CNI infra container. +// Locks "rootless-cni-infra.lck". +// +// When the infra container is not running, it is created. +// +// AllocRootlessCNI does not lock c. c should be already locked. +func AllocRootlessCNI(ctx context.Context, c *Container) (ns.NetNS, []*cnitypes.Result, error) { + if len(c.config.Networks) == 0 { + return nil, nil, errors.New("allocRootlessCNI shall not be called when len(c.config.Networks) == 0") + } + l, err := getRootlessCNIInfraLock(c.runtime) + if err != nil { + return nil, nil, err + } + l.Lock() + defer l.Unlock() + infra, err := ensureRootlessCNIInfraContainerRunning(ctx, c.runtime) + if err != nil { + return nil, nil, err + } + k8sPodName := getPodOrContainerName(c) // passed to CNI as K8S_POD_NAME + cniResults := make([]*cnitypes.Result, len(c.config.Networks)) + for i, nw := range c.config.Networks { + cniRes, err := rootlessCNIInfraCallAlloc(infra, c.ID(), nw, k8sPodName) + if err != nil { + return nil, nil, err + } + cniResults[i] = cniRes + } + nsObj, err := rootlessCNIInfraGetNS(infra, c.ID()) + if err != nil { + return nil, nil, err + } + logrus.Debugf("rootless CNI: container %q will join %q", c.ID(), nsObj.Path()) + return nsObj, cniResults, nil +} + +// DeallocRootlessCNI deallocates a CNI netns inside the rootless CNI infra container. +// Locks "rootless-cni-infra.lck". +// +// When the infra container is no longer needed, it is removed. +// +// DeallocRootlessCNI does not lock c. c should be already locked. +func DeallocRootlessCNI(ctx context.Context, c *Container) error { + if len(c.config.Networks) == 0 { + return errors.New("deallocRootlessCNI shall not be called when len(c.config.Networks) == 0") + } + l, err := getRootlessCNIInfraLock(c.runtime) + if err != nil { + return err + } + l.Lock() + defer l.Unlock() + infra, _ := getRootlessCNIInfraContainer(c.runtime) + if infra == nil { + return nil + } + var errs *multierror.Error + for _, nw := range c.config.Networks { + err := rootlessCNIInfraCallDelloc(infra, c.ID(), nw) + if err != nil { + errs = multierror.Append(errs, err) + } + } + if isIdle, err := rootlessCNIInfraIsIdle(infra); isIdle || err != nil { + if err != nil { + logrus.Warn(err) + } + logrus.Debugf("rootless CNI: removing infra container %q", infra.ID()) + if err := c.runtime.removeContainer(ctx, infra, true, false, true); err != nil { + return err + } + logrus.Debugf("rootless CNI: removed infra container %q", infra.ID()) + } + return errs.ErrorOrNil() +} + +func getRootlessCNIInfraLock(r *Runtime) (lockfile.Locker, error) { + fname := filepath.Join(r.config.Engine.TmpDir, "rootless-cni-infra.lck") + return lockfile.GetLockfile(fname) +} + +func getPodOrContainerName(c *Container) string { + pod, err := c.runtime.GetPod(c.PodID()) + if err != nil || pod.config.Name == "" { + return c.Name() + } + return pod.config.Name +} + +func rootlessCNIInfraCallAlloc(infra *Container, id, nw, k8sPodName string) (*cnitypes.Result, error) { + logrus.Debugf("rootless CNI: alloc %q, %q, %q", id, nw, k8sPodName) + var err error + + _, err = rootlessCNIInfraExec(infra, "alloc", id, nw, k8sPodName) + if err != nil { + return nil, err + } + cniResStr, err := rootlessCNIInfraExec(infra, "print-cni-result", id, nw) + if err != nil { + return nil, err + } + var cniRes cnitypes.Result + if err := json.Unmarshal([]byte(cniResStr), &cniRes); err != nil { + return nil, errors.Wrapf(err, "unmarshaling as cnitypes.Result: %q", cniResStr) + } + return &cniRes, nil +} + +func rootlessCNIInfraCallDelloc(infra *Container, id, nw string) error { + logrus.Debugf("rootless CNI: dealloc %q, %q", id, nw) + _, err := rootlessCNIInfraExec(infra, "dealloc", id, nw) + return err +} + +func rootlessCNIInfraIsIdle(infra *Container) (bool, error) { + type isIdle struct { + Idle bool `json:"idle"` + } + resStr, err := rootlessCNIInfraExec(infra, "is-idle") + if err != nil { + return false, err + } + var res isIdle + if err := json.Unmarshal([]byte(resStr), &res); err != nil { + return false, errors.Wrapf(err, "unmarshaling as isIdle: %q", resStr) + } + return res.Idle, nil +} + +func rootlessCNIInfraGetNS(infra *Container, id string) (ns.NetNS, error) { + type printNetnsPath struct { + Path string `json:"path"` + } + resStr, err := rootlessCNIInfraExec(infra, "print-netns-path", id) + if err != nil { + return nil, err + } + var res printNetnsPath + if err := json.Unmarshal([]byte(resStr), &res); err != nil { + return nil, errors.Wrapf(err, "unmarshaling as printNetnsPath: %q", resStr) + } + nsObj, err := ns.GetNS(res.Path) + if err != nil { + return nil, err + } + return nsObj, nil +} + +func getRootlessCNIInfraContainer(r *Runtime) (*Container, error) { + containers, err := r.GetContainersWithoutLock(func(c *Container) bool { + return c.Namespace() == rootlessCNIInfraContainerNamespace && + c.Name() == rootlessCNIInfraContainerName + }) + if err != nil { + return nil, err + } + if len(containers) == 0 { + return nil, nil + } + return containers[0], nil +} + +func ensureRootlessCNIInfraContainerRunning(ctx context.Context, r *Runtime) (*Container, error) { + c, err := getRootlessCNIInfraContainer(r) + if err != nil { + return nil, err + } + if c == nil { + return startRootlessCNIInfraContainer(ctx, r) + } + st, err := c.ContainerState() + if err != nil { + return nil, err + } + if st.State == define.ContainerStateRunning { + logrus.Debugf("rootless CNI: infra container %q is already running", c.ID()) + return c, nil + } + logrus.Debugf("rootless CNI: infra container %q is %q, being started", c.ID(), st.State) + if err := c.initAndStart(ctx); err != nil { + return nil, err + } + logrus.Debugf("rootless CNI: infra container %q is running", c.ID()) + return c, nil +} + +func startRootlessCNIInfraContainer(ctx context.Context, r *Runtime) (*Container, error) { + imageName, ok := rootlessCNIInfraImage[runtime.GOARCH] + if !ok { + return nil, errors.Errorf("cannot find rootless-podman-network-sandbox image for %s", runtime.GOARCH) + } + logrus.Debugf("rootless CNI: ensuring image %q to exist", imageName) + newImage, err := r.ImageRuntime().New(ctx, imageName, "", "", nil, nil, + image.SigningOptions{}, nil, util.PullImageMissing) + if err != nil { + return nil, err + } + logrus.Debugf("rootless CNI: image %q is ready", imageName) + + g, err := generate.New("linux") + if err != nil { + return nil, err + } + g.SetupPrivileged(true) + // Set --pid=host for ease of propagating "/proc/PID/ns/net" string + if err := g.RemoveLinuxNamespace(string(spec.PIDNamespace)); err != nil { + return nil, err + } + g.RemoveMount("/proc") + procMount := spec.Mount{ + Destination: "/proc", + Type: "bind", + Source: "/proc", + Options: []string{"rbind", "nosuid", "noexec", "nodev"}, + } + g.AddMount(procMount) + // Mount CNI networks + etcCNINetD := spec.Mount{ + Destination: "/etc/cni/net.d", + Type: "bind", + Source: r.config.Network.NetworkConfigDir, + Options: []string{"ro"}, + } + g.AddMount(etcCNINetD) + // FIXME: how to propagate ProcessArgs and Envs from Dockerfile? + g.SetProcessArgs([]string{"sleep", "infinity"}) + g.AddProcessEnv("CNI_PATH", "/opt/cni/bin") + var options []CtrCreateOption + options = append(options, WithRootFSFromImage(newImage.ID(), imageName, imageName)) + options = append(options, WithCtrNamespace(rootlessCNIInfraContainerNamespace)) + options = append(options, WithName(rootlessCNIInfraContainerName)) + options = append(options, WithPrivileged(true)) + options = append(options, WithSecLabels([]string{"disable"})) + options = append(options, WithRestartPolicy("always")) + options = append(options, WithNetNS(nil, false, "slirp4netns", nil)) + c, err := r.NewContainer(ctx, g.Config, options...) + if err != nil { + return nil, err + } + logrus.Debugf("rootless CNI infra container %q is created, now being started", c.ID()) + if err := c.initAndStart(ctx); err != nil { + return nil, err + } + logrus.Debugf("rootless CNI: infra container %q is running", c.ID()) + + return c, nil +} + +func rootlessCNIInfraExec(c *Container, args ...string) (string, error) { + cmd := "rootless-cni-infra" + var ( + outB bytes.Buffer + errB bytes.Buffer + streams define.AttachStreams + config ExecConfig + ) + streams.OutputStream = &nopWriteCloser{Writer: &outB} + streams.ErrorStream = &nopWriteCloser{Writer: &errB} + streams.AttachOutput = true + streams.AttachError = true + config.Command = append([]string{cmd}, args...) + config.Privileged = true + logrus.Debugf("rootlessCNIInfraExec: c.ID()=%s, config=%+v, streams=%v, begin", + c.ID(), config, streams) + code, err := c.Exec(&config, &streams, nil) + logrus.Debugf("rootlessCNIInfraExec: c.ID()=%s, config=%+v, streams=%v, end (code=%d, err=%v)", + c.ID(), config, streams, code, err) + if err != nil { + return "", err + } + if code != 0 { + return "", errors.Errorf("command %s %v in container %s failed with status %d, stdout=%q, stderr=%q", + cmd, args, c.ID(), code, outB.String(), errB.String()) + } + return outB.String(), nil +} + +type nopWriteCloser struct { + io.Writer +} + +func (nwc *nopWriteCloser) Close() error { + return nil +} diff --git a/libpod/runtime_ctr.go b/libpod/runtime_ctr.go index 936dce2e9..241448981 100644 --- a/libpod/runtime_ctr.go +++ b/libpod/runtime_ctr.go @@ -772,7 +772,11 @@ func (r *Runtime) LookupContainer(idOrName string) (*Container, error) { func (r *Runtime) GetContainers(filters ...ContainerFilter) ([]*Container, error) { r.lock.RLock() defer r.lock.RUnlock() + return r.GetContainersWithoutLock(filters...) +} +// GetContainersWithoutLock is same as GetContainers but without lock +func (r *Runtime) GetContainersWithoutLock(filters ...ContainerFilter) ([]*Container, error) { if !r.valid { return nil, define.ErrRuntimeStopped } diff --git a/pkg/api/handlers/compat/networks.go b/pkg/api/handlers/compat/networks.go index 80b7505df..87b947549 100644 --- a/pkg/api/handlers/compat/networks.go +++ b/pkg/api/handlers/compat/networks.go @@ -5,6 +5,7 @@ import ( "net" "net/http" "os" + "strings" "syscall" "time" @@ -177,9 +178,11 @@ func ListNetworks(w http.ResponseWriter, r *http.Request) { utils.InternalServerError(w, err) return } + + filterNames, nameFilterExists := query.Filters["name"] // TODO remove when filters are implemented - if len(query.Filters) > 0 { - utils.InternalServerError(w, errors.New("filters for listing networks is not implemented")) + if (!nameFilterExists && len(query.Filters) > 0) || len(query.Filters) > 1 { + utils.InternalServerError(w, errors.New("only the name filter for listing networks is implemented")) return } netNames, err := network.GetNetworkNamesFromFileSystem(config) @@ -187,6 +190,21 @@ func ListNetworks(w http.ResponseWriter, r *http.Request) { utils.InternalServerError(w, err) return } + + // filter by name + if nameFilterExists { + names := []string{} + for _, name := range netNames { + for _, filter := range filterNames { + if strings.Contains(name, filter) { + names = append(names, name) + break + } + } + } + netNames = names + } + reports := make([]*types.NetworkResource, 0, len(netNames)) for _, name := range netNames { report, err := getNetworkResourceByName(name, runtime) diff --git a/pkg/api/handlers/libpod/networks.go b/pkg/api/handlers/libpod/networks.go index 475522664..dfece2a4e 100644 --- a/pkg/api/handlers/libpod/networks.go +++ b/pkg/api/handlers/libpod/networks.go @@ -42,7 +42,21 @@ func CreateNetwork(w http.ResponseWriter, r *http.Request) { } func ListNetworks(w http.ResponseWriter, r *http.Request) { runtime := r.Context().Value("runtime").(*libpod.Runtime) - options := entities.NetworkListOptions{} + decoder := r.Context().Value("decoder").(*schema.Decoder) + query := struct { + Filter string `schema:"filter"` + }{ + // override any golang type defaults + } + if err := decoder.Decode(&query, r.URL.Query()); err != nil { + utils.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest, + errors.Wrapf(err, "Failed to parse parameters for %s", r.URL.String())) + return + } + + options := entities.NetworkListOptions{ + Filter: query.Filter, + } ic := abi.ContainerEngine{Libpod: runtime} reports, err := ic.NetworkList(r.Context(), options) if err != nil { diff --git a/pkg/api/server/register_networks.go b/pkg/api/server/register_networks.go index 7918ad4a2..61916eedf 100644 --- a/pkg/api/server/register_networks.go +++ b/pkg/api/server/register_networks.go @@ -61,6 +61,11 @@ func (s *APIServer) registerNetworkHandlers(r *mux.Router) error { // - networks (compat) // summary: List networks // description: Display summary of network configurations + // parameters: + // - in: query + // name: filters + // type: string + // description: JSON encoded value of the filters (a map[string][]string) to process on the networks list. Only the name filter is supported. // produces: // - application/json // responses: @@ -106,7 +111,7 @@ func (s *APIServer) registerNetworkHandlers(r *mux.Router) error { // required: true // description: the name of the network // - in: query - // name: Force + // name: force // type: boolean // description: remove containers associated with network // produces: @@ -152,6 +157,11 @@ func (s *APIServer) registerNetworkHandlers(r *mux.Router) error { // - networks // summary: List networks // description: Display summary of network configurations + // parameters: + // - in: query + // name: filter + // type: string + // description: Provide filter values (e.g. 'name=podman') // produces: // - application/json // responses: diff --git a/pkg/bindings/network/network.go b/pkg/bindings/network/network.go index fd1111282..d8dc7e352 100644 --- a/pkg/bindings/network/network.go +++ b/pkg/bindings/network/network.go @@ -70,7 +70,7 @@ func Remove(ctx context.Context, nameOrID string, force *bool) ([]*entities.Netw } // List returns a summary of all CNI network configurations -func List(ctx context.Context) ([]*entities.NetworkListReport, error) { +func List(ctx context.Context, options entities.NetworkListOptions) ([]*entities.NetworkListReport, error) { var ( netList []*entities.NetworkListReport ) @@ -78,7 +78,11 @@ func List(ctx context.Context) ([]*entities.NetworkListReport, error) { if err != nil { return nil, err } - response, err := conn.DoRequest(nil, http.MethodGet, "/networks/json", nil, nil) + params := url.Values{} + if options.Filter != "" { + params.Set("filter", options.Filter) + } + response, err := conn.DoRequest(nil, http.MethodGet, "/networks/json", params, nil) if err != nil { return netList, err } diff --git a/pkg/domain/entities/manifest.go b/pkg/domain/entities/manifest.go index 853619b19..01180951a 100644 --- a/pkg/domain/entities/manifest.go +++ b/pkg/domain/entities/manifest.go @@ -9,14 +9,19 @@ type ManifestCreateOptions struct { } type ManifestAddOptions struct { - All bool `json:"all" schema:"all"` - Annotation []string `json:"annotation" schema:"annotation"` - Arch string `json:"arch" schema:"arch"` - Features []string `json:"features" schema:"features"` - Images []string `json:"images" schema:"images"` - OS string `json:"os" schema:"os"` - OSVersion string `json:"os_version" schema:"os_version"` - Variant string `json:"variant" schema:"variant"` + All bool `json:"all" schema:"all"` + Annotation []string `json:"annotation" schema:"annotation"` + Arch string `json:"arch" schema:"arch"` + Authfile string `json:"-" schema:"-"` + CertDir string `json:"-" schema:"-"` + Features []string `json:"features" schema:"features"` + Images []string `json:"images" schema:"images"` + OS string `json:"os" schema:"os"` + OSVersion string `json:"os_version" schema:"os_version"` + Password string `json:"-" schema:"-"` + SkipTLSVerify types.OptionalBool `json:"-" schema:"-"` + Username string `json:"-" schema:"-"` + Variant string `json:"variant" schema:"variant"` } type ManifestAnnotateOptions struct { diff --git a/pkg/domain/infra/abi/manifest.go b/pkg/domain/infra/abi/manifest.go index 6f3c6b902..55f73bf65 100644 --- a/pkg/domain/infra/abi/manifest.go +++ b/pkg/domain/infra/abi/manifest.go @@ -102,7 +102,24 @@ func (ir *ImageEngine) ManifestAdd(ctx context.Context, opts entities.ManifestAd } manifestAddOpts.Annotation = annotations } - listID, err := listImage.AddManifest(*ir.Libpod.SystemContext(), manifestAddOpts) + + // Set the system context. + sys := ir.Libpod.SystemContext() + if sys != nil { + sys = &types.SystemContext{} + } + sys.AuthFilePath = opts.Authfile + sys.DockerInsecureSkipTLSVerify = opts.SkipTLSVerify + sys.DockerCertPath = opts.CertDir + + if opts.Username != "" && opts.Password != "" { + sys.DockerAuthConfig = &types.DockerAuthConfig{ + Username: opts.Username, + Password: opts.Password, + } + } + + listID, err := listImage.AddManifest(*sys, manifestAddOpts) if err != nil { return listID, err } @@ -191,6 +208,7 @@ func (ir *ImageEngine) ManifestPush(ctx context.Context, names []string, opts en } sys.AuthFilePath = opts.Authfile sys.DockerInsecureSkipTLSVerify = opts.SkipTLSVerify + sys.DockerCertPath = opts.CertDir if opts.Username != "" && opts.Password != "" { sys.DockerAuthConfig = &types.DockerAuthConfig{ diff --git a/pkg/domain/infra/abi/network.go b/pkg/domain/infra/abi/network.go index c06714cbb..807e4b272 100644 --- a/pkg/domain/infra/abi/network.go +++ b/pkg/domain/infra/abi/network.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "io/ioutil" + "os" "path/filepath" "strings" @@ -216,6 +217,9 @@ func createBridge(r *libpod.Runtime, name string, options entities.NetworkCreate if err != nil { return "", err } + if err := os.MkdirAll(network.GetCNIConfDir(runtimeConfig), 0755); err != nil { + return "", err + } cniPathName := filepath.Join(network.GetCNIConfDir(runtimeConfig), fmt.Sprintf("%s.conflist", name)) err = ioutil.WriteFile(cniPathName, b, 0644) return cniPathName, err diff --git a/pkg/domain/infra/tunnel/containers.go b/pkg/domain/infra/tunnel/containers.go index cc919561f..062b38a70 100644 --- a/pkg/domain/infra/tunnel/containers.go +++ b/pkg/domain/infra/tunnel/containers.go @@ -8,11 +8,13 @@ import ( "os" "strconv" "strings" + "sync" "time" "github.com/containers/common/pkg/config" "github.com/containers/image/v5/docker/reference" "github.com/containers/podman/v2/libpod/define" + "github.com/containers/podman/v2/libpod/events" "github.com/containers/podman/v2/pkg/api/handlers" "github.com/containers/podman/v2/pkg/bindings" "github.com/containers/podman/v2/pkg/bindings/containers" @@ -507,33 +509,90 @@ func (ic *ContainerEngine) ContainerRun(ctx context.Context, opts entities.Conta for _, w := range con.Warnings { fmt.Fprintf(os.Stderr, "%s\n", w) } + report := entities.ContainerRunReport{Id: con.ID} - // Attach - if !opts.Detach { - err = startAndAttach(ic, con.ID, &opts.DetachKeys, opts.InputStream, opts.OutputStream, opts.ErrorStream) - if err == nil { - exitCode, err := containers.Wait(ic.ClientCxt, con.ID, nil) - if err == nil { - report.ExitCode = int(exitCode) - } + + if opts.Detach { + // Detach and return early + err := containers.Start(ic.ClientCxt, con.ID, nil) + if err != nil { + report.ExitCode = define.ExitCode(err) } - } else { - err = containers.Start(ic.ClientCxt, con.ID, nil) + return &report, err } - if err != nil { + + // Attach + if err := startAndAttach(ic, con.ID, &opts.DetachKeys, opts.InputStream, opts.OutputStream, opts.ErrorStream); err != nil { report.ExitCode = define.ExitCode(err) + if opts.Rm { + if rmErr := containers.Remove(ic.ClientCxt, con.ID, bindings.PFalse, bindings.PTrue); rmErr != nil { + logrus.Debugf("unable to remove container %s after failing to start and attach to it", con.ID) + } + } + return &report, err } + if opts.Rm { - if err := containers.Remove(ic.ClientCxt, con.ID, bindings.PFalse, bindings.PTrue); err != nil { - if errors.Cause(err) == define.ErrNoSuchCtr || - errors.Cause(err) == define.ErrCtrRemoved { - logrus.Warnf("Container %s does not exist: %v", con.ID, err) - } else { - logrus.Errorf("Error removing container %s: %v", con.ID, err) + // Defer the removal, so we can return early if needed and + // de-spaghetti the code. + defer func() { + if err := containers.Remove(ic.ClientCxt, con.ID, bindings.PFalse, bindings.PTrue); err != nil { + if errors.Cause(err) == define.ErrNoSuchCtr || + errors.Cause(err) == define.ErrCtrRemoved { + logrus.Warnf("Container %s does not exist: %v", con.ID, err) + } else { + logrus.Errorf("Error removing container %s: %v", con.ID, err) + } } + }() + } + + // Wait + exitCode, waitErr := containers.Wait(ic.ClientCxt, con.ID, nil) + if waitErr == nil { + report.ExitCode = int(exitCode) + return &report, nil + } + + // Determine why the wait failed. If the container doesn't exist, + // consult the events. + if !strings.Contains(waitErr.Error(), define.ErrNoSuchCtr.Error()) { + return &report, waitErr + } + + // Events + eventsChannel := make(chan *events.Event) + eventOptions := entities.EventsOptions{ + EventChan: eventsChannel, + Filter: []string{ + "type=container", + fmt.Sprintf("container=%s", con.ID), + fmt.Sprintf("event=%s", events.Exited), + }, + } + + var lastEvent *events.Event + var mutex sync.Mutex + mutex.Lock() + // Read the events. + go func() { + for e := range eventsChannel { + lastEvent = e } + mutex.Unlock() + }() + + eventsErr := ic.Events(ctx, eventOptions) + + // Wait for all events to be read + mutex.Lock() + if eventsErr != nil || lastEvent == nil { + logrus.Errorf("Cannot get exit code: %v", err) + report.ExitCode = define.ExecErrorCodeNotFound + return &report, nil // compat with local client } + report.ExitCode = lastEvent.ContainerExitCode return &report, err } diff --git a/pkg/domain/infra/tunnel/network.go b/pkg/domain/infra/tunnel/network.go index 2b197cac0..074425087 100644 --- a/pkg/domain/infra/tunnel/network.go +++ b/pkg/domain/infra/tunnel/network.go @@ -8,7 +8,7 @@ import ( ) func (ic *ContainerEngine) NetworkList(ctx context.Context, options entities.NetworkListOptions) ([]*entities.NetworkListReport, error) { - return network.List(ic.ClientCxt) + return network.List(ic.ClientCxt, options) } func (ic *ContainerEngine) NetworkInspect(ctx context.Context, namesOrIds []string, options entities.NetworkInspectOptions) ([]entities.NetworkInspectReport, error) { diff --git a/pkg/network/files.go b/pkg/network/files.go index 38ce38b97..a2090491f 100644 --- a/pkg/network/files.go +++ b/pkg/network/files.go @@ -14,11 +14,16 @@ import ( "github.com/pkg/errors" ) -func GetCNIConfDir(config *config.Config) string { - if len(config.Network.NetworkConfigDir) < 1 { - return CNIConfigDir +func GetCNIConfDir(configArg *config.Config) string { + if len(configArg.Network.NetworkConfigDir) < 1 { + dc, err := config.DefaultConfig() + if err != nil { + // Fallback to hard-coded dir + return CNIConfigDir + } + return dc.Network.NetworkConfigDir } - return config.Network.NetworkConfigDir + return configArg.Network.NetworkConfigDir } // LoadCNIConfsFromDir loads all the CNI configurations from a dir diff --git a/rootless.md b/rootless.md index 196ed52c3..22b03e340 100644 --- a/rootless.md +++ b/rootless.md @@ -28,9 +28,6 @@ can easily fail * Can not use overlayfs driver, but does support fuse-overlayfs * Ubuntu supports non root overlay, but no other Linux distros do. * Only other supported driver is VFS. -* No CNI Support - * CNI wants to modify IPTables, plus other network manipulation that requires CAP_SYS_ADMIN. - * There is potential we could probably do some sort of denylisting of the relevant plugins, and add a new plugin for rootless networking - slirp4netns as one example and there may be others * Cannot use ping out of the box. * [(Can be fixed by setting sysctl on host)](https://github.com/containers/podman/blob/master/troubleshooting.md#6-rootless-containers-cannot-ping-hosts) * Requires new shadow-utils (not found in older (RHEL7/Centos7 distros) Should be fixed in RHEL7.7 release) diff --git a/test/apiv2/35-networks.at b/test/apiv2/35-networks.at index 4c032c072..143d6c07b 100644 --- a/test/apiv2/35-networks.at +++ b/test/apiv2/35-networks.at @@ -21,6 +21,27 @@ if root; then t POST libpod/networks/create '"Subnet":{"IP":"10.10.1.0","Mask":[0,255,255,0]}' 500 \ .cause~'.*mask is invalid' + # network list + t GET libpod/networks/json 200 + t GET libpod/networks/json?filter=name=network1 200 \ + length=1 \ + .[0].Name=network1 + t GET networks 200 + + #network list docker endpoint + #filters={"name":["network1","network2"]} + t GET networks?filters=%7B%22name%22%3A%5B%22network1%22%2C%22network2%22%5D%7D 200 \ + length=2 + #filters={"name":["network"]} + t GET networks?filters=%7B%22name%22%3A%5B%22network%22%5D%7D 200 \ + length=2 + # invalid filter filters={"label":"abc"} + t GET networks?filters=%7B%22label%22%3A%5B%22abc%22%5D%7D 500 \ + .cause="only the name filter for listing networks is implemented" + # invalid filter filters={"label":"abc","name":["network"]} + t GET networks?filters=%7B%22label%22%3A%22abc%22%2C%22name%22%3A%5B%22network%22%5D%7D 500 \ + .cause="only the name filter for listing networks is implemented" + # clean the network t DELETE libpod/networks/network1 200 \ .[0].Name~network1 \ diff --git a/test/e2e/common_test.go b/test/e2e/common_test.go index ed55484e3..b6bbae15b 100644 --- a/test/e2e/common_test.go +++ b/test/e2e/common_test.go @@ -245,6 +245,12 @@ func PodmanTestCreateUtil(tempDir string, remote bool) *PodmanTestIntegration { } os.Setenv("DISABLE_HC_SYSTEMD", "true") CNIConfigDir := "/etc/cni/net.d" + if rootless.IsRootless() { + CNIConfigDir = filepath.Join(os.Getenv("HOME"), ".config/cni/net.d") + } + if err := os.MkdirAll(CNIConfigDir, 0755); err != nil { + panic(err) + } storageFs := STORAGE_FS if rootless.IsRootless() { diff --git a/test/e2e/network_create_test.go b/test/e2e/network_create_test.go index f97e6c1f1..13d515d8e 100644 --- a/test/e2e/network_create_test.go +++ b/test/e2e/network_create_test.go @@ -74,7 +74,6 @@ var _ = Describe("Podman network create", func() { ) BeforeEach(func() { - SkipIfRootless() tempdir, err = CreateTempDirInTempDir() if err != nil { os.Exit(1) @@ -180,6 +179,7 @@ var _ = Describe("Podman network create", func() { It("podman network create with name and IPv6 subnet", func() { SkipIfRemote() + SkipIfRootless() var ( results []network.NcList ) diff --git a/test/e2e/network_test.go b/test/e2e/network_test.go index f427afa67..c35b82fc1 100644 --- a/test/e2e/network_test.go +++ b/test/e2e/network_test.go @@ -1,5 +1,3 @@ -// +build !remote - package integration import ( @@ -9,6 +7,7 @@ import ( "path/filepath" "strings" + "github.com/containers/podman/v2/pkg/rootless" . "github.com/containers/podman/v2/test/utils" "github.com/containers/storage/pkg/stringid" . "github.com/onsi/ginkgo" @@ -34,7 +33,6 @@ var _ = Describe("Podman network", func() { ) BeforeEach(func() { - SkipIfRootless() tempdir, err = CreateTempDirInTempDir() if err != nil { os.Exit(1) @@ -76,13 +74,12 @@ var _ = Describe("Podman network", func() { } ] }` - cniPath = "/etc/cni/net.d" ) It("podman network list", func() { // Setup, use uuid to prevent conflict with other tests uuid := stringid.GenerateNonCryptoID() - secondPath := filepath.Join(cniPath, fmt.Sprintf("%s.conflist", uuid)) + secondPath := filepath.Join(podmanTest.CNIConfigDir, fmt.Sprintf("%s.conflist", uuid)) writeConf([]byte(secondConf), secondPath) defer removeConf(secondPath) @@ -95,7 +92,7 @@ var _ = Describe("Podman network", func() { It("podman network list -q", func() { // Setup, use uuid to prevent conflict with other tests uuid := stringid.GenerateNonCryptoID() - secondPath := filepath.Join(cniPath, fmt.Sprintf("%s.conflist", uuid)) + secondPath := filepath.Join(podmanTest.CNIConfigDir, fmt.Sprintf("%s.conflist", uuid)) writeConf([]byte(secondConf), secondPath) defer removeConf(secondPath) @@ -108,7 +105,7 @@ var _ = Describe("Podman network", func() { It("podman network list --filter success", func() { // Setup, use uuid to prevent conflict with other tests uuid := stringid.GenerateNonCryptoID() - secondPath := filepath.Join(cniPath, fmt.Sprintf("%s.conflist", uuid)) + secondPath := filepath.Join(podmanTest.CNIConfigDir, fmt.Sprintf("%s.conflist", uuid)) writeConf([]byte(secondConf), secondPath) defer removeConf(secondPath) @@ -121,7 +118,7 @@ var _ = Describe("Podman network", func() { It("podman network list --filter failure", func() { // Setup, use uuid to prevent conflict with other tests uuid := stringid.GenerateNonCryptoID() - secondPath := filepath.Join(cniPath, fmt.Sprintf("%s.conflist", uuid)) + secondPath := filepath.Join(podmanTest.CNIConfigDir, fmt.Sprintf("%s.conflist", uuid)) writeConf([]byte(secondConf), secondPath) defer removeConf(secondPath) @@ -140,7 +137,7 @@ var _ = Describe("Podman network", func() { It("podman network rm", func() { // Setup, use uuid to prevent conflict with other tests uuid := stringid.GenerateNonCryptoID() - secondPath := filepath.Join(cniPath, fmt.Sprintf("%s.conflist", uuid)) + secondPath := filepath.Join(podmanTest.CNIConfigDir, fmt.Sprintf("%s.conflist", uuid)) writeConf([]byte(secondConf), secondPath) defer removeConf(secondPath) @@ -168,11 +165,16 @@ var _ = Describe("Podman network", func() { It("podman network inspect", func() { // Setup, use uuid to prevent conflict with other tests uuid := stringid.GenerateNonCryptoID() - secondPath := filepath.Join(cniPath, fmt.Sprintf("%s.conflist", uuid)) + secondPath := filepath.Join(podmanTest.CNIConfigDir, fmt.Sprintf("%s.conflist", uuid)) writeConf([]byte(secondConf), secondPath) defer removeConf(secondPath) - session := podmanTest.Podman([]string{"network", "inspect", "podman-integrationtest", "podman"}) + expectedNetworks := []string{"podman-integrationtest"} + if !rootless.IsRootless() { + // rootful image contains "podman/cni/87-podman-bridge.conflist" for "podman" network + expectedNetworks = append(expectedNetworks, "podman") + } + session := podmanTest.Podman(append([]string{"network", "inspect"}, expectedNetworks...)) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) Expect(session.IsJSONOutputValid()).To(BeTrue()) @@ -181,7 +183,7 @@ var _ = Describe("Podman network", func() { It("podman network inspect", func() { // Setup, use uuid to prevent conflict with other tests uuid := stringid.GenerateNonCryptoID() - secondPath := filepath.Join(cniPath, fmt.Sprintf("%s.conflist", uuid)) + secondPath := filepath.Join(podmanTest.CNIConfigDir, fmt.Sprintf("%s.conflist", uuid)) writeConf([]byte(secondConf), secondPath) defer removeConf(secondPath) diff --git a/test/system/055-rm.bats b/test/system/055-rm.bats index 478ba0f20..c8475c3e9 100644 --- a/test/system/055-rm.bats +++ b/test/system/055-rm.bats @@ -44,8 +44,6 @@ load helpers # # See https://github.com/containers/podman/issues/3795 @test "podman rm -f" { - skip_if_remote "FIXME: pending #7117" - rand=$(random_string 30) ( sleep 3; run_podman rm -f $rand ) & run_podman 137 run --name $rand $IMAGE sleep 30 diff --git a/vendor/modules.txt b/vendor/modules.txt index fbe35e527..7af0f1110 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -701,7 +701,7 @@ gopkg.in/yaml.v3 # k8s.io/api v0.0.0-20190620084959-7cf5895f2711 k8s.io/api/apps/v1 k8s.io/api/core/v1 -# k8s.io/apimachinery v0.19.0 +# k8s.io/apimachinery v0.19.1 k8s.io/apimachinery/pkg/api/errors k8s.io/apimachinery/pkg/api/resource k8s.io/apimachinery/pkg/apis/meta/v1 |