diff options
82 files changed, 781 insertions, 2382 deletions
diff --git a/.cirrus.yml b/.cirrus.yml index 67c212c15..9d220c69a 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -46,16 +46,6 @@ env: PRIOR_UBUNTU_CACHE_IMAGE_NAME: "${PRIOR_UBUNTU_NAME}-${_BUILT_IMAGE_SUFFIX}" #### - #### Variables for composing new cache-images (used in PR testing) from - #### base-images (pre-existing in GCE) - #### - BUILT_IMAGE_SUFFIX: "-${CIRRUS_REPO_NAME}-${CIRRUS_BUILD_ID}" - # Special image w/ nested-libvirt + tools for creating new cache and base images - IMAGE_BUILDER_CACHE_IMAGE_NAME: "image-builder-image-1541772081" - # Name where this repositories VM images are stored - GCP_PROJECT_ID: libpod-218412 - - #### #### Default to NOT operating in any special-case testing mode #### SPECIALMODE: "none" # don't do anything special @@ -66,8 +56,8 @@ env: #### #### Credentials and other secret-sauces, decrypted at runtime when authorized. #### - # Freenode IRC credentials for posting status messages - IRCID: ENCRYPTED[0c4a3cc4ecda08bc47cd3d31592be8ae5c2bd0151bf3def00a9afd139ef1ab23a1bd0523319d076c027f9749ddb1f3c8] + # Name where this repositories VM images are stored + GCP_PROJECT_ID: libpod-218412 # Service-account client_email - needed to build images SERVICE_ACCOUNT: ENCRYPTED[702a8e07e27a6faf7988fcddcc068c2ef2bb182a5aa671f5ccb7fbbfb891c823aa4a7856fb17240766845dbd68bd3f90] # Service account username part of client_email - for ssh'ing into VMs @@ -138,38 +128,24 @@ gating_task: # Verify some aspects of ci/related scripts ci_script: - '${GOSRC}/${SCRIPT_BASE}/lib.sh.t |& ${TIMESTAMP}' - - '/usr/local/bin/entrypoint.sh -C ${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/packer test' - '${GOSRC}/${SCRIPT_BASE}/cirrus_yaml_test.py |& ${TIMESTAMP}' # Verify expected bash environment (-o pipefail) pipefail_enabledscript: 'if /bin/false | /bin/true; then echo "pipefail fault" && exit 72; fi' - on_failure: - failed_branch_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_branch_failure.sh' - # Ensure these container images can build container_image_build_task: alias: 'container_image_build' + name: "build gate image $DEST_BRANCH branch" depends_on: - "gating" # Only run for PRs, quay.io will automatically build after branch-push only_if: $CIRRUS_BRANCH != $DEST_BRANCH - matrix: - - name: "build in_podman image ${FEDORA_NAME} " - container: - dockerfile: Dockerfile - - name: "build in_podman image ${UBUNTU_NAME}" - container: - dockerfile: Dockerfile.ubuntu - - name: "build gate image $DEST_BRANCH branch" - container: - dockerfile: contrib/gate/Dockerfile - container: - dockerfile: Dockerfile + dockerfile: contrib/gate/Dockerfile script: make install.remote @@ -179,7 +155,6 @@ container_image_build_task: rpmbuild_task: only_if: >- - $CIRRUS_CHANGE_TITLE !=~ '.*CI:IMG.*' && $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*' && $CIRRUS_BRANCH != $DEST_BRANCH @@ -197,18 +172,13 @@ rpmbuild_task: - 'make -C ${CIRRUS_WORKING_DIR} -f ${CIRRUS_WORKING_DIR}/.copr/Makefile' - 'rpmbuild --rebuild ${CIRRUS_WORKING_DIR}/podman-*.src.rpm' - on_failure: - failed_branch_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_branch_failure.sh |& ${TIMESTAMP}' - # This task runs `make vendor` followed by ./hack/tree_status.sh to check # whether the git tree is clean. The reasoning for that is to make sure # that the vendor.conf, the code and the vendored packages in ./vendor are # in sync at all times. vendor_task: - only_if: >- - $CIRRUS_CHANGE_TITLE !=~ '.*CI:IMG.*' && - $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*' + only_if: $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*' depends_on: - "gating" @@ -230,17 +200,12 @@ vendor_task: - 'cd ${CIRRUS_WORKING_DIR} && make vendor' - 'cd ${CIRRUS_WORKING_DIR} && ./hack/tree_status.sh' - on_failure: - failed_branch_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_branch_failure.sh |& ${TIMESTAMP}' - # This task runs `make varlink_api_generate` followed by ./hack/tree_status.sh to check # whether the git tree is clean. varlink_api_task: - only_if: >- - $CIRRUS_CHANGE_TITLE !=~ '.*CI:IMG.*' && - $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*' + only_if: $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*' depends_on: - "gating" @@ -265,9 +230,6 @@ varlink_api_task: - '/usr/local/bin/entrypoint.sh BUILDTAGS="varlink" varlink_api_generate |& ${TIMESTAMP}' - 'cd ${GOSRC} && ./hack/tree_status.sh |& ${TIMESTAMP}' - on_failure: - failed_branch_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_branch_failure.sh' - build_each_commit_task: @@ -278,7 +240,6 @@ build_each_commit_task: only_if: >- $CIRRUS_BRANCH != $DEST_BRANCH && - $CIRRUS_CHANGE_TITLE !=~ '.*CI:IMG.*' && $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*' gce_instance: @@ -297,9 +258,6 @@ build_each_commit_task: - 'git fetch --depth 50 origin $DEST_BRANCH |& ${TIMESTAMP}' - 'make build-all-new-commits GIT_BASE_BRANCH=origin/$DEST_BRANCH |& ${TIMESTAMP}' - on_failure: - failed_branch_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_branch_failure.sh' - build_without_cgo_task: @@ -310,7 +268,6 @@ build_without_cgo_task: only_if: >- $CIRRUS_BRANCH != $DEST_BRANCH && - $CIRRUS_CHANGE_TITLE !=~ '.*CI:IMG.*' && $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*' gce_instance: @@ -327,9 +284,6 @@ build_without_cgo_task: - 'source $SCRIPT_BASE/lib.sh' - 'make build-no-cgo' - on_failure: - failed_branch_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_branch_failure.sh' - # Update metadata on VM images referenced by this repository state meta_task: @@ -360,32 +314,6 @@ meta_task: script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/update_meta.sh |& ${TIMESTAMP}' -# Remove old and disused images based on labels set by meta_task -image_prune_task: - - # This should ONLY ever run from the master branch, and never - # anywhere else so it's behavior is always consistent, even - # as new branches are created. - only_if: $CIRRUS_BRANCH == "master" - - depends_on: - - "meta" - - container: - image: "quay.io/libpod/imgprune:master" # see contrib/imgprune - cpu: 1 - memory: 1 - - env: - <<: *meta_env_vars - GCPJSON: ENCRYPTED[766916fedf780cbc16ac3152f7f73c5d9dcf64768fc6e80b0858c5badd31e7b41f3c864405c814189fd340e5a056ba18] - GCPNAME: ENCRYPTED[d6869741209b8cf380adb8a3858cbce4542c9cf115452fcd2024a176b08fce10112e8bf0fbcc2f0033e7b87ef4342b3a] - - timeout_in: 10m - - script: '/usr/local/bin/entrypoint.sh |& ${TIMESTAMP}' - - # This task does the unit and integration testing for every platform testing_task: @@ -399,9 +327,7 @@ testing_task: - "container_image_build" # Only test build cache-images, if that's what's requested - only_if: >- - $CIRRUS_CHANGE_TITLE !=~ '.*CI:IMG.*' && - $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*' + only_if: $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*' matrix: - name: "test ${FEDORA_NAME}" @@ -435,9 +361,6 @@ testing_task: path: "*.tar.gz" type: "application/x-tar" - on_failure: - failed_branch_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_branch_failure.sh' - always: &standardlogs package_versions_script: '$SCRIPT_BASE/logcollector.sh packages' ginkgo_node_logs_script: '$SCRIPT_BASE/logcollector.sh ginkgo' @@ -460,9 +383,7 @@ special_testing_rootless_task: - "build_each_commit" - "build_without_cgo" - only_if: >- - $CIRRUS_CHANGE_TITLE !=~ '.*CI:IMG.*' && - $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*' + only_if: $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*' env: ADD_SECOND_PARTITION: 'true' @@ -477,9 +398,6 @@ special_testing_rootless_task: system_test_script: '$SCRIPT_BASE/system_test.sh |& ${TIMESTAMP} | ${LOGFORMAT} system_test' apiv2_test_script: '$SCRIPT_BASE/apiv2_test.sh |& ${TIMESTAMP} | ${LOGFORMAT} apiv2_test' - on_failure: - failed_branch_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_branch_failure.sh' - always: <<: *standardlogs @@ -494,9 +412,7 @@ special_testing_in_podman_task: - "build_each_commit" - "build_without_cgo" - only_if: >- - $CIRRUS_CHANGE_TITLE !=~ '.*CI:IMG.*' && - $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*' + only_if: $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*' matrix: - name: "in-podman ${PRIOR_FEDORA_NAME}" @@ -515,9 +431,6 @@ special_testing_in_podman_task: setup_environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}' integration_test_script: '$SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP} | ${LOGFORMAT} integration_test' - on_failure: - failed_branch_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_branch_failure.sh' - always: <<: *standardlogs @@ -530,9 +443,7 @@ special_testing_cross_task: - "varlink_api" - "vendor" - only_if: >- - $CIRRUS_CHANGE_TITLE !=~ '.*CI:IMG.*' && - $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*' + only_if: $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*' matrix: - name: 'cross-platform: windows' @@ -548,9 +459,6 @@ special_testing_cross_task: setup_environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}' build_release_script: '$SCRIPT_BASE/build_release.sh |& ${TIMESTAMP}' - on_failure: - failed_branch_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_branch_failure.sh' - # When examining a particular run, provide convenient access to release files. zip_artifacts: path: "*.zip" @@ -568,9 +476,7 @@ special_testing_bindings_task: - "varlink_api" - "vendor" - only_if: >- - $CIRRUS_CHANGE_TITLE !=~ '.*CI:IMG.*' && - $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*' + only_if: $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*' env: SPECIALMODE: 'bindings' # See docs @@ -581,9 +487,6 @@ special_testing_bindings_task: setup_environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}' integration_test_script: '$SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP} | ${LOGFORMAT} integration_test' - on_failure: - failed_branch_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_branch_failure.sh' - always: <<: *standardlogs @@ -595,9 +498,7 @@ special_testing_endpoint_task: - "varlink_api" - "vendor" - only_if: >- - $CIRRUS_CHANGE_TITLE !=~ '.*CI:IMG.*' && - $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*' + only_if: $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*' env: SPECIALMODE: 'endpoint' # See docs @@ -606,96 +507,12 @@ special_testing_endpoint_task: setup_environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}' integration_test_script: '$SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP} | ${LOGFORMAT} integration_test' - - on_failure: - failed_branch_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_branch_failure.sh' - - always: - <<: *standardlogs - - -# Test building of new cache-images for future PR testing, in this PR. -test_build_cache_images_task: - - only_if: >- - $CIRRUS_BRANCH != $DEST_BRANCH && - $CIRRUS_CHANGE_TITLE =~ '.*CI:IMG.*' && - $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*' - - depends_on: - - "gating" - - 'container_image_build' - - # VMs created by packer are not cleaned up by cirrus, must allow task to complete - auto_cancellation: $CI != "true" - - gce_instance: - image_project: $GCP_PROJECT_ID - zone: "us-central1-a" - cpu: 4 - memory: "4Gb" - disk: 200 - image_name: "${IMAGE_BUILDER_CACHE_IMAGE_NAME}" - scopes: # required for image building - - compute - - devstorage.full_control - - networking_script: '${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/networking.sh' - build_vm_images_script: '$SCRIPT_BASE/build_vm_images.sh |& ${TIMESTAMP}' - - on_failure: - failed_df_script: '${DFCMD}' - failed_journalctl_b_script: 'journalctl -b || echo "Uh oh, journalctl -b failed"' - - -# Test building of new cache-images for future PR testing, in this PR. -verify_test_built_images_task: - - only_if: >- - $CIRRUS_BRANCH != $DEST_BRANCH && - $CIRRUS_CHANGE_TITLE =~ '.*CI:IMG.*' && - $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*' - - - depends_on: - - "gating" - - "test_build_cache_images" - - gce_instance: - # Images generated by test_build_cache_images_task (above) - image_name: "${PACKER_BUILDER_NAME}${BUILT_IMAGE_SUFFIX}" - - env: - ADD_SECOND_PARTITION: 'true' - matrix: - - RCLI: 'true' - - RCLI: 'false' - matrix: - PACKER_BUILDER_NAME: "${FEDORA_NAME}" - PACKER_BUILDER_NAME: "${PRIOR_FEDORA_NAME}" - PACKER_BUILDER_NAME: "${UBUNTU_NAME}" - PACKER_BUILDER_NAME: "${PRIOR_UBUNTU_NAME}" - - networking_script: '${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/networking.sh' - installed_packages_script: '$SCRIPT_BASE/logcollector.sh packages' - environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}' - # Verify expectations of built images - check_image_script: '$SCRIPT_BASE/check_image.sh |& ${TIMESTAMP}' - # Note: A truncated form of normal testing. It only needs to confirm new images - # "probably" work. A full round of testing will happen again after $*_CACHE_IMAGE_NAME - # are updated in this or another PR (w/o '***CIRRUS: TEST IMAGES***'). - integration_test_script: '$SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP}' - system_test_script: '$SCRIPT_BASE/system_test.sh |& ${TIMESTAMP}' - always: <<: *standardlogs docs_task: - # Don't run this when building/testing new VM images - only_if: $CIRRUS_CHANGE_TITLE !=~ '.*CI:IMG.*' - depends_on: - "gating" env: @@ -724,7 +541,6 @@ success_task: - "build_without_cgo" - "container_image_build" - "meta" - - "image_prune" - "testing" - "rpmbuild" - "special_testing_rootless" @@ -732,8 +548,6 @@ success_task: - "special_testing_cross" - "special_testing_endpoint" - "special_testing_bindings" - - "test_build_cache_images" - - "verify_test_built_images" - "docs" - "static_build" - "darwin_build" @@ -749,7 +563,7 @@ success_task: cpu: 1 memory: 1 - success_script: '/usr/local/bin/entrypoint.sh ./$SCRIPT_BASE/success.sh |& ${TIMESTAMP}' + success_script: /bin/true # Build the static binary static_build_task: diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index c5120e440..000000000 --- a/Dockerfile +++ /dev/null @@ -1,21 +0,0 @@ -FROM registry.fedoraproject.org/fedora:latest - -# This container image is utilized by the containers CI automation system -# for building and testing libpod inside a container environment. -# It is assumed that the source to be tested will overwrite $GOSRC (below) -# at runtime. -ENV GOPATH=/var/tmp/go -ENV GOSRC=$GOPATH/src/github.com/containers/podman -ENV SCRIPT_BASE=./contrib/cirrus -ENV PACKER_BASE=$SCRIPT_BASE/packer - -ADD / $GOSRC -WORKDIR $GOSRC - -# Re-use repositories and package setup as in VMs under CI -RUN bash $PACKER_BASE/fedora_packaging.sh && \ - dnf clean all && \ - rm -rf /var/cache/dnf - -# Mirror steps taken under CI -RUN bash -c 'source $GOSRC/$SCRIPT_BASE/lib.sh && install_test_configs' diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu deleted file mode 100644 index df4ebf3d4..000000000 --- a/Dockerfile.ubuntu +++ /dev/null @@ -1,28 +0,0 @@ -# Must resemble $UBUNTU_BASE_IMAGE in ./contrib/cirrus/lib.sh -FROM ubuntu:20.04 - -# This container image is intended for building and testing libpod -# from inside a container environment. It is assumed that the source -# to be tested will overwrite $GOSRC (below) at runtime. -ENV GOPATH=/var/tmp/go -ENV GOSRC=$GOPATH/src/github.com/containers/podman -ENV SCRIPT_BASE=./contrib/cirrus -ENV PACKER_BASE=$SCRIPT_BASE/packer - -RUN export DEBIAN_FRONTEND="noninteractive" && \ - apt-get -qq update --yes && \ - apt-get -qq upgrade --yes && \ - apt-get -qq install curl git && \ - apt-get -qq autoremove --yes && \ - rm -rf /var/cache/apt - -ADD / $GOSRC -WORKDIR $GOSRC - -# Re-use repositories and package setup as in VMs under CI -RUN bash $PACKER_BASE/ubuntu_packaging.sh && \ - apt-get -qq autoremove --yes && \ - rm -rf /var/cache/apt - -# Mirror steps taken under CI -RUN bash -c 'source $GOSRC/$SCRIPT_BASE/lib.sh && install_test_configs' @@ -177,8 +177,8 @@ familiar container cli commands. For more details, see the [Container Tools Guide](https://github.com/containers/buildah/tree/master/docs/containertools). ## Podman Legacy API (Varlink) -Podman offers a Varlink-based API for remote management of containers. -However, this API has been deprecated by the REST API. +Podman offers a [Varlink-based API](https://github.com/containers/podman/blob/master/docs/tutorials/varlink_remote_client.md) +for remote management of containers. However, this API has been deprecated by the REST API. Varlink support is in maintenance mode, and will be removed in a future release. For more details, you can see [this blog](https://podman.io/blogs/2020/01/17/podman-new-api.html). diff --git a/cmd/podman/containers/ps.go b/cmd/podman/containers/ps.go index ebb6ed98f..2aa3b3a9b 100644 --- a/cmd/podman/containers/ps.go +++ b/cmd/podman/containers/ps.go @@ -13,6 +13,7 @@ import ( tm "github.com/buger/goterm" "github.com/containers/buildah/pkg/formats" "github.com/containers/podman/v2/cmd/podman/registry" + "github.com/containers/podman/v2/cmd/podman/utils" "github.com/containers/podman/v2/cmd/podman/validate" "github.com/containers/podman/v2/pkg/domain/entities" "github.com/cri-o/ocicni/pkg/ocicni" @@ -56,9 +57,9 @@ func init() { func listFlagSet(flags *pflag.FlagSet) { flags.BoolVarP(&listOpts.All, "all", "a", false, "Show all the containers, default is only running containers") flags.StringSliceVarP(&filters, "filter", "f", []string{}, "Filter output based on conditions given") + flags.BoolVar(&listOpts.Storage, "storage", false, "Show containers in storage not controlled by Podman") flags.StringVar(&listOpts.Format, "format", "", "Pretty-print containers to JSON or using a Go template") flags.IntVarP(&listOpts.Last, "last", "n", -1, "Print the n last created containers (all states)") - flags.BoolVar(&listOpts.Namespace, "namespace", false, "Display namespace information") flags.BoolVar(&listOpts.Namespace, "ns", false, "Display namespace information") flags.BoolVar(&noTrunc, "no-trunc", false, "Display the extended information") flags.BoolVarP(&listOpts.Pod, "pod", "p", false, "Print the ID and name of the pod the containers are associated with") @@ -69,6 +70,7 @@ func listFlagSet(flags *pflag.FlagSet) { sort := validate.Value(&listOpts.Sort, "command", "created", "id", "image", "names", "runningfor", "size", "status") flags.Var(sort, "sort", "Sort output by: "+sort.Choices()) + flags.SetNormalizeFunc(utils.AliasFlags) } func checkFlags(c *cobra.Command) error { // latest, and last are mutually exclusive. @@ -102,6 +104,14 @@ func checkFlags(c *cobra.Command) error { if listOpts.Watch > 0 && listOpts.Latest { return errors.New("the watch and latest flags cannot be used together") } + cfg := registry.PodmanConfig() + if cfg.Engine.Namespace != "" { + if c.Flag("storage").Changed && listOpts.Storage { + return errors.New("--namespace and --storage flags can not both be set") + } + listOpts.Storage = false + } + return nil } diff --git a/cmd/podman/images/build.go b/cmd/podman/images/build.go index 400f960cc..923109b15 100644 --- a/cmd/podman/images/build.go +++ b/cmd/podman/images/build.go @@ -211,7 +211,16 @@ func build(cmd *cobra.Command, args []string) error { return err } - apiBuildOpts, err := buildFlagsWrapperToOptions(cmd, contextDir, &buildOpts) + var logfile *os.File + if cmd.Flag("logfile").Changed { + logfile, err = os.OpenFile(buildOpts.Logfile, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600) + if err != nil { + return errors.Errorf("error opening logfile %q: %v", buildOpts.Logfile, err) + } + defer logfile.Close() + } + + apiBuildOpts, err := buildFlagsWrapperToOptions(cmd, contextDir, &buildOpts, logfile) if err != nil { return err } @@ -225,7 +234,7 @@ func build(cmd *cobra.Command, args []string) error { // conversion here prevents the API from doing that (redundantly). // // TODO: this code should really be in Buildah. -func buildFlagsWrapperToOptions(c *cobra.Command, contextDir string, flags *buildFlagsWrapper) (*entities.BuildOptions, error) { +func buildFlagsWrapperToOptions(c *cobra.Command, contextDir string, flags *buildFlagsWrapper, logfile *os.File) (*entities.BuildOptions, error) { output := "" tags := []string{} if c.Flag("tag").Changed { @@ -284,16 +293,11 @@ func buildFlagsWrapperToOptions(c *cobra.Command, contextDir string, flags *buil stderr = os.Stderr reporter = os.Stderr - if c.Flag("logfile").Changed { - f, err := os.OpenFile(flags.Logfile, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600) - if err != nil { - return nil, errors.Errorf("error opening logfile %q: %v", flags.Logfile, err) - } - defer f.Close() - logrus.SetOutput(f) - stdout = f - stderr = f - reporter = f + if logfile != nil { + logrus.SetOutput(logfile) + stdout = logfile + stderr = logfile + reporter = logfile } var memoryLimit, memorySwap int64 diff --git a/cmd/podman/manifest/add.go b/cmd/podman/manifest/add.go index ca633263d..128bf66a7 100644 --- a/cmd/podman/manifest/add.go +++ b/cmd/podman/manifest/add.go @@ -4,14 +4,26 @@ import ( "context" "fmt" + "github.com/containers/common/pkg/auth" + "github.com/containers/image/v5/types" "github.com/containers/podman/v2/cmd/podman/registry" "github.com/containers/podman/v2/pkg/domain/entities" + "github.com/containers/podman/v2/pkg/util" "github.com/pkg/errors" "github.com/spf13/cobra" ) +// manifestAddOptsWrapper wraps entities.ManifestAddOptions and prevents leaking +// CLI-only fields into the API types. +type manifestAddOptsWrapper struct { + entities.ManifestAddOptions + + TLSVerifyCLI bool // CLI only + CredentialsCLI string +} + var ( - manifestAddOpts = entities.ManifestAddOptions{} + manifestAddOpts = manifestAddOptsWrapper{} addCmd = &cobra.Command{ Use: "add [flags] LIST LIST", Short: "Add images to a manifest list or image index", @@ -33,15 +45,48 @@ func init() { flags.BoolVar(&manifestAddOpts.All, "all", false, "add all of the list's images if the image is a list") flags.StringSliceVar(&manifestAddOpts.Annotation, "annotation", nil, "set an `annotation` for the specified image") flags.StringVar(&manifestAddOpts.Arch, "arch", "", "override the `architecture` of the specified image") + flags.StringVar(&manifestAddOpts.Authfile, "authfile", auth.GetDefaultAuthFile(), "path of the authentication file. Use REGISTRY_AUTH_FILE environment variable to override") + flags.StringVar(&manifestAddOpts.CertDir, "cert-dir", "", "use certificates at the specified path to access the registry") + flags.StringVar(&manifestAddOpts.CredentialsCLI, "creds", "", "use `[username[:password]]` for accessing the registry") + flags.StringSliceVar(&manifestAddOpts.Features, "features", nil, "override the `features` of the specified image") flags.StringVar(&manifestAddOpts.OS, "os", "", "override the `OS` of the specified image") flags.StringVar(&manifestAddOpts.OSVersion, "os-version", "", "override the OS `version` of the specified image") + flags.BoolVar(&manifestAddOpts.TLSVerifyCLI, "tls-verify", true, "require HTTPS and verify certificates when accessing the registry") flags.StringVar(&manifestAddOpts.Variant, "variant", "", "override the `Variant` of the specified image") + + if registry.IsRemote() { + _ = flags.MarkHidden("authfile") + _ = flags.MarkHidden("cert-dir") + _ = flags.MarkHidden("tls-verify") + } } func add(cmd *cobra.Command, args []string) error { + if err := auth.CheckAuthFile(manifestPushOpts.Authfile); err != nil { + return err + } + manifestAddOpts.Images = []string{args[1], args[0]} - listID, err := registry.ImageEngine().ManifestAdd(context.Background(), manifestAddOpts) + + if manifestAddOpts.CredentialsCLI != "" { + creds, err := util.ParseRegistryCreds(manifestAddOpts.CredentialsCLI) + if err != nil { + return err + } + manifestAddOpts.Username = creds.Username + manifestAddOpts.Password = creds.Password + } + + // TLS verification in c/image is controlled via a `types.OptionalBool` + // which allows for distinguishing among set-true, set-false, unspecified + // which is important to implement a sane way of dealing with defaults of + // boolean CLI flags. + if cmd.Flags().Changed("tls-verify") { + manifestAddOpts.SkipTLSVerify = types.NewOptionalBool(!manifestAddOpts.TLSVerifyCLI) + } + + listID, err := registry.ImageEngine().ManifestAdd(context.Background(), manifestAddOpts.ManifestAddOptions) if err != nil { return errors.Wrapf(err, "error adding to manifest list %s", args[0]) } diff --git a/cmd/podman/utils/alias.go b/cmd/podman/utils/alias.go index e484461c5..ff31e82ea 100644 --- a/cmd/podman/utils/alias.go +++ b/cmd/podman/utils/alias.go @@ -19,6 +19,8 @@ func AliasFlags(f *pflag.FlagSet, name string) pflag.NormalizedName { name = "network" case "timeout": name = "time" + case "namespace": + name = "ns" } return pflag.NormalizedName(name) } diff --git a/completions/bash/podman b/completions/bash/podman index 3b50af1a9..e8185235b 100644 --- a/completions/bash/podman +++ b/completions/bash/podman @@ -1846,6 +1846,9 @@ _podman_manifest() { _podman_manifest_add() { local options_with_args=" --annotation + --authfile + --cert-dir + --creds --arch --features --os @@ -1857,6 +1860,7 @@ _podman_manifest_add() { --all --help -h + --tls-verify " _complete_ "$options_with_args" "$boolean_options" @@ -2679,6 +2683,7 @@ _podman_ps() { --pod -p --quiet -q --size -s + --storage --namespace --ns --sync " diff --git a/contrib/cirrus/README.md b/contrib/cirrus/README.md index 977762293..f66560cc8 100644 --- a/contrib/cirrus/README.md +++ b/contrib/cirrus/README.md @@ -76,95 +76,6 @@ exercising cgroups v2 with Podman integration tests. Also depends on having `SPECIALMODE` set to 'cgroupv2` -### ``test_build_cache_images_task`` Task - -Modifying the contents of cache-images is tested by making changes to -one or more of the ``./contrib/cirrus/packer/*_setup.sh`` files. Then -in the PR description, add the magic string: ``[CI:IMG]`` - -***N/B: Steps below are performed by automation*** - -1. ``setup_environment.sh``: Same as for other tasks. - -2. ``build_vm_images.sh``: Utilize [the packer tool](http://packer.io/docs/) - to produce new VM images. Create a new VM from each base-image, connect - to them with ``ssh``, and perform the steps as defined by the - ``$PACKER_BASE/libpod_images.yml`` file: - - 1. On a base-image VM, as root, copy the current state of the repository - into ``/tmp/libpod``. - 2. Execute distribution-specific scripts to prepare the image for - use. For example, ``fedora_setup.sh``. - 3. If successful, shut down each VM and record the names, and dates - into a json manifest file. - 4. Move the manifest file, into a google storage bucket object. - This is a retained as a secondary method for tracking/auditing - creation of VM images, should it ever be needed. - -### ``verify_test_built_images`` Task - -Only runs following successful ``test_build_cache_images_task`` task. Uses -images following the standard naming format; ***however, only runs a limited -sub-set of automated tests***. Validating newly built images fully, requires -updating ``.cirrus.yml``. - -***N/B: Steps below are performed by automation*** - -1. Using the just build VM images, launch VMs and wait for them to boot. - -2. Execute the `setup_environment.sh` as in the `testing` task. - -2. Execute the `integration_test.sh` as in the `testing` task. - - -***Manual Steps:*** Assuming the automated steps pass, then -you'll find the new image names displayed at the end of the -`test_build_cache_images`. For example: - - -``` -...cut... - -[+0747s] ==> Builds finished. The artifacts of successful builds are: -[+0747s] --> ubuntu-18: A disk image was created: ubuntu-18-libpod-5664838702858240 -[+0747s] --> fedora-29: A disk image was created: fedora-29-libpod-5664838702858240 -[+0747s] --> fedora-30: A disk image was created: fedora-30-libpod-5664838702858240 -[+0747s] --> ubuntu-19: A disk image was created: ubuntu-19-libpod-5664838702858240 -``` - -Notice the suffix on all the image names comes from the env. var. set in -*.cirrus.yml*: `BUILT_IMAGE_SUFFIX: "-${CIRRUS_REPO_NAME}-${CIRRUS_BUILD_ID}"`. -Edit `.cirrus.yml`, in the top-level `env` section, update the suffix variable -used at runtime to launch VMs for testing: - - -```yaml -env: - ...cut... - #### - #### Cache-image names to test with (double-quotes around names are critical) - ### - _BUILT_IMAGE_SUFFIX: "libpod-5664838702858240" - FEDORA_CACHE_IMAGE_NAME: "fedora-30-${_BUILT_IMAGE_SUFFIX}" - PRIOR_FEDORA_CACHE_IMAGE_NAME: "fedora-29-${_BUILT_IMAGE_SUFFIX}" - ...cut... -``` - -***NOTES:*** -* If re-using the same PR with new images in `.cirrus.yml`, - take care to also *update the PR description* to remove - the magic ``[CI:IMG]`` string. Keeping it and - `--force` pushing would needlessly cause Cirrus-CI to build - and test images again. -* In the future, if you need to review the log from the build that produced - the referenced image: - - * Note the Build ID from the image name (for example `5664838702858240`). - * Go to that build in the Cirrus-CI WebUI, using the build ID in the URL. - (For example `https://cirrus-ci.com/build/5664838702858240`. - * Choose the *test_build_cache_images* task. - * Open the *build_vm_images* script section. - ### `docs` Task Builds swagger API documentation YAML and uploads to google storage (an online @@ -226,99 +137,6 @@ gsutil cors set /path/to/file.json gs://libpod-master-releases file. Therefore, if it is not functioning or misconfigured, a person must have altered it or changes were made to the referring site (e.g. `docs.podman.io`). -## Base-images - -Base-images are VM disk-images specially prepared for executing as GCE VMs. -In particular, they run services on startup similar in purpose/function -as the standard 'cloud-init' services. - -* The google services are required for full support of ssh-key management - and GCE OAuth capabilities. Google provides native images in GCE - with services pre-installed, for many platforms. For example, - RHEL, CentOS, and Ubuntu. - -* Google does ***not*** provide any images for Fedora (as of 5/2019), nor do - they provide a base-image prepared to run packer for creating other images - in the ``test_build_vm_images`` Task (above). - -* Base images do not need to be produced often, but doing so completely - manually would be time-consuming and error-prone. Therefore a special - semi-automatic *Makefile* target is provided to assist with producing - all the base-images: ``libpod_base_images`` - -To produce new base-images, including an `image-builder-image` (used by -the ``cache_images`` Task) some input parameters are required: - -* ``GCP_PROJECT_ID``: The complete GCP project ID string e.g. foobar-12345 - identifying where the images will be stored. - -* ``GOOGLE_APPLICATION_CREDENTIALS``: A *JSON* file containing - credentials for a GCE service account. This can be [a service - account](https://cloud.google.com/docs/authentication/production#obtaining_and_providing_service_account_credentials_manually) - or [end-user - credentials](https://cloud.google.com/docs/authentication/end-user#creating_your_client_credentials) - -* Optionally, CSV's may be specified to ``PACKER_BUILDS`` - to limit the base-images produced. For example, - ``PACKER_BUILDS=fedora,image-builder-image``. - -If there is no existing 'image-builder-image' within GCE, a new -one may be bootstrapped by creating a CentOS 7 VM with support for -nested-virtualization, and with elevated cloud privileges (to access -GCE, from within the GCE VM). For example: - -``` -$ alias pgcloud='sudo podman run -it --rm -e AS_ID=$UID - -e AS_USER=$USER -v $HOME:$HOME:z quay.io/cevich/gcloud_centos:latest' - -$ URL=https://www.googleapis.com/auth -$ SCOPES=$URL/userinfo.email,$URL/compute,$URL/devstorage.full_control - -# The --min-cpu-platform is critical for nested-virt. -$ pgcloud compute instances create $USER-image-builder \ - --image-family centos-7 \ - --boot-disk-size "200GB" \ - --min-cpu-platform "Intel Haswell" \ - --machine-type n1-standard-2 \ - --scopes $SCOPES -``` - -Then from that VM, execute the -``contrib/cirrus/packer/image-builder-image_base_setup.sh`` script. -Shutdown the VM, and convert it into a new image-builder-image. - -Building new base images is done by first creating a VM from an -image-builder-image and copying the credentials json file to it. - -``` -$ hack/get_ci_vm.sh image-builder-image-1541772081 -...in another terminal... -$ pgcloud compute scp /path/to/gac.json $USER-image-builder-image-1541772081:. -``` - -Then, on the VM, change to the ``packer`` sub-directory, and build the images: - -``` -$ cd libpod/contrib/cirrus/packer -$ make libpod_base_images GCP_PROJECT_ID=<VALUE> \ - GOOGLE_APPLICATION_CREDENTIALS=/path/to/gac.json \ - PACKER_BUILDS=<OPTIONAL> -``` - -Assuming this is successful (hence the semi-automatic part), packer will -produce a ``packer-manifest.json`` output file. This contains the base-image -names suitable for updating in ``.cirrus.yml``, `env` keys ``*_BASE_IMAGE``. - -On failure, it should be possible to determine the problem from the packer -output. Sometimes that means setting `PACKER_LOG=1` and troubleshooting -the nested virt calls. It's also possible to observe the (nested) qemu-kvm -console output. Simply set the ``TTYDEV`` parameter, for example: - -``` -$ make libpod_base_images ... TTYDEV=$(tty) - ... -``` - ## `$SPECIALMODE` Some tasks alter their behavior based on this value. A summary of supported diff --git a/contrib/cirrus/add_second_partition.sh b/contrib/cirrus/add_second_partition.sh index 3c2f9f056..d0407be86 100644 --- a/contrib/cirrus/add_second_partition.sh +++ b/contrib/cirrus/add_second_partition.sh @@ -7,8 +7,7 @@ SLASH_DEVICE="/dev/sda" # Always the case on GCP # The unallocated space results from the difference in disk-size between VM Image -# and runtime request. The check_image.sh test includes a minimum-space check, -# with the Image size set initially lower by contrib/cirrus/packer/libpod_images.yml +# and runtime request. NEW_PART_START="50%" NEW_PART_END="100%" diff --git a/contrib/cirrus/build_vm_images.sh b/contrib/cirrus/build_vm_images.sh deleted file mode 100755 index be1c82185..000000000 --- a/contrib/cirrus/build_vm_images.sh +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env bash - -set -e -source $(dirname $0)/lib.sh - -BASE_IMAGE_VARS='FEDORA_BASE_IMAGE PRIOR_FEDORA_BASE_IMAGE UBUNTU_BASE_IMAGE PRIOR_UBUNTU_BASE_IMAGE' -ENV_VARS="PACKER_BUILDS BUILT_IMAGE_SUFFIX $BASE_IMAGE_VARS SERVICE_ACCOUNT GCE_SSH_USERNAME GCP_PROJECT_ID PACKER_VER SCRIPT_BASE PACKER_BASE CIRRUS_BUILD_ID CIRRUS_CHANGE_IN_REPO" -req_env_var $ENV_VARS -# Must also be made available through make, into packer process -export $ENV_VARS - -# Everything here is running on the 'image-builder-image' GCE image -# Assume basic dependencies are all met, but there could be a newer version -# of the packer binary -PACKER_FILENAME="packer_${PACKER_VER}_linux_amd64.zip" -if [[ -d "$HOME/packer" ]] -then - cd "$HOME/packer" - # image_builder_image has packer pre-installed, check if same version requested - if [[ -r "$PACKER_FILENAME" ]] - then - cp $PACKER_FILENAME "$GOSRC/$PACKER_BASE/" - cp packer "$GOSRC/$PACKER_BASE/" - fi -fi - -cd "$GOSRC/$PACKER_BASE" -# Add/update labels on base-images used in this build to prevent premature deletion -ARGS=" -" -for base_image_var in $BASE_IMAGE_VARS -do - # See entrypoint.sh in contrib/imgts and contrib/imgprune - # These updates can take a while, run them in the background, check later - gcloud compute images update \ - --update-labels=last-used=$(date +%s) \ - --update-labels=build-id=$CIRRUS_BUILD_ID \ - --update-labels=repo-ref=$CIRRUS_CHANGE_IN_REPO \ - --update-labels=project=$GCP_PROJECT_ID \ - ${!base_image_var} & -done - -make libpod_images \ - PACKER_BUILDS=$PACKER_BUILDS \ - PACKER_VER=$PACKER_VER \ - GOSRC=$GOSRC \ - SCRIPT_BASE=$SCRIPT_BASE \ - PACKER_BASE=$PACKER_BASE \ - BUILT_IMAGE_SUFFIX=$BUILT_IMAGE_SUFFIX - -# Separate PR-produced images from those produced on master. -if [[ "${CIRRUS_BRANCH:-}" == "master" ]] -then - POST_MERGE_BUCKET_SUFFIX="-master" -else - POST_MERGE_BUCKET_SUFFIX="" -fi - -# When successful, upload manifest of produced images using a filename unique -# to this build. -URI="gs://packer-import${POST_MERGE_BUCKET_SUFFIX}/manifest${BUILT_IMAGE_SUFFIX}.json" -gsutil cp packer-manifest.json "$URI" - -# Ensure any background 'gcloud compute images update' processes finish -wait # No -n option in CentOS, this is the best that can be done :( - -echo "Finished. A JSON manifest of produced images is available at $URI" diff --git a/contrib/cirrus/check_image.sh b/contrib/cirrus/check_image.sh deleted file mode 100755 index 04867ca64..000000000 --- a/contrib/cirrus/check_image.sh +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env bash - -set -eo pipefail - -source $(dirname $0)/lib.sh - -EVIL_UNITS="$($CIRRUS_WORKING_DIR/$PACKER_BASE/systemd_banish.sh --list)" - -req_env_var PACKER_BUILDER_NAME RCLI EVIL_UNITS OS_RELEASE_ID CG_FS_TYPE - -NFAILS=0 -echo "Validating VM image" - -MIN_SLASH_GIGS=30 -read SLASH_DEVICE SLASH_FSTYPE SLASH_SIZE JUNK <<<$(findmnt --df --first-only --noheadings / | cut -d '.' -f 1) -SLASH_SIZE_GIGS=$(echo "$SLASH_SIZE" | sed -r -e 's/G|g//') -item_test "Minimum available disk space" $SLASH_SIZE_GIGS -gt $MIN_SLASH_GIGS || let "NFAILS+=1" - -MIN_MEM_MB=2000 -read JUNK TOTAL USED MEM_FREE JUNK <<<$(free -tm | tail -1) -item_test 'Minimum available memory' $MEM_FREE -ge $MIN_MEM_MB || let "NFAILS+=1" - -# We're testing a custom-built podman; make sure there isn't a distro-provided -# binary anywhere; that could potentially taint our results. -remove_packaged_podman_files -item_test "remove_packaged_podman_files() does it's job" -z "$(type -P podman)" || let "NFAILS+=1" - -MIN_ZIP_VER='3.0' -VER_RE='.+([[:digit:]]+\.[[:digit:]]+).+' -ACTUAL_VER=$(zip --version 2>&1 | egrep -m 1 "Zip$VER_RE" | sed -r -e "s/$VER_RE/\\1/") -item_test "minimum zip version" "$MIN_ZIP_VER" = $(echo -e "$MIN_ZIP_VER\n$ACTUAL_VER" | sort -V | head -1) || let "NFAILS+=1" - -for REQ_UNIT in google-accounts-daemon.service \ - google-clock-skew-daemon.service \ - google-instance-setup.service \ - google-network-daemon.service \ - google-shutdown-scripts.service \ - google-startup-scripts.service -do - # enabled/disabled appears at the end of the line, on some Ubuntu's it appears twice - service_status=$(systemctl list-unit-files --no-legend $REQ_UNIT | tac -s ' ' | head -1) - item_test "required $REQ_UNIT status is enabled" \ - "$service_status" = "enabled" || let "NFAILS+=1" -done - -for evil_unit in $EVIL_UNITS -do - # Exits zero if any unit matching pattern is running - unit_status=$(systemctl is-active $evil_unit &> /dev/null; echo $?) - item_test "No $evil_unit unit is present or active:" "$unit_status" -ne "0" || let "NFAILS+=1" -done - -echo "Checking items specific to ${PACKER_BUILDER_NAME}${BUILT_IMAGE_SUFFIX}" -case "$PACKER_BUILDER_NAME" in - ubuntu*) - item_test "On ubuntu, no periodic apt crap is enabled" -z "$(egrep $PERIODIC_APT_RE /etc/apt/apt.conf.d/*)" - ;; - fedora*) - # Only runc -OR- crun should be installed, never both - case "$CG_FS_TYPE" in - tmpfs) - HAS=runc - HAS_NOT=crun - ;; - cgroup2fs) - HAS=crun - HAS_NOT=runc - ;; - esac - HAS_RC=$(rpm -qV $HAS &> /dev/null; echo $?) - HAS_NOT_RC=$(rpm -qV $HAS_NOT &> /dev/null; echo $?) - item_test "With a cgroups-fs type $CG_FS_TYPE, the $HAS package is installed" $HAS_RC -eq 0 - item_test "With a cgroups-fs type $CG_FS_TYPE, the $HAS_NOT package is not installed" $HAS_NOT_RC -ne 0 - ;; - xfedora*) - echo "Kernel Command-line: $(cat /proc/cmdline)" - item_test \ - "On ${PACKER_BUILDER_NAME} images, the /sys/fs/cgroup/unified directory does NOT exist" \ - "!" "-d" "/sys/fs/cgroup/unified" || let "NFAILS+=1" - ;; - *) echo "No vm-image specific items to check" -esac - -echo "Total failed tests: $NFAILS" -exit $NFAILS diff --git a/contrib/cirrus/git_authors_to_irc_nicks.csv b/contrib/cirrus/git_authors_to_irc_nicks.csv deleted file mode 100644 index a584cc76a..000000000 --- a/contrib/cirrus/git_authors_to_irc_nicks.csv +++ /dev/null @@ -1,12 +0,0 @@ -# Comma separated mapping of author e-mail, to Freenode IRC nick. -# When no match is found here, the username portion of the e-mail is used. -# Sorting is done at runtime - first-found e-mail match wins. -# Comments (like this) and blank lines are ignored. - -bbaude@redhat.com,baude -matthew.heon@pm.me,mheon -matthew.heon@gmail.com,mheon -emilien@redhat.com,EmilienM -rothberg@redhat.com,vrothberg -santiago@redhat.com,edsantiago -gscrivan@redhat.com,giuseppe diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh index 3292e9d14..f125dd76d 100644 --- a/contrib/cirrus/lib.sh +++ b/contrib/cirrus/lib.sh @@ -35,10 +35,8 @@ export PATH="$HOME/bin:$GOPATH/bin:/usr/local/bin:$PATH" export LD_LIBRARY_PATH="/usr/local/lib${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}" # Saves typing / in case location ever moves SCRIPT_BASE=${SCRIPT_BASE:-./contrib/cirrus} -PACKER_BASE=${PACKER_BASE:-./contrib/cirrus/packer} # Important filepaths SETUP_MARKER_FILEPATH="${SETUP_MARKER_FILEPATH:-/var/tmp/.setup_environment_sh_complete}" -AUTHOR_NICKS_FILEPATH="${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/git_authors_to_irc_nicks.csv" # Downloaded, but not installed packages. PACKAGE_DOWNLOAD_DIR=/var/cache/download @@ -61,22 +59,15 @@ CONTINUOUS_INTEGRATION="${CONTINUOUS_INTEGRATION:-false}" CIRRUS_REPO_NAME=${CIRRUS_REPO_NAME:-libpod} CIRRUS_BASE_SHA=${CIRRUS_BASE_SHA:-unknown$(date +%s)} # difficult to reliably discover CIRRUS_BUILD_ID=${CIRRUS_BUILD_ID:-$RANDOM$(date +%s)} # must be short and unique -# Vars. for image-building -PACKER_VER="1.4.2" -# CSV of cache-image names to build (see $PACKER_BASE/libpod_images.json) - -# List of cache imaes to build for 'CI:IMG' mode via build_vm_images.sh -# Exists to support manual single-image building in case of emergency -export PACKER_BUILDS="${PACKER_BUILDS:-ubuntu-20,ubuntu-19,fedora-32,fedora-31}" -# Google cloud provides these, we just make copies (see $SCRIPT_BASE/README.md) for use -export UBUNTU_BASE_IMAGE="ubuntu-2004-focal-v20200506" -export PRIOR_UBUNTU_BASE_IMAGE="ubuntu-1910-eoan-v20200211" -# Manually produced base-image names (see $SCRIPT_BASE/README.md) -export FEDORA_BASE_IMAGE="fedora-cloud-base-32-1-6-1588257430" -export PRIOR_FEDORA_BASE_IMAGE="fedora-cloud-base-31-1-9-1588257430" -export BUILT_IMAGE_SUFFIX="${BUILT_IMAGE_SUFFIX:--$CIRRUS_REPO_NAME-${CIRRUS_BUILD_ID}}" + +OS_RELEASE_ID="$(source /etc/os-release; echo $ID)" +# GCE image-name compatible string representation of distribution _major_ version +OS_RELEASE_VER="$(source /etc/os-release; echo $VERSION_ID | cut -d '.' -f 1)" +# Combined to ease soe usage +OS_REL_VER="${OS_RELEASE_ID}-${OS_RELEASE_VER}" + # IN_PODMAN container image -IN_PODMAN_IMAGE="quay.io/libpod/in_podman:$DEST_BRANCH" +IN_PODMAN_IMAGE="quay.io/libpod/${OS_RELEASE_ID}_podman:$_BUILT_IMAGE_SUFFIX" # Image for uploading releases UPLDREL_IMAGE="quay.io/libpod/upldrel:master" @@ -98,7 +89,7 @@ BIGTO="timeout_attempt_delay_command 300s 5 60s" # Safe env. vars. to transfer from root -> $ROOTLESS_USER (go env handled separately) ROOTLESS_ENV_RE='(CIRRUS_.+)|(ROOTLESS_.+)|(.+_IMAGE.*)|(.+_BASE)|(.*DIRPATH)|(.*FILEPATH)|(SOURCE.*)|(DEPEND.*)|(.+_DEPS_.+)|(OS_REL.*)|(.+_ENV_RE)|(TRAVIS)|(CI.+)|(REMOTE.*)' # Unsafe env. vars for display -SECRET_ENV_RE='(IRCID)|(ACCOUNT)|(GC[EP]..+)|(SSH)' +SECRET_ENV_RE='(ACCOUNT)|(GC[EP]..+)|(SSH)' SPECIALMODE="${SPECIALMODE:-none}" RCLI="${RCLI:-false}" @@ -111,22 +102,9 @@ then else ROOTLESS_USER="${ROOTLESS_USER:-$USER}" fi - -# GCE image-name compatible string representation of distribution name -OS_RELEASE_ID="$(source /etc/os-release; echo $ID)" -# GCE image-name compatible string representation of distribution _major_ version -OS_RELEASE_VER="$(source /etc/os-release; echo $VERSION_ID | cut -d '.' -f 1)" -# Combined to ease soe usage -OS_REL_VER="${OS_RELEASE_ID}-${OS_RELEASE_VER}" # Type of filesystem used for cgroups CG_FS_TYPE="$(stat -f -c %T /sys/fs/cgroup)" -# When building images, the version of automation tooling to install -INSTALL_AUTOMATION_VERSION=1.1.3 - -# Installed into cache-images, supports overrides -# by user-data in case of breakage or for debugging. -CUSTOM_CLOUD_CONFIG_DEFAULTS="$GOSRC/$PACKER_BASE/cloud-init/$OS_RELEASE_ID/cloud.cfg.d" # Pass in a list of one or more envariable names; exit non-zero with # helpful error message if any value is empty req_env_var() { @@ -237,67 +215,6 @@ timeout_attempt_delay_command() { fi } -ircmsg() { - req_env_var CIRRUS_TASK_ID IRCID - [[ -n "$*" ]] || die 9 "ircmsg() invoked without message text argument" - # Sometimes setup_environment.sh didn't run - SCRIPT="$(dirname $0)/podbot.py" - NICK="podbot_$CIRRUS_TASK_ID" - NICK="${NICK:0:15}" # Any longer will break things - set +e - $SCRIPT $NICK $@ - echo "Ignoring exit($?)" - set -e -} - -# This covers all possible human & CI workflow parallel & serial combinations -# where at least one caller must definitively discover if within a commit range -# there is at least one release tag not having any '-' characters (return 0) -# or otherwise (return non-0). -is_release() { - unset RELVER - local ret - req_env_var CIRRUS_CHANGE_IN_REPO - if [[ -n "$CIRRUS_TAG" ]]; then - RELVER="$CIRRUS_TAG" - elif [[ ! "$CIRRUS_BASE_SHA" =~ "unknown" ]] - then - # Normally not possible for this to be empty, except when unittesting. - req_env_var CIRRUS_BASE_SHA - local range="${CIRRUS_BASE_SHA}..${CIRRUS_CHANGE_IN_REPO}" - if echo "${range}$CIRRUS_TAG" | grep -iq 'unknown'; then - die 11 "is_release() unusable range ${range} or tag $CIRRUS_TAG" - fi - - if type -P git &> /dev/null - then - git fetch --all --tags &> /dev/null|| \ - die 12 "is_release() failed to fetch tags" - RELVER=$(git log --pretty='format:%d' $range | \ - grep '(tag:' | sed -r -e 's/\s+[(]tag:\s+(v[0-9].*)[)]/\1/' | \ - sort -uV | tail -1) - ret=$? - else - warn -1 "Git command not found while checking for release" - ret="-1" - fi - [[ "$ret" -eq "0" ]] || \ - die 13 "is_release() failed to parse tags" - else # Not testing a PR, but neither CIRRUS_BASE_SHA or CIRRUS_TAG are set - return 1 - fi - if [[ -n "$RELVER" ]]; then - echo "Found \$RELVER $RELVER" - if echo "$RELVER" | grep -q '-'; then - return 2 # development tag - else - return 0 - fi - else - return 1 # not a release - fi -} - setup_rootless() { req_env_var ROOTLESS_USER GOPATH GOSRC SECRET_ENV_RE ROOTLESS_ENV_RE @@ -369,20 +286,6 @@ setup_rootless() { die 11 "Timeout exceeded waiting for localhost ssh capability" } -# Grab a newer version of git from software collections -# https://www.softwarecollections.org/en/ -# and use it with a wrapper -install_scl_git() { - echo "Installing SoftwareCollections updated 'git' version." - ooe.sh $SUDO yum -y install rh-git29 - cat << "EOF" | $SUDO tee /usr/bin/git -#!/usr/bin/env bash - -scl enable rh-git29 -- git $@ -EOF - $SUDO chmod 755 /usr/bin/git -} - install_test_configs() { echo "Installing cni config, policy and registry config" req_env_var GOSRC SCRIPT_BASE @@ -457,66 +360,3 @@ $FEDORA_BASE_IMAGE $PRIOR_FEDORA_BASE_IMAGE " } - -systemd_banish() { - $GOSRC/$PACKER_BASE/systemd_banish.sh -} - -# This can be removed when the kernel bug fix is included in Fedora -workaround_bfq_bug() { - if [[ "$OS_RELEASE_ID" == "fedora" ]] && [[ $OS_RELEASE_VER -le 32 ]]; then - warn "Switching io scheduler to 'deadline' to avoid RHBZ 1767539" - warn "aka https://bugzilla.kernel.org/show_bug.cgi?id=205447" - echo "mq-deadline" | sudo tee /sys/block/sda/queue/scheduler > /dev/null - echo -n "IO Scheduler set to: " - $SUDO cat /sys/block/sda/queue/scheduler - fi -} - -# Warning: DO NOT USE. -# This is called by other functions as the very last step during the VM Image build -# process. It's purpose is to "reset" the image, so all the first-boot operations -# happen at test runtime (like generating new ssh host keys, resizing partitions, etc.) -_finalize() { - set +e # Don't fail at the very end - if [[ -d "$CUSTOM_CLOUD_CONFIG_DEFAULTS" ]] - then - echo "Installing custom cloud-init defaults" - $SUDO cp -v "$CUSTOM_CLOUD_CONFIG_DEFAULTS"/* /etc/cloud/cloud.cfg.d/ - else - echo "Could not find any files in $CUSTOM_CLOUD_CONFIG_DEFAULTS" - fi - echo "Re-initializing so next boot does 'first-boot' setup again." - cd / - $SUDO rm -rf $GOPATH/src # Actual source will be cloned at runtime - $SUDO rm -rf /var/lib/cloud/instanc* - $SUDO rm -rf /root/.ssh/* - $SUDO rm -rf /etc/ssh/*key* - $SUDO rm -rf /etc/ssh/moduli - $SUDO rm -rf /home/* - $SUDO rm -rf /tmp/* - $SUDO rm -rf /tmp/.??* - $SUDO sync - $SUDO fstrim -av -} - -# Called during VM Image setup, not intended for general use. -rh_finalize() { - set +e # Don't fail at the very end - echo "Resetting to fresh-state for usage as cloud-image." - PKG=$(type -P dnf || type -P yum || echo "") - $SUDO $PKG clean all - $SUDO rm -rf /var/cache/{yum,dnf} - $SUDO rm -f /etc/udev/rules.d/*-persistent-*.rules - $SUDO touch /.unconfigured # force firstboot to run - _finalize -} - -# Called during VM Image setup, not intended for general use. -ubuntu_finalize() { - set +e # Don't fail at the very end - echo "Resetting to fresh-state for usage as cloud-image." - $LILTO $SUDOAPTGET autoremove - $SUDO rm -rf /var/cache/apt - _finalize -} diff --git a/contrib/cirrus/lib.sh.t b/contrib/cirrus/lib.sh.t index 204af1245..643b5513d 100755 --- a/contrib/cirrus/lib.sh.t +++ b/contrib/cirrus/lib.sh.t @@ -84,7 +84,7 @@ BAR=1 test_rev "FOO BAR" 0 '' ############################################################################### -# tests for test_okay() +# tests for item_test() function test_item_test { local exp_msg=$1 @@ -118,46 +118,4 @@ test_item_test "ok okay enough" 0 "okay enough" "line 1 line2" "=" "line 1 line2" -############################################################################### -# tests for is_release() - -# N/B: Assuming tests run in their own process, so wiping out the local -# CIRRUS_BASE_SHA CIRRUS_CHANGE_IN_REPO and CIRRUS_TAG will be okay. -function test_is_release() { - CIRRUS_BASE_SHA="$1" - CIRRUS_CHANGE_IN_REPO="$2" - CIRRUS_TAG="$3" - local exp_status=$4 - local exp_msg=$5 - local msg - msg=$(is_release) - local status=$? - - check_result "$msg" "$exp_msg" "is_release(CIRRUS_BASE_SHA='$1' CIRRUS_CHANGE_IN_REPO='$2' CIRRUS_TAG='$3')" - check_result "$status" "$exp_status" "is_release(...) returned $status" -} - -# FROM TO TAG RET MSG -test_is_release "" "" "" "9" "FATAL: is_release() requires \$CIRRUS_CHANGE_IN_REPO to be non-empty" -test_is_release "x" "" "" "9" "FATAL: is_release() requires \$CIRRUS_CHANGE_IN_REPO to be non-empty" - -# post-merge / tag-push testing, FROM will be set 'unknown' by (lib.sh default) -test_is_release "unknown" "x" "" "1" "" -# post-merge / tag-push testing, oddball tag is set, FROM will be set 'unknown' -test_is_release "unknown" "unknown" "test-tag" "2" "Found \$RELVER test-tag" -# post-merge / tag-push testing, sane tag is set, FROM will be set 'unknown' -test_is_release "unknown" "unknown" "0.0.0" "0" "Found \$RELVER 0.0.0" -# hack/get_ci_vm or PR testing, FROM and TO are set, no tag is set -test_is_release "x" "x" "" "1" "" - -# Negative-testing git with this function is very difficult, assume git works -# test_is_release ... "is_release() failed to fetch tags" -# test_is_release ... "is_release() failed to parse tags" - -BF_V1=$(git rev-parse v1.0.0^) -AT_V1=$(git rev-parse v1.0.0) -test_is_release "$BF_V1" "$BF_V1" "v9.8.7-dev" "2" "Found \$RELVER v9.8.7-dev" -test_is_release "$BF_V1" "$AT_V1" "v9.8.7-dev" "2" "Found \$RELVER v9.8.7-dev" -test_is_release "$BF_V1" "$AT_V1" "" "0" "Found \$RELVER v1.0.0" - exit $rc diff --git a/contrib/cirrus/notice_branch_failure.sh b/contrib/cirrus/notice_branch_failure.sh deleted file mode 100755 index b810bd266..000000000 --- a/contrib/cirrus/notice_branch_failure.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash - -set -e - -source $(dirname $0)/lib.sh - -# mIRC "escape" codes are the most standard, for a non-standard client-side interpretation. -ETX="$(echo -n -e '\x03')" -RED="${ETX}4" -NOR="$(echo -n -e '\x0f')" - -if [[ "$CIRRUS_BRANCH" = "$DEST_BRANCH" ]] -then - BURL="https://cirrus-ci.com/build/$CIRRUS_BUILD_ID" - ircmsg "${RED}[Action Recommended]: ${NOR}Post-merge testing on ${RED}$CIRRUS_BRANCH failed${NOR} in $CIRRUS_TASK_NAME on ${OS_RELEASE_ID}-${OS_RELEASE_VER}: $BURL. Please investigate, and re-run if appropriate." -fi - -# This script assumed to be executed on failure -die 1 "Testing Failed" diff --git a/contrib/cirrus/packer/.gitignore b/contrib/cirrus/packer/.gitignore deleted file mode 100644 index 8f7bdeaf7..000000000 --- a/contrib/cirrus/packer/.gitignore +++ /dev/null @@ -1,7 +0,0 @@ -*json -packer -packer*zip -packer_cache -cidata* -meta-data -user-data diff --git a/contrib/cirrus/packer/Makefile b/contrib/cirrus/packer/Makefile deleted file mode 100644 index c5a8e4cac..000000000 --- a/contrib/cirrus/packer/Makefile +++ /dev/null @@ -1,94 +0,0 @@ -PACKER_VER ?= 1.4.3 -GOARCH=$(shell go env GOARCH) -ARCH=$(uname -m) -PACKER_DIST_FILENAME := packer_${PACKER_VER}_linux_${GOARCH}.zip - -# Only needed for libpod_base_images target -TIMESTAMP := $(shell date +%s) -GOPATH ?= /var/tmp/go -GOSRC ?= $(GOPATH)/src/github.com/containers/libpod -PACKER_BASE ?= contrib/cirrus/packer -SCRIPT_BASE ?= contrib/cirrus -POST_MERGE_BUCKET_SUFFIX ?= - -UBUNTU_BASE_IMAGE = $(shell source ../lib.sh && echo "$$UBUNTU_BASE_IMAGE") -PRIOR_UBUNTU_BASE_IMAGE = $(shell source ../lib.sh && echo "$$PRIOR_UBUNTU_BASE_IMAGE") - -# For debugging nested-virt, use -#TTYDEV := $(shell tty) -TTYDEV := /dev/null - -.PHONY: all -all: libpod_images - -# Utility target for checking required parameters -.PHONY: guard-% -guard-%: - @if [[ -z "$($*)" ]]; then \ - echo "Missing or empty required make variable '$*'."; \ - exit 1; \ - fi; - -%.json: %.yml - @python3 -c 'import json,yaml; json.dump( yaml.safe_load(open("$<").read()), open("$@","w"), indent=2);' - -${PACKER_DIST_FILENAME}: - @curl -L --silent --show-error \ - -O https://releases.hashicorp.com/packer/${PACKER_VER}/${PACKER_DIST_FILENAME} - -packer: ${PACKER_DIST_FILENAME} - @curl -L --silent --show-error \ - https://releases.hashicorp.com/packer/${PACKER_VER}/packer_${PACKER_VER}_SHA256SUMS \ - | grep linux_${GOARCH} > /tmp/packer_sha256sums - @sha256sum --check /tmp/packer_sha256sums - @unzip -o ${PACKER_DIST_FILENAME} - @touch --reference=Makefile ${PACKER_DIST_FILENAME} - -.PHONY: test -test: libpod_base_images.json libpod_images.json packer - ./packer inspect libpod_base_images.json > /dev/null - ./packer inspect libpod_images.json > /dev/null - @echo "All good" - -.PHONY: libpod_images -libpod_images: guard-PACKER_BUILDS libpod_images.json packer - ./packer build \ - -force \ - $(shell test -z "${PACKER_BUILDS}" || echo "-only=${PACKER_BUILDS}") \ - -var GOPATH=$(GOPATH) \ - -var GOSRC=$(GOSRC) \ - -var PACKER_BASE=$(PACKER_BASE) \ - -var SCRIPT_BASE=$(SCRIPT_BASE) \ - libpod_images.json - -cidata.ssh: - ssh-keygen -f $@ -P "" -q - -cidata.ssh.pub: cidata.ssh - touch $@ - -meta-data: - echo "local-hostname: localhost.localdomain" > $@ - -user-data: cidata.ssh.pub - bash make-user-data.sh - -cidata.iso: user-data meta-data - genisoimage -output cidata.iso -volid cidata -input-charset utf-8 -joliet -rock user-data meta-data - -# This is intended to be run by a human, with admin access to the libpod GCE project. -.PHONY: libpod_base_images -libpod_base_images: guard-GCP_PROJECT_ID guard-GOOGLE_APPLICATION_CREDENTIALS libpod_base_images.json cidata.iso cidata.ssh packer - PACKER_CACHE_DIR=/tmp ./packer build \ - $(shell test -z "${PACKER_BUILDS}" || echo "-only=${PACKER_BUILDS}") \ - -force \ - -var TIMESTAMP=$(TIMESTAMP) \ - -var TTYDEV=$(TTYDEV) \ - -var GCP_PROJECT_ID=$(GCP_PROJECT_ID) \ - -var GOOGLE_APPLICATION_CREDENTIALS=$(GOOGLE_APPLICATION_CREDENTIALS) \ - -var GOSRC=$(GOSRC) \ - -var PACKER_BASE=$(PACKER_BASE) \ - -var SCRIPT_BASE=$(SCRIPT_BASE) \ - -var UBUNTU_BASE_IMAGE=$(UBUNTU_BASE_IMAGE) \ - -var PRIOR_UBUNTU_BASE_IMAGE=$(PRIOR_UBUNTU_BASE_IMAGE) \ - libpod_base_images.json diff --git a/contrib/cirrus/packer/README.how-to-update-cirrus-vms b/contrib/cirrus/packer/README.how-to-update-cirrus-vms deleted file mode 100644 index ac2902ffb..000000000 --- a/contrib/cirrus/packer/README.how-to-update-cirrus-vms +++ /dev/null @@ -1,89 +0,0 @@ -This document briefly describes how to update VMs on Cirrus. - -Examples of when you need to do this: - - - to update crun, conmon, or some other package(s) - - to add and/or remove an OS (eg drop f31, add f33) - - to change system config (eg containers.conf or other /etc files) - - to change kernel command-line (boot time) options - -This is a TWO-STEP process: you need to submit a PR with a magic [CI:IMG] -description string, wait for it to finish, grab a magic string from the -results, then resubmit without [CI:IMG]. - -Procedure, Part One of Two: - - 1) Create a working branch: - - $ git co -b my_branch_name - - 2) Make your changes. Typically, zero or more of the following files: - - .cirrus.yml - contrib/cirrus/packer/*_packaging.sh - - I said zero because sometimes you just want to update VMs - with the latest in dnf or ubuntu repos. That doesn't require - changing anything here, simply running new dnf/apt installs. - - 3) Commit your changes. Be sure to include the magic [CI:IMG] string: - - $ git commit -asm'[CI:IMG] this is my commit message' - - 4) Submit your PR: - - $ gh pr create --fill --web - - - -------------------------- INTERMISSION -------------------------- - ...in which we wait for CI to turn green. In particular, although - we only really need 'test_build_cache_images' (45 minutes or so) - to get the required magic number strings, please be a decent - human being and wait for 'verify_test_built_images' (another hour) - so we can all have confidence in our process. Thank you. - -------------------------- INTERMISSION -------------------------- - - -Procedure, Part Two of Two: - - 1) When 'test_build_cache_images' completes, click it, then click - 'View more details on Cirrus CI', then expand the 'Run build_vm_image' - accordion. This gives you a garishly colorful display of lines. - Each color is a different VM. - - 2) Verify that each VM has the packages you require. (The garish log - doesn't actually list this for all packages, so you may need to - look in the 'verify_test_built_images' log for each individual - VM. Click the 'package_versions' accordion.) - - 3) At the bottom of this log you will see a block like: - - Builds finished. The artifacts of successful builds are: - ubuntu-19: A disk image was created: ubuntu-19-podman-6439450735542272 - fedora-31: A disk image was created: fedora-31-podman-6439450735542272 - ..... - - The long numbers at the end should (MUST!) be all identical. - - 4) Edit .cirrus.yml locally. Find '_BUILT_IMAGE_SUFFIX' near the - top. Copy that long number ("6439450735542272", above) and paste - it here, replacing the previous long number. - - 5) Wait for CI to turn green. I know you might have skipped that, - because 'test_build_cache_images' finishes long before 'verify', - and maybe you're in a hurry, but come on. Be responsible. - - 6) Edit the PR description in github: remove '[CI:IMG]' from the - title. Again, *in github*, in the web UI, use the 'Edit' button - at top right next to the PR title. Remove the '[CI:IMG]' string - from the PR title, press Save. If you forget to do this, the - VM-building steps will run again (taking a long time) but it - will be a waste of time. - - 7) Update your PR: - - $ git add .cirrus.yml (to get the new magic IMAGE_SUFFIX string) - $ git commit --amend (remove [CI:IMG] for consistency with 6) - $ git push --force - -You can probably take it from here. diff --git a/contrib/cirrus/packer/README.md b/contrib/cirrus/packer/README.md deleted file mode 100644 index 9a07ed960..000000000 --- a/contrib/cirrus/packer/README.md +++ /dev/null @@ -1,3 +0,0 @@ -These are definitions and scripts consumed by packer to produce the -various distribution images used for CI testing. For more details -see the [Cirrus CI documentation](../README.md) diff --git a/contrib/cirrus/packer/cloud-init/fedora/cloud-init.service b/contrib/cirrus/packer/cloud-init/fedora/cloud-init.service deleted file mode 100644 index 4d2197d87..000000000 --- a/contrib/cirrus/packer/cloud-init/fedora/cloud-init.service +++ /dev/null @@ -1,20 +0,0 @@ -[Unit] -Description=Initial cloud-init job (metadata service crawler) -DefaultDependencies=no -Wants=cloud-init-local.service -After=cloud-init-local.service -Wants=google-network-daemon.service -After=google-network-daemon.service -Before=systemd-user-sessions.service - -[Service] -Type=oneshot -ExecStart=/usr/bin/cloud-init init -RemainAfterExit=yes -TimeoutSec=0 - -# Output needs to appear in instance console output -StandardOutput=journal+console - -[Install] -WantedBy=cloud-init.target diff --git a/contrib/cirrus/packer/cloud-init/fedora/cloud.cfg.d/40_enable_root.cfg b/contrib/cirrus/packer/cloud-init/fedora/cloud.cfg.d/40_enable_root.cfg deleted file mode 100644 index 672d1907b..000000000 --- a/contrib/cirrus/packer/cloud-init/fedora/cloud.cfg.d/40_enable_root.cfg +++ /dev/null @@ -1 +0,0 @@ -disable_root: 0 diff --git a/contrib/cirrus/packer/cloud-init/fedora/cloud.cfg.d/50_custom_disk_setup.cfg b/contrib/cirrus/packer/cloud-init/fedora/cloud.cfg.d/50_custom_disk_setup.cfg deleted file mode 100644 index c0fdf0e23..000000000 --- a/contrib/cirrus/packer/cloud-init/fedora/cloud.cfg.d/50_custom_disk_setup.cfg +++ /dev/null @@ -1,4 +0,0 @@ -#cloud-config -growpart: - mode: false -resize_rootfs: false diff --git a/contrib/cirrus/packer/cloud-init/ubuntu/cloud.cfg.d/40_enable_root.cfg b/contrib/cirrus/packer/cloud-init/ubuntu/cloud.cfg.d/40_enable_root.cfg deleted file mode 100644 index 672d1907b..000000000 --- a/contrib/cirrus/packer/cloud-init/ubuntu/cloud.cfg.d/40_enable_root.cfg +++ /dev/null @@ -1 +0,0 @@ -disable_root: 0 diff --git a/contrib/cirrus/packer/fedora_base-setup.sh b/contrib/cirrus/packer/fedora_base-setup.sh deleted file mode 100644 index bf29a1aec..000000000 --- a/contrib/cirrus/packer/fedora_base-setup.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env bash - -# N/B: This script is not intended to be run by humans. It is used to configure the -# fedora base image for importing, so that it will boot in GCE - -set -e - -# Load in library (copied by packer, before this script was run) -source $GOSRC/$SCRIPT_BASE/lib.sh - -echo "Updating packages" -dnf -y update - -echo "Installing necessary packages and google services" -dnf -y install rng-tools google-compute-engine-tools google-compute-engine-oslogin ethtool - -echo "Enabling services" -systemctl enable rngd - -# There is a race that can happen on boot between the GCE services configuring -# the VM, and cloud-init trying to do similar activities. Use a customized -# unit file to make sure cloud-init starts after the google-compute-* services. -echo "Setting cloud-init service to start after google-network-daemon.service" -cp -v $GOSRC/$PACKER_BASE/cloud-init/fedora/cloud-init.service /etc/systemd/system/ - -# ref: https://cloud.google.com/compute/docs/startupscript -# The mechanism used by Cirrus-CI to execute tasks on the system is through an -# "agent" process launched as a GCP startup-script (from the metadata service). -# This agent is responsible for cloning the repository and executing all task -# scripts and other operations. Therefor, on SELinux-enforcing systems, the -# service must be labeled properly to ensure it's child processes can -# run with the proper contexts. -METADATA_SERVICE_CTX=unconfined_u:unconfined_r:unconfined_t:s0 -METADATA_SERVICE_PATH=systemd/system/google-startup-scripts.service -sed -r -e \ - "s/Type=oneshot/Type=oneshot\nSELinuxContext=$METADATA_SERVICE_CTX/" \ - /lib/$METADATA_SERVICE_PATH > /etc/$METADATA_SERVICE_PATH - -# Ensure there are no disruptive periodic services enabled by default in image -systemd_banish - -rh_finalize - -echo "SUCCESS!" diff --git a/contrib/cirrus/packer/fedora_packaging.sh b/contrib/cirrus/packer/fedora_packaging.sh deleted file mode 100644 index fcf9eb93f..000000000 --- a/contrib/cirrus/packer/fedora_packaging.sh +++ /dev/null @@ -1,194 +0,0 @@ -#!/usr/bin/env bash - -# This script is called from fedora_setup.sh and various Dockerfiles. -# It's not intended to be used outside of those contexts. It assumes the lib.sh -# library has already been sourced, and that all "ground-up" package-related activity -# needs to be done, including repository setup and initial update. - -set -e - -echo "Updating/Installing repos and packages for $OS_REL_VER" - -source $GOSRC/$SCRIPT_BASE/lib.sh - -req_env_var GOSRC SCRIPT_BASE BIGTO INSTALL_AUTOMATION_VERSION FEDORA_BASE_IMAGE PRIOR_FEDORA_BASE_IMAGE - -# Pre-req. to install automation tooing -$LILTO $SUDO dnf install -y git - -# Install common automation tooling (i.e. ooe.sh) -curl --silent --show-error --location \ - --url "https://raw.githubusercontent.com/containers/automation/master/bin/install_automation.sh" | \ - $SUDO env INSTALL_PREFIX=/usr/share /bin/bash -s - "$INSTALL_AUTOMATION_VERSION" -# Reload installed environment right now (happens automatically in a new process) -source /usr/share/automation/environment - -# Set this to 1 to NOT enable updates-testing repository -DISABLE_UPDATES_TESTING=${DISABLE_UPDATES_TESTING:0} - -# Do not enable updates-testing on the previous Fedora release -if ((DISABLE_UPDATES_TESTING!=0)); then - warn "Enabling updates-testing repository for image based on $FEDORA_BASE_IMAGE" - $LILTO $SUDO ooe.sh dnf install -y 'dnf-command(config-manager)' - $LILTO $SUDO ooe.sh dnf config-manager --set-enabled updates-testing -else - warn "NOT enabling updates-testing repository for image based on $PRIOR_FEDORA_BASE_IMAGE" -fi - -$BIGTO ooe.sh $SUDO dnf update -y - -# Fedora, as of 31, uses cgroups v2 by default. runc does not support -# cgroups v2, only crun does. (As of 2020-07-30 runc support is -# forthcoming but not even close to ready yet). To ensure a reliable -# runtime environment, force-remove runc if it is present. -# However, because a few other repos. which use these images still need -# it, ensure the runc package is cached in $PACKAGE_DOWNLOAD_DIR so -# it may be swap it in when required. -REMOVE_PACKAGES=(runc) - -INSTALL_PACKAGES=(\ - autoconf - automake - bash-completion - bats - bridge-utils - btrfs-progs-devel - buildah - bzip2 - conmon - container-selinux - containernetworking-plugins - containers-common - criu - crun - curl - device-mapper-devel - dnsmasq - e2fsprogs-devel - emacs-nox - file - findutils - fuse3 - fuse3-devel - gcc - git - glib2-devel - glibc-devel - glibc-static - gnupg - go-md2man - golang - gpgme - gpgme-devel - grubby - hostname - httpd-tools - iproute - iptables - jq - krb5-workstation - libassuan - libassuan-devel - libblkid-devel - libcap-devel - libffi-devel - libgpg-error-devel - libguestfs-tools - libmsi1 - libnet - libnet-devel - libnl3-devel - libseccomp - libseccomp-devel - libselinux-devel - libtool - libvarlink-util - libxml2-devel - libxslt-devel - lsof - make - mlocate - msitools - nfs-utils - nmap-ncat - openssl - openssl-devel - ostree-devel - pandoc - pkgconfig - podman - policycoreutils - procps-ng - protobuf - protobuf-c - protobuf-c-devel - protobuf-devel - python2 - python3-PyYAML - python3-dateutil - python3-libselinux - python3-libsemanage - python3-libvirt - python3-psutil - python3-pytoml - python3-requests - redhat-rpm-config - rpcbind - rsync - sed - selinux-policy-devel - skopeo - skopeo-containers - slirp4netns - socat - tar - unzip - vim - wget - which - xz - zip - zlib-devel -) -DOWNLOAD_PACKAGES=(\ - "cri-o-$(get_kubernetes_version)*" - cri-tools - "kubernetes-$(get_kubernetes_version)*" - runc - oci-umount - parallel -) - -echo "Installing general build/test dependencies for Fedora '$OS_RELEASE_VER'" -$BIGTO ooe.sh $SUDO dnf install -y ${INSTALL_PACKAGES[@]} - -# AD-HOC CODE FOR SPECIAL-CASE SITUATIONS! -# On 2020-07-23 we needed this code to upgrade crun on f31, a build -# that is not yet in stable. Since CI:IMG PRs are a two-step process, -# the key part is that we UN-COMMENT-THIS-OUT during the first step, -# then re-comment it on the second (once we have the built images). -# That way this will be dead code in future CI:IMG PRs but will -# serve as an example for anyone in a similar future situation. -# $BIGTO ooe.sh $SUDO dnf --enablerepo=updates-testing -y upgrade crun - -[[ ${#REMOVE_PACKAGES[@]} -eq 0 ]] || \ - $LILTO ooe.sh $SUDO dnf erase -y "${REMOVE_PACKAGES[@]}" - -if [[ ${#DOWNLOAD_PACKAGES[@]} -gt 0 ]]; then - echo "Downloading packages for optional installation at runtime, as needed." - # Required for cri-o - ooe.sh $SUDO dnf -y module enable cri-o:$(get_kubernetes_version) - $SUDO mkdir -p "$PACKAGE_DOWNLOAD_DIR" - cd "$PACKAGE_DOWNLOAD_DIR" - $LILTO ooe.sh $SUDO dnf download -y --resolve "${DOWNLOAD_PACKAGES[@]}" -fi - -echo "Installing runtime tooling" -# Save some runtime by having these already available -cd $GOSRC -# Required since initially go was not installed -source $GOSRC/$SCRIPT_BASE/lib.sh -echo "Go environment has been setup:" -go env -$SUDO make install.tools -$SUDO $GOSRC/hack/install_catatonit.sh diff --git a/contrib/cirrus/packer/fedora_setup.sh b/contrib/cirrus/packer/fedora_setup.sh deleted file mode 100644 index 16ae87d8a..000000000 --- a/contrib/cirrus/packer/fedora_setup.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash - -# This script is called by packer on the subject fedora VM, to setup the podman -# build/test environment. It's not intended to be used outside of this context. - -set -e - -# Load in library (copied by packer, before this script was run) -source $GOSRC/$SCRIPT_BASE/lib.sh - -req_env_var SCRIPT_BASE PACKER_BASE INSTALL_AUTOMATION_VERSION PACKER_BUILDER_NAME GOSRC FEDORA_BASE_IMAGE OS_RELEASE_ID OS_RELEASE_VER - -workaround_bfq_bug - -# Do not enable updates-testing on the previous Fedora release -if [[ "$PRIOR_FEDORA_BASE_IMAGE" =~ "${OS_RELEASE_ID}-cloud-base-${OS_RELEASE_VER}" ]]; then - DISABLE_UPDATES_TESTING=1 -else - DISABLE_UPDATES_TESTING=0 -fi - -bash $PACKER_BASE/fedora_packaging.sh -# Load installed environment right now (happens automatically in a new process) -source /usr/share/automation/environment - -echo "Enabling cgroup management from containers" -ooe.sh sudo setsebool container_manage_cgroup true - -# Ensure there are no disruptive periodic services enabled by default in image -systemd_banish - -rh_finalize - -echo "SUCCESS!" diff --git a/contrib/cirrus/packer/image-builder-image_base-setup.sh b/contrib/cirrus/packer/image-builder-image_base-setup.sh deleted file mode 100644 index 26fbe2903..000000000 --- a/contrib/cirrus/packer/image-builder-image_base-setup.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env bash - -# This script is called by packer on a vanilla CentOS VM, to setup the image -# used for building images FROM base images. It's not intended to be used -# outside of this context. - -set -e - -[[ "$1" == "post" ]] || exit 0 # pre stage not needed - -# Load in library (copied by packer, before this script was run) -source $GOSRC/$SCRIPT_BASE/lib.sh - -req_env_var TIMESTAMP GOSRC SCRIPT_BASE PACKER_BASE - -install_ooe - -echo "Updating packages" -ooe.sh sudo yum -y update - -echo "Configuring repositories" -ooe.sh sudo yum -y install centos-release-scl epel-release - -echo "Installing packages" -ooe.sh sudo yum -y install \ - genisoimage \ - golang \ - google-cloud-sdk \ - libvirt \ - libvirt-admin \ - libvirt-client \ - libvirt-daemon \ - make \ - python36 \ - python36-PyYAML \ - qemu-img \ - qemu-kvm \ - qemu-kvm-tools \ - qemu-user \ - rsync \ - rng-tools \ - unzip \ - util-linux \ - vim - -sudo systemctl enable rngd - -sudo ln -s /usr/libexec/qemu-kvm /usr/bin/ - -sudo tee /etc/modprobe.d/kvm-nested.conf <<EOF -options kvm-intel nested=1 -options kvm-intel enable_shadow_vmcs=1 -options kvm-intel enable_apicv=1 -options kvm-intel ept=1 -EOF - -echo "Installing packer" -sudo mkdir -p /root/$(basename $PACKER_BASE) -sudo cp $GOSRC/$PACKER_BASE/*packer* /root/$(basename $PACKER_BASE) -sudo mkdir -p /root/$(basename $SCRIPT_BASE) -sudo cp $GOSRC/$SCRIPT_BASE/*.sh /root/$(basename $SCRIPT_BASE) - -install_scl_git - -echo "Cleaning up" -cd / -rm -rf $GOSRC - -rh_finalize - -echo "SUCCESS!" diff --git a/contrib/cirrus/packer/libpod_base_images.yml b/contrib/cirrus/packer/libpod_base_images.yml deleted file mode 100644 index f53bfafc5..000000000 --- a/contrib/cirrus/packer/libpod_base_images.yml +++ /dev/null @@ -1,164 +0,0 @@ ---- - -variables: - # Complete local path to this repository (Required) - GOSRC: - # Relative path to this (packer) subdirectory (Required) - PACKER_BASE: - # Relative path to cirrus scripts subdirectory (Required) - SCRIPT_BASE: - # Unique ID for naming new base-images (required) - TIMESTAMP: - # Required for output from qemu builders - TTYDEV: - - # Ubuntu releases are merely copied to this project for control purposes - UBUNTU_BASE_IMAGE: - PRIOR_UBUNTU_BASE_IMAGE: - - # Latest Fedora release - FEDORA_IMAGE_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/32/Cloud/x86_64/images/Fedora-Cloud-Base-32-1.6.x86_64.qcow2" - FEDORA_CSUM_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/32/Cloud/x86_64/images/Fedora-Cloud-32-1.6-x86_64-CHECKSUM" - FEDORA_BASE_IMAGE_NAME: 'fedora-cloud-base-32-1-6' - - # Prior Fedora release - PRIOR_FEDORA_IMAGE_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/31/Cloud/x86_64/images/Fedora-Cloud-Base-31-1.9.x86_64.qcow2" - PRIOR_FEDORA_CSUM_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/31/Cloud/x86_64/images/Fedora-Cloud-31-1.9-x86_64-CHECKSUM" - PRIOR_FEDORA_BASE_IMAGE_NAME: 'fedora-cloud-base-31-1-9' - - # The name of the image in GCE used for packer build libpod_images.yml - IBI_BASE_NAME: 'image-builder-image' - CIDATA_ISO: 'cidata.iso' # produced by Makefile - - # Path to json file (required, likely ~/.config/gcloud/legacy_credentials/*/adc.json) - GOOGLE_APPLICATION_CREDENTIALS: - # The complete project ID (required, not the short name) - GCP_PROJECT_ID: - # Pre-existing storage bucket w/ lifecycle-enabled - XFERBUCKET: "packer-import" # pre-created, globally unique, lifecycle-enabled - -# Don't leak sensitive values in error messages / output -sensitive-variables: - - 'GOOGLE_APPLICATION_CREDENTIALS' - - 'GCP_PROJECT_ID' - -# What images to produce in which cloud -builders: - - &nested_virt - name: 'fedora' - type: 'qemu' - accelerator: "kvm" - iso_url: '{{user `FEDORA_IMAGE_URL`}}' - disk_image: true - format: "raw" - disk_size: 5120 - iso_checksum_url: '{{user `FEDORA_CSUM_URL`}}' - iso_checksum_type: "sha256" - output_directory: '/tmp/{{build_name}}' - vm_name: "disk.raw" # actually qcow2, name required for post-processing - boot_wait: '5s' - shutdown_command: 'shutdown -h now' - headless: true - qemu_binary: "/usr/libexec/qemu-kvm" - qemuargs: # List-of-list format required to override packer-generated args - - - "-m" - - "1024" - - - "-cpu" - - "host" - - - "-device" - - "virtio-rng-pci" - - - "-chardev" - - "tty,id=pts,path={{user `TTYDEV`}}" - - - "-device" - - "isa-serial,chardev=pts" - - - "-cdrom" - - "{{user `CIDATA_ISO`}}" - - - "-netdev" - - "user,id=net0,hostfwd=tcp::{{ .SSHHostPort }}-:22" - - - "-device" - - "virtio-net,netdev=net0" - communicator: 'ssh' - ssh_private_key_file: 'cidata.ssh' - ssh_username: 'root' - - - <<: *nested_virt - name: 'prior-fedora' - iso_url: '{{user `PRIOR_FEDORA_IMAGE_URL`}}' - iso_checksum_url: '{{user `PRIOR_FEDORA_CSUM_URL`}}' - - - &imgcopy - name: 'ubuntu' - type: 'googlecompute' - image_name: '{{user `UBUNTU_BASE_IMAGE`}}' - image_family: '{{build_name}}-base' - source_image: '{{user `UBUNTU_BASE_IMAGE`}}' - source_image_project_id: 'ubuntu-os-cloud' - project_id: '{{user `GCP_PROJECT_ID`}}' - account_file: '{{user `GOOGLE_APPLICATION_CREDENTIALS`}}' - startup_script_file: "systemd_banish.sh" - zone: 'us-central1-a' - disk_size: 20 - communicator: 'none' - - - <<: *imgcopy - name: 'prior-ubuntu' - image_name: '{{user `PRIOR_UBUNTU_BASE_IMAGE`}}' - source_image: '{{user `PRIOR_UBUNTU_BASE_IMAGE`}}' - -provisioners: - - type: 'shell' - only: ['fedora', 'prior-fedora'] - inline: - - 'mkdir -p /tmp/libpod/{{user `SCRIPT_BASE`}}' - - 'mkdir -p /tmp/libpod/{{user `PACKER_BASE`}}' - - - type: 'file' - only: ['fedora', 'prior-fedora'] - source: '{{user `GOSRC`}}/.cirrus.yml' - destination: '/tmp/libpod/.cirrus.yml' - - - type: 'file' - only: ['fedora', 'prior-fedora'] - source: '{{user `GOSRC`}}/{{user `SCRIPT_BASE`}}/' - destination: '/tmp/libpod/{{user `SCRIPT_BASE`}}/' - - - type: 'file' - only: ['fedora', 'prior-fedora'] - source: '{{user `GOSRC`}}/{{user `PACKER_BASE`}}/' - destination: '/tmp/libpod/{{user `PACKER_BASE`}}/' - - - &shell_script - only: ['fedora', 'prior-fedora'] - type: 'shell' - inline: - - 'chmod +x /tmp/libpod/{{user `PACKER_BASE`}}/*.sh' - - '/tmp/libpod/{{user `PACKER_BASE`}}/{{build_name}}_base-setup.sh' - expect_disconnect: true # Allow this to reboot the VM if needed - environment_vars: - - 'TIMESTAMP={{user `TIMESTAMP`}}' - - 'GOSRC=/tmp/libpod' - - 'SCRIPT_BASE={{user `SCRIPT_BASE`}}' - - 'PACKER_BASE={{user `PACKER_BASE`}}' - -post-processors: - - - type: "compress" - only: ['fedora', 'prior-fedora'] - output: '/tmp/{{build_name}}/disk.raw.tar.gz' - format: '.tar.gz' - compression_level: 9 - - &gcp_import - only: ['fedora'] - type: "googlecompute-import" - project_id: '{{user `GCP_PROJECT_ID`}}' - account_file: '{{user `GOOGLE_APPLICATION_CREDENTIALS`}}' - bucket: '{{user `XFERBUCKET`}}' - gcs_object_name: '{{build_name}}-{{user `TIMESTAMP`}}.tar.gz' - image_name: "{{user `FEDORA_BASE_IMAGE_NAME`}}-{{user `TIMESTAMP`}}" - image_description: 'Based on {{user `FEDORA_IMAGE_URL`}}' - image_family: '{{build_name}}-base' - - <<: *gcp_import - only: ['prior-fedora'] - image_name: "{{user `PRIOR_FEDORA_BASE_IMAGE_NAME`}}-{{user `TIMESTAMP`}}" - image_description: 'Based on {{user `PRIOR_FEDORA_IMAGE_URL`}}' - image_family: '{{build_name}}-base' - - type: 'manifest' diff --git a/contrib/cirrus/packer/libpod_images.yml b/contrib/cirrus/packer/libpod_images.yml deleted file mode 100644 index 38f5a8250..000000000 --- a/contrib/cirrus/packer/libpod_images.yml +++ /dev/null @@ -1,86 +0,0 @@ ---- - -# All of these are required -variables: - BUILT_IMAGE_SUFFIX: '{{env `BUILT_IMAGE_SUFFIX`}}' - GOPATH: '{{env `GOPATH`}}' - GOSRC: '{{env `GOSRC`}}' - PACKER_BASE: '{{env `PACKER_BASE`}}' - SCRIPT_BASE: '{{env `SCRIPT_BASE`}}' - - # Base-image names are required. Using image family-names breaks parallelism - UBUNTU_BASE_IMAGE: '{{env `UBUNTU_BASE_IMAGE`}}' - PRIOR_UBUNTU_BASE_IMAGE: '{{env `PRIOR_UBUNTU_BASE_IMAGE`}}' - FEDORA_BASE_IMAGE: '{{env `FEDORA_BASE_IMAGE`}}' - PRIOR_FEDORA_BASE_IMAGE: '{{env `PRIOR_FEDORA_BASE_IMAGE`}}' - - # Protected credentials, decrypted by Cirrus at runtime - GCE_SSH_USERNAME: '{{env `GCE_SSH_USERNAME`}}' - GCP_PROJECT_ID: '{{env `GCP_PROJECT_ID`}}' - SERVICE_ACCOUNT: '{{env `SERVICE_ACCOUNT`}}' - GOOGLE_APPLICATION_CREDENTIALS: '{{env `GOOGLE_APPLICATION_CREDENTIALS`}}' - -# Don't leak sensitive values in error messages / output -sensitive-variables: - - 'GCE_SSH_USERNAME' - - 'GCP_PROJECT_ID' - - 'SERVICE_ACCOUNT' - -# What images to produce in which cloud -builders: - # v----- is a YAML anchor, allows referencing this object by name (below) - - &gce_hosted_image - name: 'ubuntu-20' - type: 'googlecompute' - image_name: '{{build_name}}{{user `BUILT_IMAGE_SUFFIX`}}' - image_family: '{{build_name}}-cache' - source_image: '{{user `UBUNTU_BASE_IMAGE`}}' # precedence over family - source_image_family: 'ubuntu-base' # for ref. only - disk_size: 20 # REQUIRED: Runtime allocation > this value - project_id: '{{user `GCP_PROJECT_ID`}}' - service_account_email: '{{user `SERVICE_ACCOUNT`}}' - communicator: 'ssh' - ssh_username: '{{user `GCE_SSH_USERNAME`}}' - ssh_pty: 'true' - # The only supported zone in Cirrus-CI, as of addition of this comment - zone: 'us-central1-a' - - # v----- is a YAML alias, allows partial re-use of the anchor object - - <<: *gce_hosted_image - name: 'ubuntu-19' - source_image: '{{user `PRIOR_UBUNTU_BASE_IMAGE`}}' - source_image_family: 'prior-ubuntu-base' - - - <<: *gce_hosted_image - name: 'fedora-32' - source_image: '{{user `FEDORA_BASE_IMAGE`}}' - source_image_family: 'fedora-base' - - - <<: *gce_hosted_image - name: 'fedora-31' - source_image: '{{user `PRIOR_FEDORA_BASE_IMAGE`}}' - source_image_family: 'prior-fedora-base' - -# The brains of the operation, making actual modifications to the base-image. -provisioners: - - type: 'shell' - inline: - - 'set -ex' - # The 'file' provisioner item (below) will create the final component - - 'mkdir -vp $(dirname {{user `GOSRC`}})' - - - type: 'file' - source: '{{user `GOSRC`}}' - destination: '{{user `GOSRC`}}' - - - type: 'shell' - script: '{{user `GOSRC`}}/{{user `PACKER_BASE`}}/{{split build_name "-" 0}}_setup.sh' - environment_vars: - - 'PACKER_BUILDER_NAME={{build_name}}' - - 'GOPATH={{user `GOPATH`}}' - - 'GOSRC={{user `GOSRC`}}' - - 'PACKER_BASE={{user `PACKER_BASE`}}' - - 'SCRIPT_BASE={{user `SCRIPT_BASE`}}' - -post-processors: - - type: 'manifest' # writes packer-manifest.json diff --git a/contrib/cirrus/packer/make-user-data.sh b/contrib/cirrus/packer/make-user-data.sh deleted file mode 100644 index 676a50f5c..000000000 --- a/contrib/cirrus/packer/make-user-data.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash - -# This script is utilized by Makefile, it's not intended to be run by humans - -cat <<EOF > user-data -#cloud-config -timezone: US/Eastern -growpart: - mode: auto -disable_root: false -ssh_pwauth: True -ssh_import_id: [root] -ssh_authorized_keys: - - $(cat cidata.ssh.pub) -users: - - name: root - primary-group: root - homedir: /root - system: true -EOF diff --git a/contrib/cirrus/packer/prior-fedora_base-setup.sh b/contrib/cirrus/packer/prior-fedora_base-setup.sh deleted file mode 100644 index bf29a1aec..000000000 --- a/contrib/cirrus/packer/prior-fedora_base-setup.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env bash - -# N/B: This script is not intended to be run by humans. It is used to configure the -# fedora base image for importing, so that it will boot in GCE - -set -e - -# Load in library (copied by packer, before this script was run) -source $GOSRC/$SCRIPT_BASE/lib.sh - -echo "Updating packages" -dnf -y update - -echo "Installing necessary packages and google services" -dnf -y install rng-tools google-compute-engine-tools google-compute-engine-oslogin ethtool - -echo "Enabling services" -systemctl enable rngd - -# There is a race that can happen on boot between the GCE services configuring -# the VM, and cloud-init trying to do similar activities. Use a customized -# unit file to make sure cloud-init starts after the google-compute-* services. -echo "Setting cloud-init service to start after google-network-daemon.service" -cp -v $GOSRC/$PACKER_BASE/cloud-init/fedora/cloud-init.service /etc/systemd/system/ - -# ref: https://cloud.google.com/compute/docs/startupscript -# The mechanism used by Cirrus-CI to execute tasks on the system is through an -# "agent" process launched as a GCP startup-script (from the metadata service). -# This agent is responsible for cloning the repository and executing all task -# scripts and other operations. Therefor, on SELinux-enforcing systems, the -# service must be labeled properly to ensure it's child processes can -# run with the proper contexts. -METADATA_SERVICE_CTX=unconfined_u:unconfined_r:unconfined_t:s0 -METADATA_SERVICE_PATH=systemd/system/google-startup-scripts.service -sed -r -e \ - "s/Type=oneshot/Type=oneshot\nSELinuxContext=$METADATA_SERVICE_CTX/" \ - /lib/$METADATA_SERVICE_PATH > /etc/$METADATA_SERVICE_PATH - -# Ensure there are no disruptive periodic services enabled by default in image -systemd_banish - -rh_finalize - -echo "SUCCESS!" diff --git a/contrib/cirrus/packer/systemd_banish.sh b/contrib/cirrus/packer/systemd_banish.sh deleted file mode 100755 index 2219f2a4f..000000000 --- a/contrib/cirrus/packer/systemd_banish.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash - -set +e # Not all of these exist on every platform - -# This is intended to be executed on VMs as a startup script on initial-boot. -# Alternatively, it may be executed with the '--list' option to return the list -# of systemd units defined for disablement (useful for testing). - -EVIL_UNITS="cron crond atd apt-daily-upgrade apt-daily fstrim motd-news systemd-tmpfiles-clean" - -if [[ "$1" == "--list" ]] -then - echo "$EVIL_UNITS" - exit 0 -fi - -echo "Disabling periodic services that could destabilize testing:" -for unit in $EVIL_UNITS -do - echo "Banishing $unit (ignoring errors)" - ( - sudo systemctl stop $unit - sudo systemctl disable $unit - sudo systemctl disable $unit.timer - sudo systemctl mask $unit - sudo systemctl mask $unit.timer - ) &> /dev/null -done diff --git a/contrib/cirrus/packer/ubuntu_packaging.sh b/contrib/cirrus/packer/ubuntu_packaging.sh deleted file mode 100644 index c478028b5..000000000 --- a/contrib/cirrus/packer/ubuntu_packaging.sh +++ /dev/null @@ -1,175 +0,0 @@ -#!/usr/bin/env bash - -# This script is called from ubuntu_setup.sh and various Dockerfiles. -# It's not intended to be used outside of those contexts. It assumes the lib.sh -# library has already been sourced, and that all "ground-up" package-related activity -# needs to be done, including repository setup and initial update. - -set -e - -echo "Updating/Installing repos and packages for $OS_REL_VER" - -source $GOSRC/$SCRIPT_BASE/lib.sh - -req_env_var GOSRC SCRIPT_BASE BIGTO SUDOAPTGET INSTALL_AUTOMATION_VERSION - -echo "Updating/configuring package repositories." -$BIGTO $SUDOAPTGET update - -echo "Installing deps to add third-party repositories and automation tooling" -$LILTO $SUDOAPTGET install software-properties-common git curl - -# Install common automation tooling (i.e. ooe.sh) -curl --silent --show-error --location \ - --url "https://raw.githubusercontent.com/containers/automation/master/bin/install_automation.sh" | \ - $SUDO env INSTALL_PREFIX=/usr/share /bin/bash -s - "$INSTALL_AUTOMATION_VERSION" -# Reload installed environment right now (happens automatically in a new process) -source /usr/share/automation/environment - -$LILTO ooe.sh $SUDOAPTADD ppa:criu/ppa - -echo "Configuring/Instaling deps from Open build server" -VERSION_ID=$(source /etc/os-release; echo $VERSION_ID) -echo "deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_$VERSION_ID/ /" \ - | ooe.sh $SUDO tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list -ooe.sh curl -L -o /tmp/Release.key "https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/xUbuntu_${VERSION_ID}/Release.key" -ooe.sh $SUDO apt-key add - < /tmp/Release.key - -INSTALL_PACKAGES=(\ - apache2-utils - apparmor - aufs-tools - autoconf - automake - bash-completion - bats - bison - btrfs-progs - build-essential - buildah - bzip2 - conmon - containernetworking-plugins - containers-common - coreutils - cri-o-runc - criu - curl - dnsmasq - e2fslibs-dev - emacs-nox - file - fuse3 - gawk - gcc - gettext - git - go-md2man - golang-1.14 - iproute2 - iptables - jq - libaio-dev - libapparmor-dev - libbtrfs-dev - libcap-dev - libdevmapper-dev - libdevmapper1.02.1 - libfuse-dev - libfuse2 - libfuse3-dev - libglib2.0-dev - libgpgme11-dev - liblzma-dev - libnet1 - libnet1-dev - libnl-3-dev - libprotobuf-c-dev - libprotobuf-dev - libseccomp-dev - libseccomp2 - libselinux-dev - libsystemd-dev - libtool - libudev-dev - libvarlink - lsof - make - netcat - openssl - pkg-config - podman - protobuf-c-compiler - protobuf-compiler - python-dateutil - python-protobuf - python2 - python3-dateutil - python3-pip - python3-psutil - python3-pytoml - python3-requests - python3-setuptools - rsync - runc - scons - skopeo - slirp4netns - socat - sudo - unzip - vim - wget - xz-utils - zip - zlib1g-dev -) -DOWNLOAD_PACKAGES=(\ - cri-o-$(get_kubernetes_version) - cri-tools - parallel -) - -# These aren't resolvable on Ubuntu 20 -if [[ "$OS_RELEASE_VER" -le 19 ]]; then - INSTALL_PACKAGES+=(\ - python-future - python-minimal - yum-utils - ) -else - INSTALL_PACKAGES+=(\ - python-is-python3 - ) -fi - -# Do this at the last possible moment to avoid dpkg lock conflicts -echo "Upgrading all packages" -$BIGTO ooe.sh $SUDOAPTGET upgrade - -echo "Installing general testing and system dependencies" -# Necessary to update cache of newly added repos -$LILTO ooe.sh $SUDOAPTGET update -$BIGTO ooe.sh $SUDOAPTGET install "${INSTALL_PACKAGES[@]}" - -if [[ ${#DOWNLOAD_PACKAGES[@]} -gt 0 ]]; then - echo "Downloading packages for optional installation at runtime, as needed." - $SUDO ln -s /var/cache/apt/archives "$PACKAGE_DOWNLOAD_DIR" - $LILTO ooe.sh $SUDOAPTGET install --download-only "${DOWNLOAD_PACKAGES[@]}" -fi - -echo "Configuring Go environment" -# There are multiple (otherwise conflicting) versions of golang available -# on Ubuntu. Being primarily localized by env. vars and defaults, dropping -# a symlink is the appropriate way to "install" a specific version system-wide. -$SUDO ln -sf /usr/lib/go-1.14/bin/go /usr/bin/go -# Initially go was not installed -cd $GOSRC -source $SCRIPT_BASE/lib.sh -echo "Go environment has been setup:" -go env - -echo "Building/Installing runtime tooling" -$SUDO hack/install_catatonit.sh -$SUDO make install.libseccomp.sudo -$SUDO make install.tools GO_BUILD='go build' # -mod=vendor breaks this diff --git a/contrib/cirrus/packer/ubuntu_setup.sh b/contrib/cirrus/packer/ubuntu_setup.sh deleted file mode 100644 index d650e6c76..000000000 --- a/contrib/cirrus/packer/ubuntu_setup.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env bash - -# This script is called by packer on the subject Ubuntu VM, to setup the podman -# build/test environment. It's not intended to be used outside of this context. - -set -e - -# Load in library (copied by packer, before this script was run) -source $GOSRC/$SCRIPT_BASE/lib.sh - -req_env_var SCRIPT_BASE PACKER_BASE INSTALL_AUTOMATION_VERSION PACKER_BUILDER_NAME GOSRC UBUNTU_BASE_IMAGE OS_RELEASE_ID OS_RELEASE_VER - -# Ensure there are no disruptive periodic services enabled by default in image -systemd_banish - -# Stop disruption upon boot ASAP after booting -echo "Disabling all packaging activity on boot" -for filename in $(sudo ls -1 /etc/apt/apt.conf.d); do \ - echo "Checking/Patching $filename" - sudo sed -i -r -e "s/$PERIODIC_APT_RE/"'\10"\;/' "/etc/apt/apt.conf.d/$filename"; done - -bash $PACKER_BASE/ubuntu_packaging.sh - -# Load installed environment right now (happens automatically in a new process) -source /usr/share/automation/environment - -echo "Making Ubuntu kernel to enable cgroup swap accounting as it is not the default." -SEDCMD='s/^GRUB_CMDLINE_LINUX="(.*)"/GRUB_CMDLINE_LINUX="\1 cgroup_enable=memory swapaccount=1"/g' -ooe.sh sudo sed -re "$SEDCMD" -i /etc/default/grub.d/* -ooe.sh sudo sed -re "$SEDCMD" -i /etc/default/grub -ooe.sh sudo update-grub - -ubuntu_finalize - -echo "SUCCESS!" diff --git a/contrib/cirrus/packer/xfedora_setup.sh b/contrib/cirrus/packer/xfedora_setup.sh deleted file mode 100644 index 16ae87d8a..000000000 --- a/contrib/cirrus/packer/xfedora_setup.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash - -# This script is called by packer on the subject fedora VM, to setup the podman -# build/test environment. It's not intended to be used outside of this context. - -set -e - -# Load in library (copied by packer, before this script was run) -source $GOSRC/$SCRIPT_BASE/lib.sh - -req_env_var SCRIPT_BASE PACKER_BASE INSTALL_AUTOMATION_VERSION PACKER_BUILDER_NAME GOSRC FEDORA_BASE_IMAGE OS_RELEASE_ID OS_RELEASE_VER - -workaround_bfq_bug - -# Do not enable updates-testing on the previous Fedora release -if [[ "$PRIOR_FEDORA_BASE_IMAGE" =~ "${OS_RELEASE_ID}-cloud-base-${OS_RELEASE_VER}" ]]; then - DISABLE_UPDATES_TESTING=1 -else - DISABLE_UPDATES_TESTING=0 -fi - -bash $PACKER_BASE/fedora_packaging.sh -# Load installed environment right now (happens automatically in a new process) -source /usr/share/automation/environment - -echo "Enabling cgroup management from containers" -ooe.sh sudo setsebool container_manage_cgroup true - -# Ensure there are no disruptive periodic services enabled by default in image -systemd_banish - -rh_finalize - -echo "SUCCESS!" diff --git a/contrib/cirrus/podbot.py b/contrib/cirrus/podbot.py deleted file mode 100755 index 9ca4915a7..000000000 --- a/contrib/cirrus/podbot.py +++ /dev/null @@ -1,105 +0,0 @@ -#!/usr/bin/env python3 - -# Simple and dumb script to send a message to the #podman IRC channel on frenode -# Based on example from: https://pythonspot.com/building-an-irc-bot/ - -import os -import time -import random -import errno -import socket -import sys - -class IRC: - - response_timeout = 30 # seconds - irc = socket.socket() - - def __init__(self, server, nickname, channel): - self.server = server - self.nickname = nickname - self.channel = channel - self.irc = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - - def _send(self, cmdstr): - self.irc.send(bytes(cmdstr + '\r\n', 'utf-8')) - - def message(self, msg): - data = 'PRIVMSG {0} :{1}\r\n'.format(self.channel, msg) - print(data) - self._send(data) - - @staticmethod - def fix_newlines(bufr): - return bufr.replace('\\r\\n', '\n') - - def _required_response(self, needle, haystack): - start = time.time() - end = start + self.response_timeout - while time.time() < end: - if haystack.find(needle) != -1: - return (False, haystack) - time.sleep(0.1) - try: - haystack += str(self.irc.recv(4096, socket.MSG_DONTWAIT)) - except socket.error as serr: - if serr.errno == errno.EWOULDBLOCK: - continue - raise # can't handle this - return (True, haystack) # Error - - def connect(self, username, password): - # This is ugly as sin, but seems to be a working send/expect sequence - - print("connecting to: {0}".format(self.server)) - self.irc.connect((self.server, 6667)) #connects to the server - self._send("USER {0} {0} {0} :I am {0}".format(self.nickname)) - self._send("NICK {0}".format(self.nickname)) - - err, haystack = self._required_response('End of /MOTD command.' - ''.format(self.nickname), "") - if err: - print(self.fix_newlines(haystack)) - print("Error connecting to {0}".format(self.server)) - return True - - print("Logging in as {0}".format(username)) - self._send("PRIVMSG NickServ :IDENTIFY {0} {1}".format(username, password)) - err, _ = self._required_response("You are now identified for", "") - if err: - print("Error logging in to {0} as {1}".format(self.server, username)) - return True - - print("Joining {0}".format(self.channel)) - self._send("JOIN {0}".format(self.channel)) - err, haystack = self._required_response("{0} {1} :End of /NAMES list." - "".format(self.nickname, self.channel), - haystack) - print(self.fix_newlines(haystack)) - if err: - print("Error joining {0}".format(self.channel)) - return True - return False - - def quit(self): - print("Quitting") - self._send("QUIT :my work is done here") - self.irc.close() - - -if len(sys.argv) < 3: - print("Error: Must pass desired nick and message as parameters") -else: - for try_again in (True,False): - irc = IRC("irc.freenode.net", sys.argv[1], "#podman") - err = irc.connect(*os.environ.get('IRCID', 'Big Bug').split(" ", 2)) - if err and try_again: - print("Trying again in 5 seconds...") - time.sleep(5) - continue - elif err: - break - irc.message(" ".join(sys.argv[2:])) - time.sleep(5.0) # avoid join/quit spam - irc.quit() - break diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh index 855866477..b406d7b5c 100755 --- a/contrib/cirrus/setup_environment.sh +++ b/contrib/cirrus/setup_environment.sh @@ -33,9 +33,6 @@ done # Sometimes environment setup needs to vary between distros # Note: This should only be used for environment variables, and temporary workarounds. -# Anything externally dependent, should be made fixed-in-time by adding to -# contrib/cirrus/packer/*_setup.sh to be incorporated into VM cache-images -# (see docs). cd "${GOSRC}/" case "${OS_RELEASE_ID}" in ubuntu) @@ -44,8 +41,6 @@ case "${OS_RELEASE_ID}" in # All SELinux distros need this for systemd-in-a-container setsebool container_manage_cgroup true - workaround_bfq_bug - if [[ "$ADD_SECOND_PARTITION" == "true" ]]; then bash "$SCRIPT_BASE/add_second_partition.sh" fi @@ -67,14 +62,14 @@ source "$SCRIPT_BASE/lib.sh" case "$CG_FS_TYPE" in tmpfs) warn "Forcing testing with runc instead of crun" - # On ubuntu, the default runc is usually not new enough. - if [[ "${OS_RELEASE_ID}" == "ubuntu" ]]; then - X=$(echo "export OCI_RUNTIME=/usr/lib/cri-o-runc/sbin/runc" | \ - tee -a /etc/environment) && eval "$X" && echo "$X" - else - X=$(echo "export OCI_RUNTIME=/usr/bin/runc" | \ - tee -a /etc/environment) && eval "$X" && echo "$X" - fi + # On ubuntu, the default runc is usually not new enough. + if [[ "$OS_RELEASE_ID" == "ubuntu" ]]; then + X=$(echo "export OCI_RUNTIME=/usr/lib/cri-o-runc/sbin/runc" | \ + tee -a /etc/environment) && eval "$X" && echo "$X" + else + X=$(echo "export OCI_RUNTIME=/usr/bin/runc" | \ + tee -a /etc/environment) && eval "$X" && echo "$X" + fi ;; cgroup2fs) # This is necessary since we've built/installed from source, which uses runc as the default. diff --git a/contrib/cirrus/success.sh b/contrib/cirrus/success.sh deleted file mode 100755 index 8783f6b81..000000000 --- a/contrib/cirrus/success.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env bash - -set -e - -source $(dirname $0)/lib.sh - -req_env_var CIRRUS_BRANCH CIRRUS_REPO_FULL_NAME CIRRUS_BASE_SHA CIRRUS_CHANGE_IN_REPO CIRRUS_CHANGE_MESSAGE - -cd $CIRRUS_WORKING_DIR - -if [[ "$CIRRUS_BRANCH" =~ "pull" ]] -then - echo "Retrieving latest HEADS and tags" - git fetch --all --tags - echo "Finding commit authors for PR $CIRRUS_PR" - unset NICKS - if [[ -r "$AUTHOR_NICKS_FILEPATH" ]] - then - SHARANGE="${CIRRUS_BASE_SHA}..${CIRRUS_CHANGE_IN_REPO}" - EXCLUDE_RE='merge-robot' - EMAILCSET='[:alnum:]-+_@.' - AUTHOR_NICKS=$(egrep -v '(^[[:space:]]*$)|(^[[:space:]]*#)' "$AUTHOR_NICKS_FILEPATH" | sort -u) - # Depending on branch-state, it's possible SHARANGE could be _WAY_ too big - MAX_NICKS=10 - # newline separated - GITLOG="git log --format='%ae'" - COMMIT_AUTHORS=$($GITLOG $SHARANGE || $GITLOG -1 HEAD | \ - tr --delete --complement "$EMAILCSET[:space:]" | \ - egrep -v "$EXCLUDE_RE" | \ - sort -u | \ - tail -$MAX_NICKS) - - for c_email in $COMMIT_AUTHORS - do - c_email=$(echo "$c_email" | tr --delete --complement "$EMAILCSET") - echo -e "\tExamining $c_email" - NICK=$(echo "$AUTHOR_NICKS" | grep -m 1 "$c_email" | \ - awk --field-separator ',' '{print $2}' | tr -d '[[:blank:]]') - if [[ -n "$NICK" ]] - then - echo -e "\t\tFound $c_email -> $NICK in $(basename $AUTHOR_NICKS_FILEPATH)" - else - echo -e "\t\tNot found in $(basename $AUTHOR_NICKS_FILEPATH), using e-mail username." - NICK=$(echo "$c_email" | cut -d '@' -f 1) - fi - if ! echo "$NICKS" | grep -q "$NICK" - then - echo -e "\tUsing nick $NICK" - NICKS="${NICKS:+$NICKS, }$NICK" - else - echo -e "\tNot re-adding duplicate nick $NICK" - fi - done - fi - - unset MENTION_PREFIX - [[ -z "$NICKS" ]] || \ - MENTION_PREFIX="$NICKS: " - - URL="https://github.com/$CIRRUS_REPO_FULL_NAME/pull/$CIRRUS_PR" - PR_SUBJECT=$(echo "$CIRRUS_CHANGE_MESSAGE" | head -1) - ircmsg "${MENTION_PREFIX}Cirrus-CI testing successful for PR '$PR_SUBJECT': $URL" -else - URL="https://cirrus-ci.com/github/containers/libpod/$CIRRUS_BRANCH" - ircmsg "Cirrus-CI testing branch $(basename $CIRRUS_BRANCH) successful: $URL" -fi diff --git a/docs/source/Tutorials.rst b/docs/source/Tutorials.rst index 33e4ae3d3..83818e3ae 100644 --- a/docs/source/Tutorials.rst +++ b/docs/source/Tutorials.rst @@ -6,7 +6,7 @@ Here are a number of useful tutorials to get you up and running with Podman. If * `Basic Setup and Use of Podman <https://github.com/containers/podman/blob/master/docs/tutorials/podman_tutorial.md>`_: Learn how to setup Podman and perform some basic commands with the utility. * `Basic Setup and Use of Podman in a Rootless environment <https://github.com/containers/podman/blob/master/docs/tutorials/rootless_tutorial.md>`_: The steps required to setup rootless Podman are enumerated. -* `Podman Mac Client tutorial <https://github.com/containers/podman/blob/master/docs/tutorials/mac_client.md>`_: Special setup for running the Podman remote client on a Mac and connecting to Podman running on a Linux VM are documented. +* `Podman Mac/Windows tutorial <https://github.com/containers/podman/blob/master/docs/tutorials/mac_win_client.md>`_: Special setup for running the Podman remote client on a Mac or Windows PC and connecting to Podman running on a Linux VM are documented. * `How to sign and distribute container images using Podman <https://github.com/containers/podman/blob/master/docs/tutorials/image_signing.md>`_: Learn how to setup and use image signing with Podman. * `Podman remote-client tutorial <https://github.com/containers/podman/blob/master/docs/tutorials/remote_client.md>`_: A brief how-to on using the Podman remote-client. * `How to use libpod for custom/derivative projects <https://github.com/containers/podman/blob/master/docs/tutorials/podman-derivative-api.md>`_: How the libpod API can be used within your own project. diff --git a/docs/source/markdown/podman-build.1.md b/docs/source/markdown/podman-build.1.md index 6618df1b9..c38424a11 100644 --- a/docs/source/markdown/podman-build.1.md +++ b/docs/source/markdown/podman-build.1.md @@ -23,6 +23,8 @@ When the URL is an Containerfile, the Containerfile is downloaded to a temporary When a Git repository is set as the URL, the repository is cloned locally and then set as the context. +NOTE: `podman build` uses code sourced from the `buildah` project to build container images. This `buildah` code creates `buildah` containers for the `RUN` options in container storage. In certain situations, when the `podman build` crashes or users kill the `podman build` process, these external containers can be left in container storage. Use the `podman ps --all --storage` command to see these contaienrs. External containers can be removed with the `podman rm --storage` command. + ## OPTIONS **--add-host**=*host* @@ -804,7 +806,7 @@ If you are using a useradd command within a Containerfile with a large UID/GID, If you are using `useradd` within your build script, you should pass the `--no-log-init or -l` option to the `useradd` command. This option tells useradd to stop creating the lastlog file. ## SEE ALSO -podman(1), buildah(1), containers-registries.conf(5), crun(8), runc(8), useradd(8) +podman(1), buildah(1), containers-registries.conf(5), crun(8), runc(8), useradd(8), podman-ps(1), podman-rm(1) ## HISTORY Aug 2020, Additional options and .dockerignore added by Dan Walsh <dwalsh@redhat.com> diff --git a/docs/source/markdown/podman-manifest-add.1.md b/docs/source/markdown/podman-manifest-add.1.md index 44815def5..c4d4417c4 100644 --- a/docs/source/markdown/podman-manifest-add.1.md +++ b/docs/source/markdown/podman-manifest-add.1.md @@ -33,6 +33,25 @@ the image. If *imageName* refers to a manifest list or image index, the architecture information will be retrieved from it. Otherwise, it will be retrieved from the image's configuration information. +**--authfile**=*path* + +Path of the authentication file. Default is ${XDG\_RUNTIME\_DIR}/containers/auth.json, which is set using `podman login`. +If the authorization state is not found there, $HOME/.docker/config.json is checked, which is set using `docker login`. (Not available for remote commands) + +Note: You can also override the default path of the authentication file by setting the REGISTRY\_AUTH\_FILE +environment variable. `export REGISTRY_AUTH_FILE=path` + +**--cert-dir**=*path* + +Use certificates at *path* (\*.crt, \*.cert, \*.key) to connect to the registry. +Default certificates directory is _/etc/containers/certs.d_. (Not available for remote commands) + +**--creds**=*creds* + +The [username[:password]] to use to authenticate with the registry if required. +If one or both values are not supplied, a command line prompt will appear and the +value can be entered. The password is entered without echo. + **--features** Specify the features list which the list or index records as requirements for @@ -50,6 +69,10 @@ configuration information. Specify the OS version which the list or index records as a requirement for the image. This option is rarely used. +**--tls-verify** + +Require HTTPS and verify certificates when talking to container registries (defaults to true). (Not available for remote commands) + **--variant** Specify the variant which the list or index records for the image. This option diff --git a/docs/source/markdown/podman-ps.1.md b/docs/source/markdown/podman-ps.1.md index 2f8112aab..58d3358e5 100644 --- a/docs/source/markdown/podman-ps.1.md +++ b/docs/source/markdown/podman-ps.1.md @@ -32,12 +32,18 @@ all the containers information. By default it lists: **--all**, **-a** -Show all the containers, default is only running containers +Show all the containers created by Podman, default is only running containers. + +Note: Podman shares containers storage with other tools such as Buildah and CRI-O. In some cases these `external` containers might also exist in the same storage. Use the `--storage` option to see these external containers. External containers show the 'storage' status. **--pod**, **-p** Display the pods the containers are associated with +**--storage** + +Display external containers that are not controlled by Podman but are stored in containers storage. These external containers are generally created via other container technology such as Buildah or CRI-O and may depend on the same container images that Podman is also using. External containers are denoted with either a 'buildah' or 'storage' in the COMMAND and STATUS column of the ps output. Only used with the --all option. + **--no-trunc** Display the extended information @@ -174,11 +180,20 @@ CONTAINER ID IMAGE COMMAND CREATED STATUS ``` +``` +$ podman ps --storage -a +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +69ed779d8ef9f redis:alpine "redis-server" 25 hours ago Created 6379/tcp k8s_container1_podsandbox1_redhat.test.crio_redhat-test-crio_1 +38a8a78596f9 docker.io/library/busybox:latest buildah 2 hours ago storage busybox-working-container +fd7b786b5c32 docker.io/library/alpine:latest buildah 2 hours ago storage alpine-working-container +f78620804e00 scratch buildah 2 hours ago storage working-container +``` + ## ps Print a list of containers ## SEE ALSO -podman(1) +podman(1), buildah(1), crio(8) ## HISTORY August 2017, Originally compiled by Urvashi Mohnani <umohnani@redhat.com> diff --git a/docs/source/markdown/podman-rm.1.md b/docs/source/markdown/podman-rm.1.md index cddf06e3e..990af0cd1 100644 --- a/docs/source/markdown/podman-rm.1.md +++ b/docs/source/markdown/podman-rm.1.md @@ -45,9 +45,9 @@ The latest option is not supported on the remote client. **--storage** -Remove the container from the storage library only. -This is only possible with containers that are not present in libpod (cannot be seen by **podman ps**). -It is used to remove containers from **podman build** and **buildah**, and orphan containers which were only partially removed by **podman rm**. +Remove external containers from the storage library. +This is only possible with containers that are not present in libpod can be seen by **podman ps --all --storage**). +It is used to remove external containers from **podman build** and **buildah**, and orphan containers which were only partially removed by **podman rm**. The storage option conflicts with the **--all**, **--latest**, and **--volumes** options. **--volumes**, **-v** @@ -96,7 +96,7 @@ $ podman rm -f --latest **125** The command fails for a reason other than container did not exist or is paused/running ## SEE ALSO -podman(1), podman-image-rm(1) +podman(1), podman-image-rm(1), podman-ps(1), podman-build(1) ## HISTORY August 2017, Originally compiled by Ryan Cole <rycole@redhat.com> diff --git a/docs/tutorials/README.md b/docs/tutorials/README.md index 4beb069ac..7f7b4853d 100644 --- a/docs/tutorials/README.md +++ b/docs/tutorials/README.md @@ -12,9 +12,9 @@ Learn how to setup Podman and perform some basic commands with the utility. The steps required to setup rootless Podman are enumerated. -**[Setup on OS X](mac_client.md)** +**[Setup Mac/Windows](mac_win_client.md) -Special setup for running the Podman remote client on a Mac and connecting to Podman running on a Linux VM are documented. +Special setup for running the Podman remote client on a Mac or Windows PC and connecting to Podman running on a Linux VM are documented. **[Remote Client](remote_client.md)** diff --git a/docs/tutorials/mac_client.md b/docs/tutorials/mac_client.md index f6c9160a8..f406ca54d 100644 --- a/docs/tutorials/mac_client.md +++ b/docs/tutorials/mac_client.md @@ -1,99 +1,2 @@ -# Podman Mac Client tutorial - -## What is the Podman Mac Client - -First and foremost, the Mac Client is under heavy development. We are working on getting the -Mac client to be packaged and run for a native-like experience. This is the setup tutorial -for the Mac client at its current stage of development and packaging. - -The purpose of the Mac client for Podman is to allow users to run Podman on a Mac. Since Podman is a Linux -container engine, The Mac client is actually a version of the [Podman-remote client](remote_client.md), -edited to that the client side works on a Mac machine, and connects to a Podman "backend" on a Linux -machine, virtual or physical. The goal is to have a native-like experience when working with the Mac -client, so the command line interface of the remote client is exactly the same as the regular Podman -commands with the exception of some flags and commands that do not apply to the Mac client. - -## What you need - -To use the Mac client, you will need a binary built for MacOS and a Podman "backend" on a Linux machine; -hereafter referred to as the Podman node. In this context, a Podman node is a Linux system with Podman -installed on it and the varlink service activated. You will also need to be able to ssh into this -system as a user with privileges to the varlink socket (more on this later). - -For best results, use the most recent version of MacOS - -## Getting the Mac client -The Mac client is available through [Homebrew](https://brew.sh/). -``` -$ brew cask install podman -``` - -## Setting up the client and Podman node connection - -To use the Mac client, you must perform some setup on both the Mac and Podman nodes. In this case, -the Mac node refers to the Mac on which Podman is being run; and the Podman node refers to where -Podman and its storage reside. - -### Connection settings -Your Linux box must have ssh enabled, and you must copy your Mac's public key from `~/.sconf sh/id.pub` to -`/root/.ssh/authorized_keys` on your Linux box using `ssh-copy-id` This allows for the use of SSH keys -for remote access. - -You may need to edit your `/etc/ssh/sshd_config` in your Linux machine as follows: -``` -PermitRootLogin yes -``` - -Use of SSH keys are strongly encouraged to ensure a secure login. However, if you wish to avoid ‘logging in’ every -time you run a Podman command, you may edit your `/etc/ssh/sshd_config` on your Linux machine as follows: -``` -PasswordAuthentication no -PermitRootLogin without-password -``` - -### Podman node setup -The Podman node must be running a Linux distribution that supports Podman and must have Podman (not the Mac -client) installed. You must also have root access to the node. Check if your system uses systemd: -``` -$cat /proc/1/comm -systemd -``` -If it does, then simply start the Podman varlink socket: -``` -$ sudo systemctl start io.podman.socket -$ sudo systemctl enable io.podman.socket -``` - -If your system cannot use systemd, then you can manually establish the varlink socket with the Podman -command: -``` -$ sudo podman --log-level debug varlink --timeout 0 unix://run/podman/io.podman -``` - -### Required permissions -For now, the Mac client requires that you be able to run a privileged Podman and have privileged ssh -access to the remote system. This limitation is being worked on. - -#### Running the remote client -There are three different ways to pass connection information into the client: flags, conf file, and -environment variables. All three require information on username and a remote host ip address. Most often, -your username should be root and you can obtain your remote-host-ip using `ip addr` - -To connect using flags, you can use -``` -$ podman --remote-host remote-host-ip --username root images -REPOSITORY TAG IMAGE ID CREATED SIZE -quay.io/podman/stable latest 9c1e323be87f 10 days ago 414 MB -localhost/test latest 4b8c27c343e1 4 weeks ago 253 MB -k8s.gcr.io/pause 3.1 da86e6ba6ca1 20 months ago 747 kB -``` -If the conf file is set up, you may simply use Podman as you would on the linux machine. Take a look at -[podman-remote.conf.5.md](https://github.com/containers/podman/blob/master/docs/podman-remote.conf.5.md) on how to use the conf file: - -``` -$ podman images -REPOSITORY TAG IMAGE ID CREATED SIZE -quay.io/podman/stable latest 9c1e323be87f 10 days ago 414 MB -localhost/test latest 4b8c27c343e1 4 weeks ago 253 MB -k8s.gcr.io/pause 3.1 da86e6ba6ca1 20 months ago 747 kB -``` +# [Podman Mac Client tutorial](https://github.com/containers/podman/blob/master/docs/tutorials/mac_win_client.md) +This tutorial has moved! You can find out how to set up Podman on MacOS (as well as Windows) [here](https://github.com/containers/podman/blob/master/docs/tutorials/mac_win_client.md) diff --git a/docs/tutorials/mac_win_client.md b/docs/tutorials/mac_win_client.md new file mode 100644 index 000000000..63830a5b1 --- /dev/null +++ b/docs/tutorials/mac_win_client.md @@ -0,0 +1,111 @@ +# Podman Remote clients for MacOS and Windows + +## Introduction + +The core Podman runtime environment can only run on Linux operating systems. But other operating systems can use the “remote client” to manage their containers to a Linux backend. This remote client is nearly identical to the standard Podman program. Certain functions that do not make sense for remote clients have been removed. For example, the “--latest” switch for container commands has been removed. + +### Brief architecture + +The remote client uses a client-server model. You need Podman installed on a Linux machine or VM that also has the SSH daemon running. On the local operating system, when you execute a Podman command, Podman connects to the server via SSH. It then connects to the Podman service by using systemd socket activation. The Podman commands are executed on the server. From the client's point of view, it seems like Podman runs locally. + +## Obtaining and installing Podman + +### Windows + +Installing the Windows Podman client begins by downloading the Podman windows installer. The windows installer is built with each Podman release and is downloadable from its [release description page](https://github.com/containers/podman/releases/latest). You can also build the installer from source using the `podman.msi` Makefile endpoint. + +Once you have downloaded the installer, simply double click the installer and Podman will be installed. The path is also set to put `podman` in the default user path. + +Podman must be run at a command prompt using the Windows ‘cmd” or powershell applications. + +### MacOS + +The Mac Client is available through [Homebrew](https://brew.sh/). You can download homebrew via the instructions on their site. Install podman using: +``` +$ brew install podman +``` + +## Creating the first connection + +### Enable the Podman service on the server machine. + +Before performing any Podman client commands, you must enable the podman.sock SystemD service on the Linux server. In these examples, we are running Podman as a normal, unprivileged user, also known as a rootless user. By default, the rootless socket listens at `/run/user/${UID}/podman/podman.sock`. You can enable this socket, permanently using the following command: +``` +$ systemctl --user enable podman.socket +``` +You will need to enable linger for this user in order for the socket to work when the user is not logged in. + +``` +$ sudo loginctl enable-linger $USER +``` + +You can verify that the socket is listening with a simple Podman command. + +``` +$ podman --remote info +host: + arch: amd64 + buildahVersion: 1.16.0-dev + cgroupVersion: v2 + conmon: + package: conmon-2.0.19-1.fc32.x86_64 +``` + +#### Enable sshd + +In order for the client to communicate with the server you need to enable and start the SSH daemon on your Linux machine, if it is not currently enabled. +``` +$ sudo systemctl enable -s sshd +``` + +#### Setting up SSH +Remote podman uses SSH to communicate between the client and server. The remote client works considerably smoother using SSH keys. To set up your ssh connection, you need to generate an ssh key pair from your client machine. +``` +$ ssh-keygen +``` +Your public key by default should be in your home directory under .ssh\id_rsa.pub. You then need to copy the contents of id_rsa.pub and append it into ~/.ssh/authorized_keys on the Linux server. On a Mac, you can automate this using ssh-copy-id. + +If you do not wish to use SSH keys, you will be prompted with each Podman command for your login password. + +## Using the client + +The first step in using the Podman remote client is to configure a connection.. + +You can add a connection by using the `podman system connection add` command. + +``` +C:\Users\baude> podman system connection add baude --identity c:\Users\baude\.ssh\id_rsa ssh://192.168.122.1/run/user/1000/podman/podman.sock +``` + +This will add a remote connection to Podman and if it is the first connection added, it will mark the connection as the default. You can observe your connections with `podman system connection list` + +``` +C:\Users\baude> podman system connection list +Name Identity URI +baude* id_rsa ssh://baude@192.168.122.1/run/user/1000/podman/podman.sock +``` + +Now we can test the connection with `podman info`. + +``` +C:\Users\baude> podman info +host: + arch: amd64 + buildahVersion: 1.16.0-dev + cgroupVersion: v2 + conmon: + package: conmon-2.0.19-1.fc32.x86_64 +``` + +Podman has also introduced a “--connection” flag where you can use other connections you have defined. If no connection is provided, the default connection will be used. + +``` +C:\Users\baude> podman system connection --help +``` + +## Wrap up + +You can use the podman remote clients to manage your containers running on a Linux server. The communication between client and server relies heavily on SSH connections and the use of SSH keys are encouraged. Once you have Podman installed on your remote client, you should set up a connection using `podman system connection add` which will then be used by subsequent Podman commands. + +## History +Originally published on [Red Hat Enable Sysadmin](https://www.redhat.com/sysadmin/podman-clients-macos-windows) diff --git a/docs/tutorials/podman_tutorial.md b/docs/tutorials/podman_tutorial.md index 97268fc41..85b95af04 100644 --- a/docs/tutorials/podman_tutorial.md +++ b/docs/tutorials/podman_tutorial.md @@ -5,7 +5,7 @@ Podman is a utility provided as part of the libpod library. It can be used to c containers. The following tutorial will teach you how to set up Podman and perform some basic commands with Podman. -If you are running on a Mac, you should instead follow the [Mac tutorial](https://github.com/containers/podman/blob/master/docs/tutorials/mac_client.md) +If you are running on a Mac or Windows PC, you should instead follow the [Mac and Windows tutorial](https://github.com/containers/podman/blob/master/docs/tutorials/mac_win_client.md) to set up the remote Podman client. **NOTE**: the code samples are intended to be run as a non-root user, and use `sudo` where diff --git a/docs/tutorials/remote_client.md b/docs/tutorials/remote_client.md index d4c43dda2..ad506d19a 100644 --- a/docs/tutorials/remote_client.md +++ b/docs/tutorials/remote_client.md @@ -1,88 +1,112 @@ # Podman remote-client tutorial -## What is the remote-client +## Introduction +The purpose of the Podman remote-client is to allow users to interact with a Podman "backend" while on a separate client. The command line interface of the remote client is exactly the same as the regular Podman commands with the exception of some flags being removed as they do not apply to the remote-client. -First and foremost, the remote-client is under heavy development. We are adding new -commands and functions frequently. We also are working on a rootless implementation that -does not require privileged users. +The remote client takes advantage of a client-server model. You need Podman installed on a Linux machine or VM that also has the SSH daemon running. On the local operating system, when you execute a Podman command, Podman connects to the server via SSH. It then connects to the Podman service by using systemd socket activation, and hitting our [Rest API](https://docs.podman.io/en/latest/_static/api.html). The Podman commands are executed on the server. From the client's point of view, it seems like Podman runs locally. -The purpose of the Podman remote-client is to allow users to interact with a Podman "backend" -while on a separate client. The command line interface of the remote client is exactly the -same as the regular Podman commands with the exception of some flags being removed as they -do not apply to the remote-client. +This tutorial is for running Podman remotely on Linux. If you are using a Mac or a Windows PC, please follow the [Mac and Windows tutorial](https://github.com/containers/podman/blob/master/docs/tutorials/mac_win_client.md) -## What you need -To use the remote-client, you will need a binary for your client and a Podman "backend"; hereafter -referred to as the Podman node. In this context, a Podman node is a Linux system with Podman -installed on it and the varlink service activated. You will also need to be able to ssh into this -system as a user with privileges to the varlink socket (more on this later). +## Obtaining and installing Podman -## Building the remote client -At this time, the Podman remote-client is not being packaged for any distribution. It must be built from -source. To set up your build environment, see [Installation notes](https://github.com/containers/podman/blob/master/install.md) and follow the -section [Building from scratch](https://github.com/containers/podman/blob/master/install.md#building-from-scratch). Once you can successfully -build the regular Podman binary, you can now build the remote-client. +### Client machine +You will need either Podman or the podman-remote client. The difference between the two is that the compiled podman-remote client can only act as a remote client connecting to a backend, while Podman can run local, standard Podman commands, as well as act as a remote client (using `podman --remote`) + +If you already have Podman installed, you do not need to install podman-remote. + +You can find out how to [install Podman here](https://podman.io/getting-started/installation) + +If you would like to install only the podman-remote client, it is downloadable from its [release description page](https://github.com/containers/podman/releases/latest). You can also build it from source using the `make podman-remote` + + +### Server Machine +You will need to [install Podman](https://podman.io/getting-started/installation) on your server machine. + + +## Creating the first connection + +### Enable the Podman service on the server machine. + +Before performing any Podman client commands, you must enable the podman.sock SystemD service on the Linux server. In these examples, we are running Podman as a normal, unprivileged user, also known as a rootless user. By default, the rootless socket listens at `/run/user/${UID}/podman/podman.sock`. You can enable this socket permanently using the following command: ``` -$ make podman-remote +$ systemctl --user enable podman.socket ``` -Like building the regular Podman, the resulting binary will be in the *bin* directory. This is the binary -you will run on the remote node later in the instructions. +You will need to enable linger for this user in order for the socket to work when the user is not logged in: -## Setting up the remote and Podman nodes +``` +$ sudo loginctl enable-linger $USER +``` +This is only required if you are not running Podman as root. -To use the remote-client, you must perform some setup on both the remote and Podman nodes. In this case, -the remote node refers to where the remote-client is being run; and the Podman node refers to where -Podman and its storage reside. +You can verify that the socket is listening with a simple Podman command. +``` +$ podman --remote info +host: + arch: amd64 + buildahVersion: 1.16.0-dev + cgroupVersion: v2 + conmon: + package: conmon-2.0.19-1.fc32.x86_64 +``` -### Podman node setup +#### Enable sshd -Varlink bridge support is provided by the varlink cli command and installed using: +In order for the Podman client to communicate with the server you need to enable and start the SSH daemon on your Linux machine, if it is not currently enabled. ``` -$ sudo dnf install varlink-cli +$ sudo systemctl enable -s sshd ``` -The Podman node must have Podman (not the remote-client) installed as normal. If your system uses systemd, -then simply start the Podman varlink socket. +#### Setting up SSH +Remote Podman uses SSH to communicate between the client and server. The remote client works considerably smoother using SSH keys. To set up your ssh connection, you need to generate an ssh key pair from your client machine. ``` -$ sudo systemctl start io.podman.socket +$ ssh-keygen ``` +Your public key by default should be in your home directory under ~/.ssh/id_rsa.pub. You then need to copy the contents of id_rsa.pub and append it into ~/.ssh/authorized_keys on the Linux server. You can automate this using ssh-copy-id. + +If you do not wish to use SSH keys, you will be prompted with each Podman command for your login password. + +## Using the client + +Note: `podman-remote` is equivalent to `podman --remote` here, depending on what you have chosen to install. + +The first step in using the Podman remote client is to configure a connection. + +You can add a connection by using the `podman-remote system connection add` command. -If your system cannot use systemd, then you can manually establish the varlink socket with the Podman -command: ``` -$ sudo podman --log-level debug varlink --timeout 0 unix://run/podman/io.podman +$ podman-remote system connection add myuser --identity ~/.ssh/id_rsa ssh://192.168.122.1/run/user/1000/podman/podman.sock ``` -### Required permissions -For now, the remote-client requires that you be able to run a privileged Podman and have privileged ssh -access to the remote system. This limitation is being worked on. +This will add a remote connection to Podman and if it is the first connection added, it will mark the connection as the default. You can observe your connections with `podman-remote system connection list`: -### Remote node setup - -#### Initiate an ssh session to the Podman node -To use the remote client, an ssh connection to the Podman server must be established. +``` +$ podman-remote system connection list +Name Identity URI +myuser* id_rsa ssh://myuser@192.168.122.1/run/user/1000/podman/podman.sock +``` -Using the varlink bridge, an ssh tunnel must be initiated to connect to the server. Podman must then be informed of the location of the sshd server on the targeted server +Now we can test the connection with `podman info`: ``` -$ export PODMAN_VARLINK_BRIDGE=$'ssh -T -p22 root@remotehost -- "varlink -A \'podman varlink \$VARLINK_ADDRESS\' bridge"' -$ bin/podman-remote images -REPOSITORY TAG IMAGE ID CREATED SIZE -docker.io/library/ubuntu latest 47b19964fb50 2 weeks ago 90.7 MB -docker.io/library/alpine latest caf27325b298 3 weeks ago 5.8 MB -quay.io/cevich/gcloud_centos latest 641dad61989a 5 weeks ago 489 MB -k8s.gcr.io/pause 3.1 da86e6ba6ca1 14 months ago 747 kB +$ podman-remote info +host: + arch: amd64 + buildahVersion: 1.16.0-dev + cgroupVersion: v2 + conmon: + package: conmon-2.0.19-1.fc32.x86_64 ``` -The PODMAN_VARLINK_BRIDGE variable may be added to your log in settings. It does not change per connection. +Podman-remote has also introduced a “--connection” flag where you can use other connections you have defined. If no connection is provided, the default connection will be used. -If coming from a Windows machine, the PODMAN_VARLINK_BRIDGE is formatted as: ``` -set PODMAN_VARLINK_BRIDGE=C:\Windows\System32\OpenSSH\ssh.exe -T -p22 root@remotehost -- varlink -A "podman varlink $VARLINK_ADDRESS" bridge +$ podman-remote system connection --help ``` -The arguments before the `--` are presented to ssh while the arguments after are for the varlink cli. The varlink arguments should be copied verbatim. - - `-p` is the port on the remote host for the ssh tunnel. `22` is the default. - - `root` is the currently supported user, while `remotehost` is the name or IP address of the host providing the Podman service. - - `-i` may be added to select an identity file. +## Wrap up + +You can use the Podman remote clients to manage your containers running on a Linux server. The communication between client and server relies heavily on SSH connections and the use of SSH keys are encouraged. Once you have Podman installed on your remote client, you should set up a connection using `podman-remote system connection add` which will then be used by subsequent Podman commands. + +## History +Adapted from the [Mac and Windows tutorial](https://github.com/containers/podman/blob/master/docs/tutorials/mac_win_client.md) diff --git a/docs/tutorials/varlink_remote_client.md b/docs/tutorials/varlink_remote_client.md new file mode 100644 index 000000000..54c648a48 --- /dev/null +++ b/docs/tutorials/varlink_remote_client.md @@ -0,0 +1,89 @@ +# Podman varlink remote-client tutorial [DEPRECATED] + +## What is the varlink client + +This API has been deprecated by the [REST API](https://docs.podman.io/en/latest/_static/api.html). +For usage on Windows and Mac, please reference the [Podman Mac/Windows tutorial](https://github.com/containers/podman/blob/master/docs/tutorials/mac_win_client.md) +Varlink support is in maintenance mode, and will be removed in a future release. +For more details, you can see [this blog](https://podman.io/blogs/2020/01/17/podman-new-api.html). + +The purpose of the Podman remote-client is to allow users to interact with a Podman "backend" +while on a separate client. The command line interface of the remote client is exactly the +same as the regular Podman commands with the exception of some flags being removed as they +do not apply to the remote-client. + +## What you need +To use the remote-client, you will need a binary for your client and a Podman "backend"; hereafter +referred to as the Podman node. In this context, a Podman node is a Linux system with Podman +installed on it and the varlink service activated. You will also need to be able to ssh into this +system as a user with privileges to the varlink socket (more on this later). + +## Building the remote client +At this time, the Podman remote-client is not being packaged for any distribution. It must be built from +source. To set up your build environment, see [Installation notes](https://github.com/containers/podman/blob/master/install.md) and follow the +section [Building from scratch](https://github.com/containers/podman/blob/master/install.md#building-from-scratch). Once you can successfully +build the regular Podman binary, you can now build the remote-client. +``` +$ make podman-remote +``` +Like building the regular Podman, the resulting binary will be in the *bin* directory. This is the binary +you will run on the remote node later in the instructions. + +## Setting up the remote and Podman nodes + +To use the remote-client, you must perform some setup on both the remote and Podman nodes. In this case, +the remote node refers to where the remote-client is being run; and the Podman node refers to where +Podman and its storage reside. + + +### Podman node setup + +Varlink bridge support is provided by the varlink cli command and installed using: +``` +$ sudo dnf install varlink-cli +``` + +The Podman node must have Podman (not the remote-client) installed as normal. If your system uses systemd, +then simply start the Podman varlink socket. +``` +$ sudo systemctl start io.podman.socket +``` + +If your system cannot use systemd, then you can manually establish the varlink socket with the Podman +command: +``` +$ sudo podman --log-level debug varlink --timeout 0 unix://run/podman/io.podman +``` + +### Required permissions +For now, the remote-client requires that you be able to run a privileged Podman and have privileged ssh +access to the remote system. This limitation is being worked on. + +### Remote node setup + +#### Initiate an ssh session to the Podman node +To use the remote client, an ssh connection to the Podman server must be established. + +Using the varlink bridge, an ssh tunnel must be initiated to connect to the server. Podman must then be informed of the location of the sshd server on the targeted server + +``` +$ export PODMAN_VARLINK_BRIDGE=$'ssh -T -p22 root@remotehost -- "varlink -A \'podman varlink \$VARLINK_ADDRESS\' bridge"' +$ bin/podman-remote images +REPOSITORY TAG IMAGE ID CREATED SIZE +docker.io/library/ubuntu latest 47b19964fb50 2 weeks ago 90.7 MB +docker.io/library/alpine latest caf27325b298 3 weeks ago 5.8 MB +quay.io/cevich/gcloud_centos latest 641dad61989a 5 weeks ago 489 MB +k8s.gcr.io/pause 3.1 da86e6ba6ca1 14 months ago 747 kB +``` + +The PODMAN_VARLINK_BRIDGE variable may be added to your log in settings. It does not change per connection. + +If coming from a Windows machine, the PODMAN_VARLINK_BRIDGE is formatted as: +``` +set PODMAN_VARLINK_BRIDGE=C:\Windows\System32\OpenSSH\ssh.exe -T -p22 root@remotehost -- varlink -A "podman varlink $VARLINK_ADDRESS" bridge +``` + +The arguments before the `--` are presented to ssh while the arguments after are for the varlink cli. The varlink arguments should be copied verbatim. + - `-p` is the port on the remote host for the ssh tunnel. `22` is the default. + - `root` is the currently supported user, while `remotehost` is the name or IP address of the host providing the Podman service. + - `-i` may be added to select an identity file. @@ -62,7 +62,7 @@ require ( golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a golang.org/x/sys v0.0.0-20200810151505-1b9f1253b3ed k8s.io/api v0.0.0-20190620084959-7cf5895f2711 - k8s.io/apimachinery v0.19.0 + k8s.io/apimachinery v0.19.1 k8s.io/client-go v0.0.0-20190620085101-78d2af792bab ) @@ -643,8 +643,8 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh k8s.io/api v0.0.0-20190620084959-7cf5895f2711 h1:BblVYz/wE5WtBsD/Gvu54KyBUTJMflolzc5I2DTvh50= k8s.io/api v0.0.0-20190620084959-7cf5895f2711/go.mod h1:TBhBqb1AWbBQbW3XRusr7n7E4v2+5ZY8r8sAMnyFC5A= k8s.io/apimachinery v0.0.0-20190612205821-1799e75a0719/go.mod h1:I4A+glKBHiTgiEjQiCCQfCAIcIMFGt291SmsvcrFzJA= -k8s.io/apimachinery v0.19.0 h1:gjKnAda/HZp5k4xQYjL0K/Yb66IvNqjthCb03QlKpaQ= -k8s.io/apimachinery v0.19.0/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA= +k8s.io/apimachinery v0.19.1 h1:cwsxZazM/LA9aUsBaL4bRS5ygoM6bYp8dFk22DSYQa4= +k8s.io/apimachinery v0.19.1/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA= k8s.io/client-go v0.0.0-20190620085101-78d2af792bab h1:E8Fecph0qbNsAbijJJQryKu4Oi9QTp5cVpjTE+nqg6g= k8s.io/client-go v0.0.0-20190620085101-78d2af792bab/go.mod h1:E95RaSlHr79aHaX0aGSwcPNfygDiPKOVXdmivCIZT0k= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= diff --git a/libpod/define/errors.go b/libpod/define/errors.go index f80b1d6e3..7714ebbf0 100644 --- a/libpod/define/errors.go +++ b/libpod/define/errors.go @@ -161,4 +161,8 @@ var ( // ErrNetworkOnPodContainer indicates the user wishes to alter network attributes on a container // in a pod. This cannot be done as the infra container has all the network information ErrNetworkOnPodContainer = errors.New("network cannot be configured when it is shared with a pod") + + // ErrStoreNotInitialized indicates that the container storage was never + // initilized. + ErrStoreNotInitialized = errors.New("the container storage was never initilized") ) diff --git a/libpod/image/image.go b/libpod/image/image.go index 9dd04e7c7..850a48eae 100644 --- a/libpod/image/image.go +++ b/libpod/image/image.go @@ -636,6 +636,14 @@ func (ir *Runtime) getImage(image string) (*storage.Image, error) { return img, nil } +func (ir *Runtime) ImageNames(id string) ([]string, error) { + myImage, err := ir.getImage(id) + if err != nil { + return nil, errors.Wrapf(err, "error getting image %s ", id) + } + return myImage.Names, nil +} + // GetImages retrieves all images present in storage func (ir *Runtime) GetImages() ([]*Image, error) { return ir.getImages(false) diff --git a/libpod/image/prune.go b/libpod/image/prune.go index 5a9ca5d8e..fcc65fb03 100644 --- a/libpod/image/prune.go +++ b/libpod/image/prune.go @@ -137,7 +137,7 @@ func (ir *Runtime) PruneImages(ctx context.Context, all bool, filter []string) ( } if err := p.Remove(ctx, true); err != nil { if errors.Cause(err) == storage.ErrImageUsedByContainer { - logrus.Warnf("Failed to prune image %s as it is in use: %v", p.ID(), err) + logrus.Warnf("Failed to prune image %s as it is in use: %v.\nA container associated with containers/storage i.e. Buildah, CRI-O, etc., maybe associated with this image.\nUsing the rmi command with the --force option will remove the container and image, but may cause failures for other dependent systems.", p.ID(), err) continue } return nil, errors.Wrap(err, "failed to prune image") diff --git a/libpod/runtime_ctr.go b/libpod/runtime_ctr.go index fa91fe002..936dce2e9 100644 --- a/libpod/runtime_ctr.go +++ b/libpod/runtime_ctr.go @@ -8,11 +8,13 @@ import ( "strings" "time" + "github.com/containers/buildah" "github.com/containers/common/pkg/config" "github.com/containers/podman/v2/libpod/define" "github.com/containers/podman/v2/libpod/events" "github.com/containers/podman/v2/pkg/cgroups" "github.com/containers/podman/v2/pkg/rootless" + "github.com/containers/storage" "github.com/containers/storage/pkg/stringid" "github.com/docker/go-units" spec "github.com/opencontainers/runtime-spec/specs-go" @@ -905,3 +907,34 @@ func (r *Runtime) PruneContainers(filterFuncs []ContainerFilter) (map[string]int } return prunedContainers, pruneErrors, nil } + +// StorageContainers returns a list of containers from containers/storage that +// are not currently known to Podman. +func (r *Runtime) StorageContainers() ([]storage.Container, error) { + + if r.store == nil { + return nil, define.ErrStoreNotInitialized + } + + storeContainers, err := r.store.Containers() + if err != nil { + return nil, errors.Wrapf(err, "error reading list of all storage containers") + } + retCtrs := []storage.Container{} + for _, container := range storeContainers { + exists, err := r.state.HasContainer(container.ID) + if err != nil && err != define.ErrNoSuchCtr { + return nil, errors.Wrapf(err, "failed to check if %s container exists in database", container.ID) + } + if exists { + continue + } + retCtrs = append(retCtrs, container) + } + + return retCtrs, nil +} + +func (r *Runtime) IsBuildahContainer(id string) (bool, error) { + return buildah.IsContainer(id, r.store) +} diff --git a/pkg/api/handlers/compat/containers.go b/pkg/api/handlers/compat/containers.go index 1ae6a990b..b1ef08cda 100644 --- a/pkg/api/handlers/compat/containers.go +++ b/pkg/api/handlers/compat/containers.go @@ -5,6 +5,7 @@ import ( "fmt" "net/http" "strings" + "syscall" "github.com/containers/podman/v2/libpod" "github.com/containers/podman/v2/libpod/define" @@ -169,16 +170,16 @@ func KillContainer(w http.ResponseWriter, r *http.Request) { return } - err = con.Kill(uint(sig)) + signal := uint(sig) + + err = con.Kill(signal) if err != nil { utils.Error(w, "Something went wrong.", http.StatusInternalServerError, errors.Wrapf(err, "unable to kill Container %s", name)) } - if utils.IsLibpodRequest(r) { - // the kill behavior for docker differs from podman in that they appear to wait - // for the Container to croak so the exit code is accurate immediately after the - // kill is sent. libpod does not. but we can add a wait here only for the docker - // side of things and mimic that behavior + // Docker waits for the container to stop if the signal is 0 or + // SIGKILL. + if !utils.IsLibpodRequest(r) && (signal == 0 || syscall.Signal(signal) == syscall.SIGKILL) { if _, err = con.Wait(); err != nil { utils.Error(w, "Something went wrong.", http.StatusInternalServerError, errors.Wrapf(err, "failed to wait for Container %s", con.ID())) return diff --git a/pkg/api/handlers/compat/networks.go b/pkg/api/handlers/compat/networks.go index 80b7505df..87b947549 100644 --- a/pkg/api/handlers/compat/networks.go +++ b/pkg/api/handlers/compat/networks.go @@ -5,6 +5,7 @@ import ( "net" "net/http" "os" + "strings" "syscall" "time" @@ -177,9 +178,11 @@ func ListNetworks(w http.ResponseWriter, r *http.Request) { utils.InternalServerError(w, err) return } + + filterNames, nameFilterExists := query.Filters["name"] // TODO remove when filters are implemented - if len(query.Filters) > 0 { - utils.InternalServerError(w, errors.New("filters for listing networks is not implemented")) + if (!nameFilterExists && len(query.Filters) > 0) || len(query.Filters) > 1 { + utils.InternalServerError(w, errors.New("only the name filter for listing networks is implemented")) return } netNames, err := network.GetNetworkNamesFromFileSystem(config) @@ -187,6 +190,21 @@ func ListNetworks(w http.ResponseWriter, r *http.Request) { utils.InternalServerError(w, err) return } + + // filter by name + if nameFilterExists { + names := []string{} + for _, name := range netNames { + for _, filter := range filterNames { + if strings.Contains(name, filter) { + names = append(names, name) + break + } + } + } + netNames = names + } + reports := make([]*types.NetworkResource, 0, len(netNames)) for _, name := range netNames { report, err := getNetworkResourceByName(name, runtime) diff --git a/pkg/api/handlers/libpod/networks.go b/pkg/api/handlers/libpod/networks.go index 475522664..dfece2a4e 100644 --- a/pkg/api/handlers/libpod/networks.go +++ b/pkg/api/handlers/libpod/networks.go @@ -42,7 +42,21 @@ func CreateNetwork(w http.ResponseWriter, r *http.Request) { } func ListNetworks(w http.ResponseWriter, r *http.Request) { runtime := r.Context().Value("runtime").(*libpod.Runtime) - options := entities.NetworkListOptions{} + decoder := r.Context().Value("decoder").(*schema.Decoder) + query := struct { + Filter string `schema:"filter"` + }{ + // override any golang type defaults + } + if err := decoder.Decode(&query, r.URL.Query()); err != nil { + utils.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest, + errors.Wrapf(err, "Failed to parse parameters for %s", r.URL.String())) + return + } + + options := entities.NetworkListOptions{ + Filter: query.Filter, + } ic := abi.ContainerEngine{Libpod: runtime} reports, err := ic.NetworkList(r.Context(), options) if err != nil { diff --git a/pkg/api/handlers/libpod/pods.go b/pkg/api/handlers/libpod/pods.go index 8f8292567..82a7299b2 100644 --- a/pkg/api/handlers/libpod/pods.go +++ b/pkg/api/handlers/libpod/pods.go @@ -327,7 +327,7 @@ func PodTop(w http.ResponseWriter, r *http.Request) { name := utils.GetName(r) pod, err := runtime.LookupPod(name) if err != nil { - utils.ContainerNotFound(w, name, err) + utils.PodNotFound(w, name, err) return } diff --git a/pkg/api/server/register_networks.go b/pkg/api/server/register_networks.go index 7918ad4a2..61916eedf 100644 --- a/pkg/api/server/register_networks.go +++ b/pkg/api/server/register_networks.go @@ -61,6 +61,11 @@ func (s *APIServer) registerNetworkHandlers(r *mux.Router) error { // - networks (compat) // summary: List networks // description: Display summary of network configurations + // parameters: + // - in: query + // name: filters + // type: string + // description: JSON encoded value of the filters (a map[string][]string) to process on the networks list. Only the name filter is supported. // produces: // - application/json // responses: @@ -106,7 +111,7 @@ func (s *APIServer) registerNetworkHandlers(r *mux.Router) error { // required: true // description: the name of the network // - in: query - // name: Force + // name: force // type: boolean // description: remove containers associated with network // produces: @@ -152,6 +157,11 @@ func (s *APIServer) registerNetworkHandlers(r *mux.Router) error { // - networks // summary: List networks // description: Display summary of network configurations + // parameters: + // - in: query + // name: filter + // type: string + // description: Provide filter values (e.g. 'name=podman') // produces: // - application/json // responses: diff --git a/pkg/bindings/network/network.go b/pkg/bindings/network/network.go index fd1111282..d8dc7e352 100644 --- a/pkg/bindings/network/network.go +++ b/pkg/bindings/network/network.go @@ -70,7 +70,7 @@ func Remove(ctx context.Context, nameOrID string, force *bool) ([]*entities.Netw } // List returns a summary of all CNI network configurations -func List(ctx context.Context) ([]*entities.NetworkListReport, error) { +func List(ctx context.Context, options entities.NetworkListOptions) ([]*entities.NetworkListReport, error) { var ( netList []*entities.NetworkListReport ) @@ -78,7 +78,11 @@ func List(ctx context.Context) ([]*entities.NetworkListReport, error) { if err != nil { return nil, err } - response, err := conn.DoRequest(nil, http.MethodGet, "/networks/json", nil, nil) + params := url.Values{} + if options.Filter != "" { + params.Set("filter", options.Filter) + } + response, err := conn.DoRequest(nil, http.MethodGet, "/networks/json", params, nil) if err != nil { return netList, err } diff --git a/pkg/domain/entities/containers.go b/pkg/domain/entities/containers.go index c8894300b..16997cdd1 100644 --- a/pkg/domain/entities/containers.go +++ b/pkg/domain/entities/containers.go @@ -282,6 +282,7 @@ type ContainerListOptions struct { Quiet bool Size bool Sort string + Storage bool Sync bool Watch uint } diff --git a/pkg/domain/entities/manifest.go b/pkg/domain/entities/manifest.go index 853619b19..01180951a 100644 --- a/pkg/domain/entities/manifest.go +++ b/pkg/domain/entities/manifest.go @@ -9,14 +9,19 @@ type ManifestCreateOptions struct { } type ManifestAddOptions struct { - All bool `json:"all" schema:"all"` - Annotation []string `json:"annotation" schema:"annotation"` - Arch string `json:"arch" schema:"arch"` - Features []string `json:"features" schema:"features"` - Images []string `json:"images" schema:"images"` - OS string `json:"os" schema:"os"` - OSVersion string `json:"os_version" schema:"os_version"` - Variant string `json:"variant" schema:"variant"` + All bool `json:"all" schema:"all"` + Annotation []string `json:"annotation" schema:"annotation"` + Arch string `json:"arch" schema:"arch"` + Authfile string `json:"-" schema:"-"` + CertDir string `json:"-" schema:"-"` + Features []string `json:"features" schema:"features"` + Images []string `json:"images" schema:"images"` + OS string `json:"os" schema:"os"` + OSVersion string `json:"os_version" schema:"os_version"` + Password string `json:"-" schema:"-"` + SkipTLSVerify types.OptionalBool `json:"-" schema:"-"` + Username string `json:"-" schema:"-"` + Variant string `json:"variant" schema:"variant"` } type ManifestAnnotateOptions struct { diff --git a/pkg/domain/infra/abi/containers.go b/pkg/domain/infra/abi/containers.go index 0537942e6..21618f555 100644 --- a/pkg/domain/infra/abi/containers.go +++ b/pkg/domain/infra/abi/containers.go @@ -798,6 +798,9 @@ func (ic *ContainerEngine) ContainerStart(ctx context.Context, namesOrIds []stri } func (ic *ContainerEngine) ContainerList(ctx context.Context, options entities.ContainerListOptions) ([]entities.ListContainer, error) { + if options.Latest { + options.Last = 1 + } return ps.GetContainerLists(ic.Libpod, options) } diff --git a/pkg/domain/infra/abi/containers_runlabel.go b/pkg/domain/infra/abi/containers_runlabel.go index ab2316d47..3983ba3a8 100644 --- a/pkg/domain/infra/abi/containers_runlabel.go +++ b/pkg/domain/infra/abi/containers_runlabel.go @@ -36,6 +36,11 @@ func (ic *ContainerEngine) ContainerRunlabel(ctx context.Context, label string, return err } + if options.Display { + fmt.Printf("command: %s\n", strings.Join(append([]string{os.Args[0]}, cmd[1:]...), " ")) + return nil + } + stdErr := os.Stderr stdOut := os.Stdout stdIn := os.Stdin diff --git a/pkg/domain/infra/abi/manifest.go b/pkg/domain/infra/abi/manifest.go index 6f3c6b902..55f73bf65 100644 --- a/pkg/domain/infra/abi/manifest.go +++ b/pkg/domain/infra/abi/manifest.go @@ -102,7 +102,24 @@ func (ir *ImageEngine) ManifestAdd(ctx context.Context, opts entities.ManifestAd } manifestAddOpts.Annotation = annotations } - listID, err := listImage.AddManifest(*ir.Libpod.SystemContext(), manifestAddOpts) + + // Set the system context. + sys := ir.Libpod.SystemContext() + if sys != nil { + sys = &types.SystemContext{} + } + sys.AuthFilePath = opts.Authfile + sys.DockerInsecureSkipTLSVerify = opts.SkipTLSVerify + sys.DockerCertPath = opts.CertDir + + if opts.Username != "" && opts.Password != "" { + sys.DockerAuthConfig = &types.DockerAuthConfig{ + Username: opts.Username, + Password: opts.Password, + } + } + + listID, err := listImage.AddManifest(*sys, manifestAddOpts) if err != nil { return listID, err } @@ -191,6 +208,7 @@ func (ir *ImageEngine) ManifestPush(ctx context.Context, names []string, opts en } sys.AuthFilePath = opts.Authfile sys.DockerInsecureSkipTLSVerify = opts.SkipTLSVerify + sys.DockerCertPath = opts.CertDir if opts.Username != "" && opts.Password != "" { sys.DockerAuthConfig = &types.DockerAuthConfig{ diff --git a/pkg/domain/infra/tunnel/network.go b/pkg/domain/infra/tunnel/network.go index 2b197cac0..074425087 100644 --- a/pkg/domain/infra/tunnel/network.go +++ b/pkg/domain/infra/tunnel/network.go @@ -8,7 +8,7 @@ import ( ) func (ic *ContainerEngine) NetworkList(ctx context.Context, options entities.NetworkListOptions) ([]*entities.NetworkListReport, error) { - return network.List(ic.ClientCxt) + return network.List(ic.ClientCxt, options) } func (ic *ContainerEngine) NetworkInspect(ctx context.Context, namesOrIds []string, options entities.NetworkInspectOptions) ([]entities.NetworkInspectReport, error) { diff --git a/pkg/ps/ps.go b/pkg/ps/ps.go index 4c5f60844..8087507e2 100644 --- a/pkg/ps/ps.go +++ b/pkg/ps/ps.go @@ -14,6 +14,7 @@ import ( lpfilters "github.com/containers/podman/v2/libpod/filters" "github.com/containers/podman/v2/pkg/domain/entities" psdefine "github.com/containers/podman/v2/pkg/ps/define" + "github.com/containers/storage" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -54,12 +55,12 @@ func GetContainerLists(runtime *libpod.Runtime, options entities.ContainerListOp return nil, err } if options.Last > 0 { - // Sort the containers we got + // Sort the libpod containers sort.Sort(SortCreateTime{SortContainers: cons}) // we should perform the lopping before we start getting // the expensive information on containers if options.Last < len(cons) { - cons = cons[len(cons)-options.Last:] + cons = cons[:options.Last] } } for _, con := range cons { @@ -68,7 +69,31 @@ func GetContainerLists(runtime *libpod.Runtime, options entities.ContainerListOp return nil, err } pss = append(pss, listCon) + } + + if options.All && options.Storage { + externCons, err := runtime.StorageContainers() + if err != nil { + return nil, err + } + + for _, con := range externCons { + listCon, err := ListStorageContainer(runtime, con, options) + if err != nil { + return nil, err + } + pss = append(pss, listCon) + } + } + + // Sort the containers we got + sort.Sort(SortPSCreateTime{SortPSContainers: pss}) + if options.Last > 0 { + // only return the "last" containers caller requested + if options.Last < len(pss) { + pss = pss[:options.Last] + } } return pss, nil } @@ -199,6 +224,48 @@ func ListContainerBatch(rt *libpod.Runtime, ctr *libpod.Container, opts entities return ps, nil } +func ListStorageContainer(rt *libpod.Runtime, ctr storage.Container, opts entities.ContainerListOptions) (entities.ListContainer, error) { + name := "unknown" + if len(ctr.Names) > 0 { + name = ctr.Names[0] + } + + ps := entities.ListContainer{ + ID: ctr.ID, + Created: ctr.Created.Unix(), + ImageID: ctr.ImageID, + State: "storage", + Names: []string{name}, + } + + buildahCtr, err := rt.IsBuildahContainer(ctr.ID) + if err != nil { + return ps, errors.Wrapf(err, "error determining buildah container for container %s", ctr.ID) + } + + if buildahCtr { + ps.Command = []string{"buildah"} + } else { + ps.Command = []string{"storage"} + } + + imageName := "" + if ctr.ImageID != "" { + names, err := rt.ImageRuntime().ImageNames(ctr.ImageID) + if err != nil { + return ps, err + } + if len(names) > 0 { + imageName = names[0] + } + } else if buildahCtr { + imageName = "scratch" + } + + ps.Image = imageName + return ps, nil +} + func getNamespaceInfo(path string) (string, error) { val, err := os.Readlink(path) if err != nil { @@ -223,5 +290,17 @@ func (a SortContainers) Swap(i, j int) { a[i], a[j] = a[j], a[i] } type SortCreateTime struct{ SortContainers } func (a SortCreateTime) Less(i, j int) bool { - return a.SortContainers[i].CreatedTime().Before(a.SortContainers[j].CreatedTime()) + return a.SortContainers[i].CreatedTime().After(a.SortContainers[j].CreatedTime()) +} + +// SortPSContainers helps us set-up ability to sort by createTime +type SortPSContainers []entities.ListContainer + +func (a SortPSContainers) Len() int { return len(a) } +func (a SortPSContainers) Swap(i, j int) { a[i], a[j] = a[j], a[i] } + +type SortPSCreateTime struct{ SortPSContainers } + +func (a SortPSCreateTime) Less(i, j int) bool { + return a.SortPSContainers[i].Created > a.SortPSContainers[j].Created } diff --git a/pkg/specgen/generate/oci.go b/pkg/specgen/generate/oci.go index fd324c6e1..b57ddf1aa 100644 --- a/pkg/specgen/generate/oci.go +++ b/pkg/specgen/generate/oci.go @@ -353,6 +353,9 @@ func SpecGenToOCI(ctx context.Context, s *specgen.SpecGenerator, rt *libpod.Runt configSpec.Annotations[define.InspectAnnotationInit] = define.InspectResponseFalse } + if s.OOMScoreAdj != nil { + g.SetProcessOOMScoreAdj(*s.OOMScoreAdj) + } setProcOpts(s, &g) return configSpec, nil diff --git a/pkg/systemd/generate/containers.go b/pkg/systemd/generate/containers.go index caf5de357..a4fdae46e 100644 --- a/pkg/systemd/generate/containers.go +++ b/pkg/systemd/generate/containers.go @@ -220,6 +220,9 @@ func executeContainerTemplate(info *containerInfo, options entities.GenerateSyst case "--replace": hasReplaceParam = true } + if strings.HasPrefix(p, "--name=") { + hasNameParam = true + } } if !hasDetachParam { diff --git a/test/apiv2/35-networks.at b/test/apiv2/35-networks.at index 4c032c072..143d6c07b 100644 --- a/test/apiv2/35-networks.at +++ b/test/apiv2/35-networks.at @@ -21,6 +21,27 @@ if root; then t POST libpod/networks/create '"Subnet":{"IP":"10.10.1.0","Mask":[0,255,255,0]}' 500 \ .cause~'.*mask is invalid' + # network list + t GET libpod/networks/json 200 + t GET libpod/networks/json?filter=name=network1 200 \ + length=1 \ + .[0].Name=network1 + t GET networks 200 + + #network list docker endpoint + #filters={"name":["network1","network2"]} + t GET networks?filters=%7B%22name%22%3A%5B%22network1%22%2C%22network2%22%5D%7D 200 \ + length=2 + #filters={"name":["network"]} + t GET networks?filters=%7B%22name%22%3A%5B%22network%22%5D%7D 200 \ + length=2 + # invalid filter filters={"label":"abc"} + t GET networks?filters=%7B%22label%22%3A%5B%22abc%22%5D%7D 500 \ + .cause="only the name filter for listing networks is implemented" + # invalid filter filters={"label":"abc","name":["network"]} + t GET networks?filters=%7B%22label%22%3A%22abc%22%2C%22name%22%3A%5B%22network%22%5D%7D 500 \ + .cause="only the name filter for listing networks is implemented" + # clean the network t DELETE libpod/networks/network1 200 \ .[0].Name~network1 \ diff --git a/test/apiv2/40-pods.at b/test/apiv2/40-pods.at index 3df541de5..fdb61a84d 100644 --- a/test/apiv2/40-pods.at +++ b/test/apiv2/40-pods.at @@ -102,6 +102,10 @@ t GET libpod/pods/stats?namesOrIDs=fakename 404 \ t DELETE libpod/pods/bar?force=true 200 +# test the fake name +t GET libpod/pods/fakename/top 404 \ + .cause="no such pod" + t GET libpod/pods/foo/top 200 \ .Processes[0][-1]="/pause " \ .Titles[-1]="COMMAND" diff --git a/test/e2e/build_test.go b/test/e2e/build_test.go index 9fd82e149..0b6e919d0 100644 --- a/test/e2e/build_test.go +++ b/test/e2e/build_test.go @@ -57,6 +57,29 @@ var _ = Describe("Podman build", func() { Expect(session.ExitCode()).To(Equal(0)) }) + It("podman build with logfile", func() { + SkipIfRemote() + logfile := filepath.Join(podmanTest.TempDir, "logfile") + session := podmanTest.PodmanNoCache([]string{"build", "--tag", "test", "--logfile", logfile, "build/basicalpine"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + + // Verify that OS and Arch are being set + inspect := podmanTest.PodmanNoCache([]string{"inspect", "test"}) + inspect.WaitWithDefaultTimeout() + data := inspect.InspectImageJSON() + Expect(data[0].Os).To(Equal(runtime.GOOS)) + Expect(data[0].Architecture).To(Equal(runtime.GOARCH)) + + st, err := os.Stat(logfile) + Expect(err).To(BeNil()) + Expect(st.Size()).To(Not(Equal(0))) + + session = podmanTest.PodmanNoCache([]string{"rmi", "alpine"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + }) + // If the context directory is pointing at a file and not a directory, // that's a no no, fail out. It("podman build context directory a file", func() { diff --git a/test/e2e/generate_systemd_test.go b/test/e2e/generate_systemd_test.go index cd3ee6e0a..da2f67754 100644 --- a/test/e2e/generate_systemd_test.go +++ b/test/e2e/generate_systemd_test.go @@ -189,7 +189,7 @@ var _ = Describe("Podman generate systemd", func() { Expect(found).To(BeTrue()) }) - It("podman generate systemd --new", func() { + It("podman generate systemd --new --name foo", func() { n := podmanTest.Podman([]string{"create", "--name", "foo", "alpine", "top"}) n.WaitWithDefaultTimeout() Expect(n.ExitCode()).To(Equal(0)) @@ -202,6 +202,29 @@ var _ = Describe("Podman generate systemd", func() { found, _ := session.GrepString("# container-foo.service") Expect(found).To(BeTrue()) + found, _ = session.GrepString(" --replace ") + Expect(found).To(BeTrue()) + + found, _ = session.GrepString("stop --ignore --cidfile %t/container-foo.ctr-id -t 42") + Expect(found).To(BeTrue()) + }) + + It("podman generate systemd --new --name=foo", func() { + n := podmanTest.Podman([]string{"create", "--name=foo", "alpine", "top"}) + n.WaitWithDefaultTimeout() + Expect(n.ExitCode()).To(Equal(0)) + + session := podmanTest.Podman([]string{"generate", "systemd", "-t", "42", "--name", "--new", "foo"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + + // Grepping the output (in addition to unit tests) + found, _ := session.GrepString("# container-foo.service") + Expect(found).To(BeTrue()) + + found, _ = session.GrepString(" --replace ") + Expect(found).To(BeTrue()) + found, _ = session.GrepString("stop --ignore --cidfile %t/container-foo.ctr-id -t 42") Expect(found).To(BeTrue()) }) diff --git a/test/e2e/network_test.go b/test/e2e/network_test.go index f427afa67..55f12f16a 100644 --- a/test/e2e/network_test.go +++ b/test/e2e/network_test.go @@ -1,5 +1,3 @@ -// +build !remote - package integration import ( diff --git a/test/e2e/ps_test.go b/test/e2e/ps_test.go index a734d399d..a2338c924 100644 --- a/test/e2e/ps_test.go +++ b/test/e2e/ps_test.go @@ -104,11 +104,13 @@ var _ = Describe("Podman ps", func() { SkipIfRemote() _, ec, _ := podmanTest.RunLsContainer("") Expect(ec).To(Equal(0)) + _, ec, _ = podmanTest.RunLsContainer("") + Expect(ec).To(Equal(0)) - result := podmanTest.Podman([]string{"ps", "--latest"}) + result := podmanTest.Podman([]string{"ps", "-q", "--latest"}) result.WaitWithDefaultTimeout() Expect(result.ExitCode()).To(Equal(0)) - Expect(len(result.OutputToStringArray())).Should(BeNumerically(">", 0)) + Expect(len(result.OutputToStringArray())).Should(Equal(1)) }) It("podman ps last flag", func() { diff --git a/test/e2e/run_test.go b/test/e2e/run_test.go index 1ac753201..a67f7df92 100644 --- a/test/e2e/run_test.go +++ b/test/e2e/run_test.go @@ -325,10 +325,10 @@ USER bin` Expect(session.ExitCode()).To(Equal(0)) } - session = podmanTest.Podman([]string{"run", "--rm", "--oom-score-adj=100", fedoraMinimal, "cat", "/proc/self/oom_score_adj"}) + session = podmanTest.Podman([]string{"run", "--rm", "--oom-score-adj=111", fedoraMinimal, "cat", "/proc/self/oom_score_adj"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) - Expect(session.OutputToString()).To(ContainSubstring("100")) + Expect(session.OutputToString()).To(Equal("111")) }) It("podman run limits host test", func() { diff --git a/test/e2e/runlabel_test.go b/test/e2e/runlabel_test.go index de79b2b98..f17b4d560 100644 --- a/test/e2e/runlabel_test.go +++ b/test/e2e/runlabel_test.go @@ -72,6 +72,21 @@ var _ = Describe("podman container runlabel", func() { result.WaitWithDefaultTimeout() Expect(result.ExitCode()).To(Equal(0)) }) + + It("podman container runlabel --display", func() { + SkipIfRemote() + image := "podman-runlabel-test:ls" + podmanTest.BuildImage(LsDockerfile, image, "false") + + result := podmanTest.Podman([]string{"container", "runlabel", "--display", "RUN", image}) + result.WaitWithDefaultTimeout() + Expect(result.ExitCode()).To(Equal(0)) + Expect(result.OutputToString()).To(ContainSubstring(podmanTest.PodmanBinary + " -la")) + + result = podmanTest.Podman([]string{"rmi", image}) + result.WaitWithDefaultTimeout() + Expect(result.ExitCode()).To(Equal(0)) + }) It("podman container runlabel bogus label should result in non-zero exit code", func() { result := podmanTest.Podman([]string{"container", "runlabel", "RUN", ALPINE}) result.WaitWithDefaultTimeout() diff --git a/test/system/130-kill.bats b/test/system/130-kill.bats index 05090f852..c16e64c58 100644 --- a/test/system/130-kill.bats +++ b/test/system/130-kill.bats @@ -6,8 +6,6 @@ load helpers @test "podman kill - test signal handling in containers" { - skip_if_remote "FIXME: pending #7135" - # podman-remote and crun interact poorly in f31: crun seems to gobble up # some signals. # Workaround: run 'env --default-signal sh' instead of just 'sh' in diff --git a/vendor/modules.txt b/vendor/modules.txt index 9dc56b08c..4edb940a3 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -701,7 +701,7 @@ gopkg.in/yaml.v3 # k8s.io/api v0.0.0-20190620084959-7cf5895f2711 k8s.io/api/apps/v1 k8s.io/api/core/v1 -# k8s.io/apimachinery v0.19.0 +# k8s.io/apimachinery v0.19.1 k8s.io/apimachinery/pkg/api/errors k8s.io/apimachinery/pkg/api/resource k8s.io/apimachinery/pkg/apis/meta/v1 |