diff options
35 files changed, 626 insertions, 324 deletions
diff --git a/.cirrus.yml b/.cirrus.yml index f49b1d312..8d915fbfe 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -62,6 +62,10 @@ env: GCE_SSH_USERNAME: cirrus-ci # Name where this repositories cloud resources are located GCP_PROJECT_ID: ENCRYPTED[7c80e728e046b1c76147afd156a32c1c57d4a1ac1eab93b7e68e718c61ca8564fc61fef815952b8ae0a64e7034b8fe4f] + RELEASE_GCPJSON: ENCRYPTED[789d8f7e9a5972ce350fd8e60f1032ccbf4a35c3938b604774b711aad280e12c21faf10e25af1e0ba33597ffb9e39e46] + RELEASE_GCPNAME: ENCRYPTED[417d50488a4bd197bcc925ba6574de5823b97e68db1a17e3a5fde4bcf26576987345e75f8d9ea1c15a156b4612c072a1] + RELEASE_GCPROJECT: ENCRYPTED[7c80e728e046b1c76147afd156a32c1c57d4a1ac1eab93b7e68e718c61ca8564fc61fef815952b8ae0a64e7034b8fe4f] + # Default VM to use unless set or modified by task @@ -339,9 +343,8 @@ testing_task: unit_test_script: '$SCRIPT_BASE/unit_test.sh |& ${TIMESTAMP}' integration_test_script: '$SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP}' system_test_script: '$SCRIPT_BASE/system_test.sh |& ${TIMESTAMP}' - cache_release_archive_script: >- - [[ "$TEST_REMOTE_CLIENT" == "false" ]] || \ - $SCRIPT_BASE/cache_release_archive.sh |& ${TIMESTAMP} + build_release_script: '$SCRIPT_BASE/build_release.sh |& ${TIMESTAMP}' + upload_release_archive_script: '$SCRIPT_BASE/upload_release_archive.sh |& ${TIMESTAMP}' on_failure: failed_branch_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_branch_failure.sh' @@ -379,9 +382,6 @@ testing_crun_task: unit_test_script: '$SCRIPT_BASE/unit_test.sh |& ${TIMESTAMP}' integration_test_script: '$SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP}' system_test_script: '$SCRIPT_BASE/system_test.sh |& ${TIMESTAMP}' - cache_release_archive_script: >- - [[ "$TEST_REMOTE_CLIENT" == "false" ]] || \ - $SCRIPT_BASE/cache_release_archive.sh |& ${TIMESTAMP} on_failure: failed_branch_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_branch_failure.sh' @@ -462,14 +462,15 @@ special_testing_cross_task: env: matrix: - SPECIALMODE: 'windows' # See docs - SPECIALMODE: 'darwin' + CROSS_PLATFORM: 'windows' + CROSS_PLATFORM: 'darwin' timeout_in: 20m networking_script: '${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/networking.sh' setup_environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}' - cache_release_archive_script: '$SCRIPT_BASE/cache_release_archive.sh |& ${TIMESTAMP}' + build_release_script: '$SCRIPT_BASE/build_release.sh |& ${TIMESTAMP}' + upload_release_archive_script: '$SCRIPT_BASE/upload_release_archive.sh |& ${TIMESTAMP}' on_failure: failed_branch_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_branch_failure.sh' @@ -618,6 +619,9 @@ verify_test_built_images_task: integration_test_script: >- [[ "$PACKER_BUILDER_NAME" == "xfedora-30" ]] || \ $SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP} + build_release_script: >- + [[ "$PACKER_BUILDER_NAME" == "xfedora-30" ]] || \ + '$SCRIPT_BASE/build_release.sh |& ${TIMESTAMP}' system_test_script: >- [[ "$PACKER_BUILDER_NAME" == "xfedora-30" ]] || \ $SCRIPT_BASE/system_test.sh |& ${TIMESTAMP} @@ -632,7 +636,7 @@ success_task: # it blocks PRs from merging if a depends_on task fails only_if: $CIRRUS_BRANCH != $DEST_BRANCH - # ignores any dependent task conditions, include everything except 'release' + # ignores any dependent task conditions depends_on: - "gating" - "vendor" @@ -663,49 +667,3 @@ success_task: memory: 1 success_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/success.sh |& ${TIMESTAMP}' - - -release_task: - - # Never do this when building images - only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\*\*\*\s*CIRRUS:\s*TEST\s*IMAGES\s*\*\*\*.*' - - # TODO: Uncomment both to not affect pass/fail status of entire job? - # allow_failures: $CI == "true" - # skip_notifications: $CI == "true" - - # Must include everything (YAML anchor/alias cannot be used here) - depends_on: - - "gating" - - "vendor" - - "varlink_api" - - "build_each_commit" - - "build_without_cgo" - - "meta" - - "image_prune" - - "testing" - - "testing_crun" - - "special_testing_rootless" - - "special_testing_in_podman" - - "special_testing_cgroupv2" - - "special_testing_cross" - - "special_testing_endpoint" - - "test_build_cache_images" - - "test_building_snap" - - "verify_test_built_images" - - "success" - - gce_instance: - image_name: "${IMAGE_BUILDER_CACHE_IMAGE_NAME}" - - timeout_in: 30m - - env: - GCPJSON: ENCRYPTED[789d8f7e9a5972ce350fd8e60f1032ccbf4a35c3938b604774b711aad280e12c21faf10e25af1e0ba33597ffb9e39e46] - GCPNAME: ENCRYPTED[417d50488a4bd197bcc925ba6574de5823b97e68db1a17e3a5fde4bcf26576987345e75f8d9ea1c15a156b4612c072a1] - GCPROJECT: ENCRYPTED[7c80e728e046b1c76147afd156a32c1c57d4a1ac1eab93b7e68e718c61ca8564fc61fef815952b8ae0a64e7034b8fe4f] - - uncache_release_archives_script: '$SCRIPT_BASE/uncache_release_archives.sh |& ${TIMESTAMP}' - - on_failure: - failed_branch_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_branch_failure.sh' diff --git a/.gitignore b/.gitignore index 2d984cb5a..d3e56ecdf 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,7 @@ __pycache__ /cmd/podman/varlink/iopodman.go .gopathok test/e2e/e2e.coverprofile -/podman*zip +release.txt +podman-remote*.zip podman*.tar.gz .idea* @@ -81,13 +81,12 @@ LIBSECCOMP_COMMIT := release-2.3 # caller may override in special circumstances if needed. GINKGOTIMEOUT ?= -timeout=90m -RELEASE_VERSION ?= $(shell git fetch --tags && git describe HEAD 2> /dev/null) -RELEASE_NUMBER ?= $(shell echo $(RELEASE_VERSION) | sed 's/-.*//') -RELEASE_DIST ?= $(shell ( source /etc/os-release; echo $$ID )) -RELEASE_DIST_VER ?= $(shell ( source /etc/os-release; echo $$VERSION_ID | cut -d '.' -f 1)) -RELEASE_ARCH ?= $(shell go env GOARCH 2> /dev/null) -RELEASE_BASENAME := $(shell basename $(PROJECT)) - +RELEASE_VERSION ?= $(shell hack/get_release_info.sh VERSION) +RELEASE_NUMBER ?= $(shell hack/get_release_info.sh NUMBER) +RELEASE_DIST ?= $(shell hack/get_release_info.sh DIST) +RELEASE_DIST_VER ?= $(shell hack/get_release_info.sh DIST_VER) +RELEASE_ARCH ?= $(shell hack/get_release_info.sh ARCH) +RELEASE_BASENAME := $(shell hack/get_release_info.sh BASENAME) # If GOPATH not specified, use one in the local directory ifeq ($(GOPATH),) @@ -164,11 +163,9 @@ podman: .gopathok $(PODMAN_VARLINK_DEPENDENCIES) ## Build with podman podman-remote: .gopathok $(PODMAN_VARLINK_DEPENDENCIES) ## Build with podman on remote environment $(GO_BUILD) $(BUILDFLAGS) -gcflags '$(GCFLAGS)' -asmflags '$(ASMFLAGS)' -ldflags '$(LDFLAGS_PODMAN)' -tags "$(BUILDTAGS) remoteclient" -o bin/$@ $(PROJECT)/cmd/podman -podman-remote-darwin: .gopathok $(PODMAN_VARLINK_DEPENDENCIES) ## Build with podman on remote OSX environment - CGO_ENABLED=0 GOOS=darwin $(GO_BUILD) -gcflags '$(GCFLAGS)' -asmflags '$(ASMFLAGS)' -ldflags '$(LDFLAGS_PODMAN)' -tags "remoteclient containers_image_openpgp exclude_graphdriver_devicemapper" -o bin/$@ $(PROJECT)/cmd/podman - -podman-remote-windows: .gopathok $(PODMAN_VARLINK_DEPENDENCIES) ## Build with podman for a remote windows environment - CGO_ENABLED=0 GOOS=windows $(GO_BUILD) -gcflags '$(GCFLAGS)' -asmflags '$(ASMFLAGS)' -ldflags '$(LDFLAGS_PODMAN)' -tags "remoteclient containers_image_openpgp exclude_graphdriver_devicemapper" -o bin/$@.exe $(PROJECT)/cmd/podman +podman-remote-%: .gopathok $(PODMAN_VARLINK_DEPENDENCIES) ## Build podman for a specific GOOS + $(eval BINSFX := $(shell test "$*" != "windows" || echo ".exe")) + CGO_ENABLED=0 GOOS=$* $(GO_BUILD) -gcflags '$(GCFLAGS)' -asmflags '$(ASMFLAGS)' -ldflags '$(LDFLAGS_PODMAN)' -tags "remoteclient containers_image_openpgp exclude_graphdriver_devicemapper" -o bin/$@$(BINSFX) $(PROJECT)/cmd/podman local-cross: $(CROSS_BUILD_TARGETS) ## Cross local compilation @@ -182,8 +179,9 @@ clean: ## Clean artifacts rm -rf \ .gopathok \ _output \ - podman*.zip \ - podman*.tar.gz \ + release.txt + $(wildcard podman-remote*.zip) \ + $(wildcard podman*.tar.gz) \ bin \ build \ docs/remote \ @@ -300,23 +298,6 @@ vagrant-check: binaries: varlink_generate podman podman-remote ## Build podman -# Zip archives are supported on all platforms + allows embedding metadata -podman.zip: binaries docs - $(eval TMPDIR := $(shell mktemp -d -p '' $@_XXXX)) - test -n "$(TMPDIR)" - $(MAKE) install "DESTDIR=$(TMPDIR)" "PREFIX=$(TMPDIR)/usr" - # Encoded RELEASE_INFO format depended upon by CI tooling - # X-RELEASE-INFO format depended upon by CI tooling - cd "$(TMPDIR)" && echo \ - "X-RELEASE-INFO: $(RELEASE_BASENAME) $(RELEASE_VERSION) $(RELEASE_DIST) $(RELEASE_DIST_VER) $(RELEASE_ARCH)" | \ - zip --recurse-paths --archive-comment "$(CURDIR)/$@" "./" - -rm -rf "$(TMPDIR)" - -podman-remote-%.zip: podman-remote-% - # Don't label darwin/windows cros-compiles with local distribution & version - echo "X-RELEASE-INFO: podman-remote $(RELEASE_VERSION) $* cc $(RELEASE_ARCH)" | \ - zip --archive-comment "$(CURDIR)/$@" ./bin/$<* - install.catatonit: ./hack/install_catatonit.sh @@ -333,19 +314,58 @@ docs: $(MANPAGES) ## Generate documentation install-podman-remote-docs: docs @(cd docs; ./podman-remote.sh ./remote) +# When publishing releases include critical build-time details +.PHONY: release.txt +release.txt: + # X-RELEASE-INFO format depended upon by automated tooling + echo -n "X-RELEASE-INFO:" > "$@" + for field in "$(RELEASE_BASENAME)" "$(RELEASE_VERSION)" \ + "$(RELEASE_DIST)" "$(RELEASE_DIST_VER)" "$(RELEASE_ARCH)"; do \ + echo -n " $$field"; done >> "$@" + echo "" >> "$@" + +podman-$(RELEASE_NUMBER).tar.gz: binaries docs release.txt + $(eval TMPDIR := $(shell mktemp -d -p '' podman_XXXX)) + $(eval SUBDIR := podman-$(RELEASE_NUMBER)) + mkdir -p "$(TMPDIR)/$(SUBDIR)" + $(MAKE) install.bin install.man install.cni install.systemd "DESTDIR=$(TMPDIR)/$(SUBDIR)" "PREFIX=/usr" + # release.txt location and content depended upon by automated tooling + cp release.txt "$(TMPDIR)/" + tar -czvf $@ --xattrs -C "$(TMPDIR)" "./release.txt" "./$(SUBDIR)" + -rm -rf "$(TMPDIR)" + +# Must call make in-line: Dependency-spec. w/ wild-card also consumes variable value. +podman-remote-$(RELEASE_NUMBER)-%.zip: + $(MAKE) podman-remote-$* install-podman-remote-docs release.txt \ + RELEASE_BASENAME=$(shell hack/get_release_info.sh REMOTENAME) \ + RELEASE_DIST=$* RELEASE_DIST_VER="-" + $(eval TMPDIR := $(shell mktemp -d -p '' $podman_remote_XXXX)) + $(eval SUBDIR := podman-$(RELEASE_VERSION)) + $(eval BINSFX := $(shell test "$*" != "windows" || echo ".exe")) + mkdir -p "$(TMPDIR)/$(SUBDIR)" + # release.txt location and content depended upon by automated tooling + cp release.txt "$(TMPDIR)/" + cp ./bin/podman-remote-$*$(BINSFX) "$(TMPDIR)/$(SUBDIR)/podman$(BINSFX)" + cp -r ./docs/remote "$(TMPDIR)/$(SUBDIR)/docs/" + $(eval DOCFILE := $(TMPDIR)/$(SUBDIR)/docs/podman.1) + cp docs/podman-remote.1 "$(DOCFILE)" + sed -i 's/podman\\*-remote/podman/g' "$(DOCFILE)" + sed -i 's/Podman\\*-remote/Podman\ for\ $*/g' "$(DOCFILE)" + sed -i 's/podman\.conf/podman\-remote\.conf/g' "$(DOCFILE)" + sed -i 's/A\ remote\ CLI\ for\ Podman\:\ //g' "$(DOCFILE)" + cd "$(TMPDIR)" && \ + zip --recurse-paths "$(CURDIR)/$@" "./release.txt" "./" + -rm -rf "$(TMPDIR)" + +.PHONY: podman-release +podman-release: + rm -f release.txt + $(MAKE) podman-$(RELEASE_NUMBER).tar.gz -brew-pkg: install-podman-remote-docs podman-remote-darwin - @mkdir -p ./brew - @cp ./bin/podman-remote-darwin ./brew/podman - @cp -r ./docs/remote ./brew/docs/ - @cp docs/podman-remote.1 ./brew/docs/podman.1 - @cp docs/podman-remote.conf.5 ./brew/docs/podman-remote.conf.5 - @sed -i 's/podman\\*-remote/podman/g' ./brew/docs/podman.1 - @sed -i 's/Podman\\*-remote/Podman\ for\ Mac/g' ./brew/docs/podman.1 - @sed -i 's/podman\.conf/podman\-remote\.conf/g' ./brew/docs/podman.1 - @sed -i 's/A\ remote\ CLI\ for\ Podman\:\ //g' ./brew/docs/podman.1 - tar -czvf podman-${RELEASE_NUMBER}.tar.gz ./brew - @rm -rf ./brew +.PHONY: podman-remote-%-release +podman-remote-%-release: + rm -f release.txt + $(MAKE) podman-remote-$(RELEASE_NUMBER)-$*.zip docker-docs: docs (cd docs; ./dckrman.sh *.1) @@ -8,6 +8,9 @@ popularized by Kubernetes. Libpod also contains the Pod Manager tool `(Podman)` * [Latest Version: 1.4.4](https://github.com/containers/libpod/releases/latest) * [Continuous Integration:](contrib/cirrus/README.md) [](https://cirrus-ci.com/github/containers/libpod/master) * [GoDoc: ](https://godoc.org/github.com/containers/libpod/libpod) +* Automated continuous release downloads (including remote-client): + * Master Branch: [https://storage.cloud.google.com/libpod-master-releases/](https://storage.cloud.google.com/libpod-master-releases/) + * Pull-requests: [https://storage.cloud.google.com/libpod-pr-releases/](https://storage.cloud.google.com/libpod-pr-releases/) ## Overview and scope diff --git a/cmd/podman/cliconfig/config.go b/cmd/podman/cliconfig/config.go index 9bc47333d..98e7aed4b 100644 --- a/cmd/podman/cliconfig/config.go +++ b/cmd/podman/cliconfig/config.go @@ -509,6 +509,7 @@ type SignValues struct { PodmanCommand Directory string SignBy string + CertDir string } type StartValues struct { diff --git a/cmd/podman/sign.go b/cmd/podman/sign.go index de289047a..63ba9b904 100644 --- a/cmd/podman/sign.go +++ b/cmd/podman/sign.go @@ -46,7 +46,7 @@ func init() { flags := signCommand.Flags() flags.StringVarP(&signCommand.Directory, "directory", "d", "", "Define an alternate directory to store signatures") flags.StringVar(&signCommand.SignBy, "sign-by", "", "Name of the signing key") - + flags.StringVar(&signCommand.CertDir, "cert-dir", "", "`Pathname` of a directory containing TLS certificates and keys") } // SignatureStoreDir defines default directory to store signatures @@ -76,6 +76,13 @@ func signCmd(c *cliconfig.SignValues) error { } } + sc := runtime.SystemContext() + sc.DockerCertPath = c.CertDir + + dockerRegistryOptions := image.DockerRegistryOptions{ + DockerCertPath: c.CertDir, + } + mech, err := signature.NewGPGSigningMechanism() if err != nil { return errors.Wrap(err, "error initializing GPG") @@ -85,7 +92,7 @@ func signCmd(c *cliconfig.SignValues) error { return errors.Wrap(err, "signing is not supported") } - systemRegistriesDirPath := trust.RegistriesDirPath(runtime.SystemContext()) + systemRegistriesDirPath := trust.RegistriesDirPath(sc) registryConfigs, err := trust.LoadAndMergeConfig(systemRegistriesDirPath) if err != nil { return errors.Wrapf(err, "error reading registry configuration") @@ -96,10 +103,14 @@ func signCmd(c *cliconfig.SignValues) error { if err != nil { return errors.Wrapf(err, "error parsing image name") } - rawSource, err := srcRef.NewImageSource(getContext(), runtime.SystemContext()) + rawSource, err := srcRef.NewImageSource(getContext(), sc) if err != nil { return errors.Wrapf(err, "error getting image source") } + err = rawSource.Close() + if err != nil { + logrus.Errorf("unable to close new image source %q", err) + } manifest, _, err := rawSource.GetManifest(getContext(), nil) if err != nil { return errors.Wrapf(err, "error getting manifest") @@ -114,7 +125,7 @@ func signCmd(c *cliconfig.SignValues) error { if err != nil { return err } - newImage, err := runtime.ImageRuntime().New(getContext(), signimage, rtc.SignaturePolicyPath, "", os.Stderr, nil, image.SigningOptions{SignBy: signby}, nil, util.PullImageMissing) + newImage, err := runtime.ImageRuntime().New(getContext(), signimage, rtc.SignaturePolicyPath, "", os.Stderr, &dockerRegistryOptions, image.SigningOptions{SignBy: signby}, nil, util.PullImageMissing) if err != nil { return errors.Wrapf(err, "error pulling image %s", signimage) } diff --git a/completions/bash/podman b/completions/bash/podman index 7280f4040..e6ffb135f 100644 --- a/completions/bash/podman +++ b/completions/bash/podman @@ -2669,6 +2669,7 @@ _podman_container_runlabel() { _podman_image_sign() { local options_with_args=" + --cert-dir -d --directory --sign-by diff --git a/contrib/cirrus/99-do-not-use-google-subnets.conflist b/contrib/cirrus/99-do-not-use-google-subnets.conflist new file mode 100644 index 000000000..e9ab638ed --- /dev/null +++ b/contrib/cirrus/99-do-not-use-google-subnets.conflist @@ -0,0 +1,21 @@ +{ + "cniVersion": "0.4.0", + "name": "do-not-use-google-subnets", + "plugins": [ + { + "type": "bridge", + "name": "do-not-use-google-subnets", + "bridge": "do-not-use-google-subnets", + "ipam": { + "type": "host-local", + "ranges": [ + [ + { + "subnet": "10.128.0.0/9" + } + ] + ] + } + } + ] +} diff --git a/contrib/cirrus/build_release.sh b/contrib/cirrus/build_release.sh new file mode 100755 index 000000000..287643f47 --- /dev/null +++ b/contrib/cirrus/build_release.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +source $(dirname $0)/lib.sh + +req_env_var TEST_REMOTE_CLIENT OS_RELEASE_ID GOSRC + +cd $GOSRC + +if [[ "$TEST_REMOTE_CLIENT" == "true" ]] && [[ -z "$CROSS_PLATFORM" ]] +then + CROSS_PLATFORM=linux +fi + +if [[ -n "$CROSS_PLATFORM" ]] +then + echo "Compiling podman-remote release archive for ${CROSS_PLATFORM}" + case "$CROSS_PLATFORM" in + linux) ;& + windows) ;& + darwin) + make podman-remote-${CROSS_PLATFORM}-release + ;; + *) + die 1 "Unknown/unsupported cross-compile platform '$CROSS_PLATFORM'" + ;; + esac +else + echo "Compiling release archive for $OS_RELEASE_ID" + make podman-release +fi diff --git a/contrib/cirrus/cache_release_archive.sh b/contrib/cirrus/cache_release_archive.sh deleted file mode 100755 index 2365f7593..000000000 --- a/contrib/cirrus/cache_release_archive.sh +++ /dev/null @@ -1,140 +0,0 @@ -#!/bin/bash - -set -eo pipefail - -source $(dirname $0)/lib.sh - -req_env_var GOSRC - -RELEASE_ARCHIVE_NAMES="" - -handle_archive() { # Assumed to be called with set +e - TASK_NUMBER=$1 - PR_OR_BRANCH=$2 - CACHE_URL=$3 - ARCHIVE_NAME="$(basename $CACHE_URL)" - req_env_var TASK_NUMBER PR_OR_BRANCH CACHE_URL ARCHIVE_NAME - - cd /tmp - curl -sO "$CACHE_URL" || return $(warn 0 "Couldn't download file, skipping.") - [[ -r "/tmp/$ARCHIVE_NAME" ]] || return $(warn 0 "Unreadable archive '/tmp/$ARCHIVE_NAME', skipping.") - - ZIPCOMMENT=$(unzip -qqz "$ARCHIVE_NAME" 2>/dev/null) # noisy bugger - if [[ "$?" -ne "0" ]] || [[ -z "$ZIPCOMMENT" ]] - then - return $(warn 0 "Could not unzip metadata from downloaded '/tmp/$ARCHIVE_NAME', skipping.") - fi - - RELEASE_INFO=$(echo "$ZIPCOMMENT" | grep -m 1 'X-RELEASE-INFO:' | sed -r -e 's/X-RELEASE-INFO:\s*(.+)/\1/') - if [[ "$?" -ne "0" ]] || [[ -z "$RELEASE_INFO" ]] - then - return $(warn 0 "Metadata empty or invalid: '$ZIPCOMMENT', skipping.") - fi - - # e.g. libpod v1.3.1-166-g60df124e fedora 29 amd64 - # or libpod v1.3.1-166-g60df124e amd64 - FIELDS="RELEASE_BASENAME RELEASE_VERSION RELEASE_DIST RELEASE_DIST_VER RELEASE_ARCH" - read $FIELDS <<< $RELEASE_INFO - for f in $FIELDS - do - [[ -n "${!f}" ]] || return $(warn 0 "Expecting $f to be non-empty in metadata: '$RELEASE_INFO', skipping.") - done - - echo -n "Preparing $RELEASE_BASENAME archive: " - # Drop version number to enable "latest" representation - # (version available w/in zip-file comment) - RELEASE_ARCHIVE_NAME="${RELEASE_BASENAME}-${PR_OR_BRANCH}-${RELEASE_DIST}-${RELEASE_DIST_VER}-${RELEASE_ARCH}.zip" - # Allow uploading all gathered files in parallel, later with gsutil. - mv -v "$ARCHIVE_NAME" "/$RELEASE_ARCHIVE_NAME" - RELEASE_ARCHIVE_NAMES="$RELEASE_ARCHIVE_NAMES $RELEASE_ARCHIVE_NAME" -} - -make_release() { - ARCHIVE_NAME="$1" - req_env_var ARCHIVE_NAME - - # There's no actual testing of windows/darwin targets yet - # but we still want to cross-compile and publish binaries - if [[ "$SPECIALMODE" == "windows" ]] || [[ "$SPECIALMODE" == "darwin" ]] - then - RELFILE="podman-remote-${SPECIALMODE}.zip" - elif [[ "$SPECIALMODE" == "none" ]] - then - RELFILE="podman.zip" - else - die 55 "$(basename $0) unable to handle \$SPECIALMODE=$SPECIALMODE for $ARCHIVE_NAME" - fi - echo "Calling make $RELFILE" - cd $GOSRC - make "$RELFILE" - echo "Renaming archive so it can be identified/downloaded for publishing" - mv -v "$RELFILE" "$ARCHIVE_NAME" - echo "Success!" -} - -[[ "$CI" == "true" ]] || \ - die 56 "$0 requires a Cirrus-CI cross-task cache to function" - -cd $GOSRC -# Same script re-used for both uploading and downloading to avoid duplication -if [[ "$(basename $0)" == "cache_release_archive.sh" ]] -then - # ref: https://cirrus-ci.org/guide/writing-tasks/#environment-variables - req_env_var CI_NODE_INDEX CIRRUS_BUILD_ID - # Use unique names for uncache_release_archives.sh to find/download them all - ARCHIVE_NAME="build-${CIRRUS_BUILD_ID}-task-${CI_NODE_INDEX}.zip" - make_release "$ARCHIVE_NAME" - - # ref: https://cirrus-ci.org/guide/writing-tasks/#http-cache - URL="http://$CIRRUS_HTTP_CACHE_HOST/${ARCHIVE_NAME}" - echo "Uploading $ARCHIVE_NAME to Cirrus-CI cache at $URL" - curl -s -X POST --data-binary "@$ARCHIVE_NAME" "$URL" -elif [[ "$(basename $0)" == "uncache_release_archives.sh" ]] -then - req_env_var CIRRUS_BUILD_ID CI_NODE_TOTAL GCPJSON GCPNAME GCPROJECT - [[ "${CI_NODE_INDEX}" -eq "$[CI_NODE_TOTAL-1]" ]] || \ - die 0 "WARNING: This task depends on cache data from other tasks, otherwise it is a no-op." - - if [[ -n "$CIRRUS_PR" ]] - then - PR_OR_BRANCH="pr$CIRRUS_PR" - BUCKET="libpod-pr-releases" - elif [[ -n "$CIRRUS_BRANCH" ]] - then - PR_OR_BRANCH="$CIRRUS_BRANCH" - BUCKET="libpod-$CIRRUS_BRANCH-releases" - else - die 10 "Expecting either \$CIRRUS_PR or \$CIRRUS_BRANCH to be non-empty." - fi - - echo "Blindly downloading Cirrus-CI cache files for task (some will fail)." - set +e # Don't stop looping until all task's cache is attempted - for (( task_number = 0 ; task_number < $CI_NODE_TOTAL ; task_number++ )) - do - ARCHIVE_NAME="build-${CIRRUS_BUILD_ID}-task-${task_number}.zip" - URL="http://$CIRRUS_HTTP_CACHE_HOST/${ARCHIVE_NAME}" - echo "Attempting to download cached archive from $URL" - handle_archive "$task_number" "$PR_OR_BRANCH" "$URL" - echo "----------------------------------------" - done - set -e - - [[ -n "$RELEASE_ARCHIVE_NAMES" ]] || \ - die 67 "Error: No release archives found in CI cache, expecting at least one." - - echo "Preparing to upload release archives." - gcloud config set project "$GCPROJECT" - echo "$GCPJSON" > /tmp/gcp.json - gcloud auth activate-service-account --key-file=/tmp/gcp.json - rm /tmp/gcp.json - # handle_archive() placed all uploadable files under / - gsutil -m cp /*.zip "gs://$BUCKET" # Upload in parallel - echo "Successfully uploaded archives:" - for ARCHIVE_NAME in $RELEASE_ARCHIVE_NAMES - do - echo " https://storage.cloud.google.com/$BUCKET/$ARCHIVE_NAME" - done - echo "These will remain available until automatic pruning by bucket policy." -else - die 9 "I don't know what to do when called $0" -fi diff --git a/contrib/cirrus/cirrus_yaml_test.py b/contrib/cirrus/cirrus_yaml_test.py index c8faee65f..c2ff8e69e 100755 --- a/contrib/cirrus/cirrus_yaml_test.py +++ b/contrib/cirrus/cirrus_yaml_test.py @@ -26,7 +26,6 @@ class TestCaseBase(unittest.TestCase): class TestDependsOn(TestCaseBase): ALL_TASK_NAMES = None - SUCCESS_RELEASE = set(['success', 'release']) def setUp(self): super().setUp() @@ -34,34 +33,22 @@ class TestDependsOn(TestCaseBase): for key, _ in self.CIRRUS_YAML.items() if key.endswith('_task')]) - def test_dicts(self): + def test_00_dicts(self): """Expected dictionaries are present and non-empty""" - for name in ('success_task', 'release_task'): - # tests all names then show specific failures - with self.subTest(name=name): - self.assertIn(name, self.CIRRUS_YAML) - self.assertIn(name.replace('_task', ''), self.ALL_TASK_NAMES) - self.assertIn('depends_on', self.CIRRUS_YAML[name]) - self.assertGreater(len(self.CIRRUS_YAML[name]['depends_on']), 0) - - def _check_dep(self, name, task_name, deps): - # name includes '_task' suffix, task_name does not - msg=('Please add "{0}" to the "depends_on" list in "{1}"' - "".format(task_name, name)) - self.assertIn(task_name, deps, msg=msg) - - def test_depends(self): - """Success and Release tasks depend on all other tasks""" - for name in ('success_task', 'release_task'): - deps = set(self.CIRRUS_YAML[name]['depends_on']) - for task_name in self.ALL_TASK_NAMES - self.SUCCESS_RELEASE: - with self.subTest(name=name, task_name=task_name): - self._check_dep(name, task_name, deps) - - def test_release(self): - """Release task must always execute last""" - deps = set(self.CIRRUS_YAML['release_task']['depends_on']) - self._check_dep('release_task', 'success', deps) + self.assertIn('success_task', self.CIRRUS_YAML) + self.assertIn('success_task'.replace('_task', ''), self.ALL_TASK_NAMES) + self.assertIn('depends_on', self.CIRRUS_YAML['success_task']) + self.assertGreater(len(self.CIRRUS_YAML['success_task']['depends_on']), 0) + + def test_01_depends(self): + """Success task depends on all other tasks""" + success_deps = set(self.CIRRUS_YAML['success_task']['depends_on']) + for task_name in self.ALL_TASK_NAMES - set(['success']): + with self.subTest(task_name=task_name): + msg=('Please add "{0}" to the "depends_on" list in "success_task"' + "".format(task_name)) + self.assertIn(task_name, success_deps, msg=msg) + if __name__ == "__main__": diff --git a/contrib/cirrus/integration_test.sh b/contrib/cirrus/integration_test.sh index a3d18d440..552f2ba73 100755 --- a/contrib/cirrus/integration_test.sh +++ b/contrib/cirrus/integration_test.sh @@ -65,10 +65,6 @@ case "$SPECIALMODE" in make local${TESTSUITE} fi ;; - windows) ;& # for podman-remote building only - darwin) - warn '' "No $SPECIALMODE remote client integration tests configured" - ;; *) die 110 "Unsupported \$SPECIALMODE: $SPECIALMODE" esac diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh index a20ee5a62..f66e63140 100644 --- a/contrib/cirrus/lib.sh +++ b/contrib/cirrus/lib.sh @@ -64,6 +64,8 @@ export PRIOR_FEDORA_BASE_IMAGE="fedora-cloud-base-29-1-2-1559164849" export BUILT_IMAGE_SUFFIX="${BUILT_IMAGE_SUFFIX:--$CIRRUS_REPO_NAME-${CIRRUS_BUILD_ID}}" # IN_PODMAN container image IN_PODMAN_IMAGE="quay.io/libpod/in_podman:latest" +# Image for uploading releases +UPLDREL_IMAGE="quay.io/libpod/upldrel:latest" # Avoid getting stuck waiting for user input export DEBIAN_FRONTEND="noninteractive" @@ -76,7 +78,7 @@ BIGTO="timeout_attempt_delay_command 300s 5 30s" # Safe env. vars. to transfer from root -> $ROOTLESS_USER (go env handled separetly) ROOTLESS_ENV_RE='(CIRRUS_.+)|(ROOTLESS_.+)|(.+_IMAGE.*)|(.+_BASE)|(.*DIRPATH)|(.*FILEPATH)|(SOURCE.*)|(DEPEND.*)|(.+_DEPS_.+)|(OS_REL.*)|(.+_ENV_RE)|(TRAVIS)|(CI.+)|(TEST_REMOTE.*)' # Unsafe env. vars for display -SECRET_ENV_RE='(IRCID)|(ACCOUNT)|(^GC[EP]..+)|(SSH)' +SECRET_ENV_RE='(IRCID)|(ACCOUNT)|(GC[EP]..+)|(SSH)' # Names of systemd units which should never be running EVIL_UNITS="cron crond atd apt-daily-upgrade apt-daily fstrim motd-news systemd-tmpfiles-clean" @@ -321,13 +323,15 @@ EOF install_test_configs(){ echo "Installing cni config, policy and registry config" - req_env_var GOSRC - sudo install -D -m 755 $GOSRC/cni/87-podman-bridge.conflist \ - /etc/cni/net.d/87-podman-bridge.conflist - sudo install -D -m 755 $GOSRC/test/policy.json \ - /etc/containers/policy.json - sudo install -D -m 755 $GOSRC/test/registries.conf \ - /etc/containers/registries.conf + req_env_var GOSRC SCRIPT_BASE + cd $GOSRC + install -v -D -m 644 ./cni/87-podman-bridge.conflist /etc/cni/net.d/ + # This config must always sort last in the list of networks (podman picks first one + # as the default). This config prevents allocation of network address space used + # by default in google cloud. https://cloud.google.com/vpc/docs/vpc#ip-ranges + install -v -D -m 644 $SCRIPT_BASE/99-do-not-use-google-subnets.conflist /etc/cni/net.d/ + install -v -D -m 644 ./test/policy.json /etc/containers/ + install -v -D -m 644 ./test/registries.conf /etc/containers/ } # Remove all files (except conmon, for now) provided by the distro version of podman. diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh index 2579229a5..7c7659169 100755 --- a/contrib/cirrus/setup_environment.sh +++ b/contrib/cirrus/setup_environment.sh @@ -44,11 +44,15 @@ case "${OS_REL_VER}" in ;; fedora-30) ;& # continue to next item fedora-29) + # All SELinux distros need this for systemd-in-a-container + setsebool container_manage_cgroup true if [[ "$ADD_SECOND_PARTITION" == "true" ]]; then bash "$SCRIPT_BASE/add_second_partition.sh"; fi ;; centos-7) # Current VM is an image-builder-image no local podman/testing - echo "No further setup required for VM image building" + echo "No further setup required for VM image building" + # All SELinux distros need this for systemd-in-a-container + setsebool container_manage_cgroup true exit 0 ;; *) bad_os_id_ver ;; @@ -57,8 +61,7 @@ esac # Reload to incorporate any changes from above source "$SCRIPT_BASE/lib.sh" -install_test_configs - +# Must execute before possible setup_rootless() make install.tools case "$SPECIALMODE" in @@ -66,7 +69,8 @@ case "$SPECIALMODE" in remove_packaged_podman_files # we're building from source ;; none) - remove_packaged_podman_files + [[ -n "$CROSS_PLATFORM" ]] || \ + remove_packaged_podman_files ;; endpoint) remove_packaged_podman_files @@ -88,8 +92,8 @@ case "$SPECIALMODE" in in_podman) # Assumed to be Fedora $SCRIPT_BASE/setup_container_environment.sh ;; - windows) ;& # for podman-remote building only - darwin) ;; *) die 111 "Unsupported \$SPECIALMODE: $SPECIALMODE" esac + +install_test_configs diff --git a/contrib/cirrus/uncache_release_archives.sh b/contrib/cirrus/uncache_release_archives.sh deleted file mode 120000 index e9fc6edff..000000000 --- a/contrib/cirrus/uncache_release_archives.sh +++ /dev/null @@ -1 +0,0 @@ -cache_release_archive.sh
\ No newline at end of file diff --git a/contrib/cirrus/unit_test.sh b/contrib/cirrus/unit_test.sh index 004839f17..c6c77d17e 100755 --- a/contrib/cirrus/unit_test.sh +++ b/contrib/cirrus/unit_test.sh @@ -16,10 +16,6 @@ case "$SPECIALMODE" in none) make ;; - windows) ;& - darwin) - make podman-remote-$SPECIALMODE - ;; *) die 109 "Unsupported \$SPECIAL_MODE: $SPECIALMODE" esac diff --git a/contrib/cirrus/upload_release_archive.sh b/contrib/cirrus/upload_release_archive.sh new file mode 100755 index 000000000..942255821 --- /dev/null +++ b/contrib/cirrus/upload_release_archive.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +set -eo pipefail + +source $(dirname $0)/lib.sh + +req_env_var CI UPLDREL_IMAGE CIRRUS_BUILD_ID GOSRC RELEASE_GCPJSON RELEASE_GCPNAME RELEASE_GCPROJECT + +[[ "$CI" == "true" ]] || \ + die 56 "$0 must be run under Cirrus-CI to function" + +unset PR_OR_BRANCH BUCKET +if [[ -n "$CIRRUS_PR" ]] +then + PR_OR_BRANCH="pr$CIRRUS_PR" + BUCKET="libpod-pr-releases" +elif [[ -n "$CIRRUS_BRANCH" ]] +then + PR_OR_BRANCH="$CIRRUS_BRANCH" + BUCKET="libpod-$CIRRUS_BRANCH-releases" +else + die 1 "Expecting either \$CIRRUS_PR or \$CIRRUS_BRANCH to be non-empty." +fi + +# Functional local podman required for uploading a release +cd $GOSRC +[[ -n "$(type -P podman)" ]] || \ + make install || \ + die 57 "$0 requires working podman binary on path to function" + +TMPF=$(mktemp -p '' $(basename $0)_XXXX.json) +trap "rm -f $TMPF" EXIT +set +x +echo "$RELEASE_GCPJSON" > "$TMPF" +unset RELEASE_GCPJSON + +cd $GOSRC +for filename in $(ls -1 *.tar.gz *.zip) +do + echo "Running podman ... $UPLDREL_IMAGE $filename" + podman run -i --rm \ + -e "GCPNAME=$RELEASE_GCPNAME" \ + -e "GCPPROJECT=$RELEASE_GCPROJECT" \ + -e "GCPJSON_FILEPATH=$TMPF" \ + -e "REL_ARC_FILEPATH=/tmp/$filename" \ + -e "PR_OR_BRANCH=$PR_OR_BRANCH" \ + -e "BUCKET=$BUCKET" \ + --security-opt label=disable \ + -v "$TMPF:$TMPF:ro" \ + -v "$GOSRC/$filename:/tmp/$filename:ro" \ + $UPLDREL_IMAGE +done diff --git a/contrib/imgts/lib_entrypoint.sh b/contrib/imgts/lib_entrypoint.sh index 7b76c823f..3f6b11128 100644 --- a/contrib/imgts/lib_entrypoint.sh +++ b/contrib/imgts/lib_entrypoint.sh @@ -35,10 +35,15 @@ req_env_var() { gcloud_init() { set +xe - TMPF=$(mktemp -p '' .$(uuidgen)XXXX) - trap "rm -f $TMPF" EXIT - echo "$GCPJSON" > $TMPF && \ - $GCLOUD auth activate-service-account --project "$GCPPROJECT" --key-file=$TMPF || \ + if [[ -n "$1" ]] && [[ -r "$1" ]] + then + TMPF="$1" + else + TMPF=$(mktemp -p '' .$(uuidgen)_XXXX.json) + trap "rm -f $TMPF &> /dev/null" EXIT + echo "$GCPJSON" > $TMPF + fi + $GCLOUD auth activate-service-account --project="$GCPPROJECT" --key-file="$TMPF" || \ die 5 FATAL auth - rm -f $TMPF + rm -f $TMPF &> /dev/null || true # ignore any read-only error } diff --git a/contrib/upldrel/Dockerfile b/contrib/upldrel/Dockerfile new file mode 100644 index 000000000..54a58c521 --- /dev/null +++ b/contrib/upldrel/Dockerfile @@ -0,0 +1,9 @@ +FROM quay.io/libpod/imgts:latest + +RUN yum -y update && \ + yum -y install unzip && \ + rpm -V unzip && \ + yum clean all + +COPY /contrib/upldrel/entrypoint.sh /usr/local/bin/entrypoint.sh +RUN chmod 755 /usr/local/bin/entrypoint.sh diff --git a/contrib/upldrel/README.md b/contrib/upldrel/README.md new file mode 100644 index 000000000..41f5ffef0 --- /dev/null +++ b/contrib/upldrel/README.md @@ -0,0 +1,9 @@ + + +A container image for canonical-naming and uploading of +libpod and remote-client archives. Only intended to ever +be used by CI/CD, and depends heavily on an embedded +`release.txt` file produced by `make`. + +Build script: [../cirrus/build_release.sh](../cirrus/build_release.sh) +Upload script: [../cirrus/upload_release_archive.sh](../cirrus/upload_release_archive.sh) diff --git a/contrib/upldrel/entrypoint.sh b/contrib/upldrel/entrypoint.sh new file mode 100755 index 000000000..985b828a0 --- /dev/null +++ b/contrib/upldrel/entrypoint.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +set -e + +source /usr/local/bin/lib_entrypoint.sh + +req_env_var GCPJSON_FILEPATH GCPNAME GCPPROJECT REL_ARC_FILEPATH PR_OR_BRANCH BUCKET + +[[ -r "$REL_ARC_FILEPATH" ]] || \ + die 2 ERROR Cannot read release archive file: "$REL_ARC_FILEPATH" + +[[ -r "$GCPJSON_FILEPATH" ]] || \ + die 3 ERROR Cannot read GCP credentials file: "$GCPJSON_FILEPATH" + +cd $TMPDIR +echo "Attempting to extract release.txt from tar or zip $REL_ARC_FILEPATH" +unset SFX +if tar xzf "$REL_ARC_FILEPATH" "./release.txt" +then + echo "It's a tarball" + SFX="tar.gz" +elif unzip "$REL_ARC_FILEPATH" release.txt +then + echo "It's a zip" + SFX="zip" +else + die 5 ERROR Could not extract release.txt from $REL_ARC_FILEPATH +fi + +echo "Parsing release.txt contents" +RELEASETXT=$(<release.txt) +cd - +[[ -n "$RELEASETXT" ]] || \ + die 3 ERROR Could not obtain metadata from release.txt in $REL_ARC_FILEPATH + +RELEASE_INFO=$(echo "$RELEASETXT" | grep -m 1 'X-RELEASE-INFO:' | sed -r -e 's/X-RELEASE-INFO:\s*(.+)/\1/') +if [[ "$?" -ne "0" ]] || [[ -z "$RELEASE_INFO" ]] +then + die 4 ERROR Metadata is empty or invalid: '$RELEASETXT' +fi + +# e.g. libpod v1.3.1-166-g60df124e fedora 29 amd64 +# or libpod v1.3.1-166-g60df124e amd64 +FIELDS="RELEASE_BASENAME RELEASE_VERSION RELEASE_DIST RELEASE_DIST_VER RELEASE_ARCH" +read $FIELDS <<< $RELEASE_INFO +for f in $FIELDS +do + [[ -n "${!f}" ]] || \ + die 5 ERROR Expecting $f to be non-empty in metadata: '$RELEASE_INFO' +done + +gcloud_init "$GCPJSON_FILEPATH" + +# Drop version number to enable "latest" representation +# (version available w/in zip-file comment) +RELEASE_ARCHIVE_NAME="${RELEASE_BASENAME}-${PR_OR_BRANCH}-${RELEASE_DIST}-${RELEASE_DIST_VER}-${RELEASE_ARCH}.${SFX}" + +echo "Uploading archive as $RELEASE_ARCHIVE_NAME" +gsutil cp "$REL_ARC_FILEPATH" "gs://$BUCKET/$RELEASE_ARCHIVE_NAME" + +echo "Release now available at:" +echo " https://storage.cloud.google.com/$BUCKET/$RELEASE_ARCHIVE_NAME" diff --git a/docs/podman-image-sign.1.md b/docs/podman-image-sign.1.md index 61df3b3bd..ca438b438 100644 --- a/docs/podman-image-sign.1.md +++ b/docs/podman-image-sign.1.md @@ -12,14 +12,23 @@ been pulled from a registry. The signature will be written to a directory derived from the registry configuration files in /etc/containers/registries.d. By default, the signature will be written into /var/lib/containers/sigstore directory. ## OPTIONS + **--help**, **-h** - Print usage statement. + +Print usage statement. + +**--cert-dir**=*path* + +Use certificates at *path* (\*.crt, \*.cert, \*.key) to connect to the registry. +Default certificates directory is _/etc/containers/certs.d_. (Not available for remote commands) **--directory**, **-d**=*dir* - Store the signatures in the specified directory. Default: /var/lib/containers/sigstore + +Store the signatures in the specified directory. Default: /var/lib/containers/sigstore **--sign-by**=*identity* - Override the default identity of the signature. + +Override the default identity of the signature. ## EXAMPLES Sign the busybox image with the identify of foo@bar.com with a user's keyring and save the signature in /tmp/signatures/. diff --git a/hack/get_release_info.sh b/hack/get_release_info.sh new file mode 100755 index 000000000..29b4237b4 --- /dev/null +++ b/hack/get_release_info.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +# This script produces various bits of metadata needed by Makefile. Using +# a script allows uniform behavior across multiple environments and +# distributions. The script expects a single argument, as reflected below. + +set -e + +cd "${GOSRC:-$(dirname $0)/../}" + +valid_args() { + REGEX='^\s+[[:upper:]]+\*[)]' + egrep --text --no-filename --group-separator=' ' --only-matching "$REGEX" "$0" | \ + cut -d '*' -f 1 +} + +unset OUTPUT +case "$1" in + # Wild-card suffix needed by valid_args() e.g. possible bad grep of "$(echo $FOO)" + VERSION*) + OUTPUT="${CIRRUS_TAG:-$(git fetch --tags && git describe HEAD 2> /dev/null)}" + ;; + NUMBER*) + OUTPUT="$($0 VERSION | sed 's/-.*//')" + ;; + DIST_VER*) + OUTPUT="$(source /etc/os-release; echo $VERSION_ID | cut -d '.' -f 1)" + ;; + DIST*) + OUTPUT="$(source /etc/os-release; echo $ID)" + ;; + ARCH*) + OUTPUT="${GOARCH:-$(go env GOARCH 2> /dev/null)}" + ;; + BASENAME*) + OUTPUT="${CIRRUS_REPO_NAME:-$(basename $(git rev-parse --show-toplevel))}" + ;; + REMOTENAME*) + OUTPUT="$($0 BASENAME)-remote" + ;; + *) + echo "Error, unknown/unsupported argument '$1', valid arguments:" + valid_args + exit 1 + ;; +esac + +if [[ -n "$OUTPUT" ]] +then + echo -n "$OUTPUT" +else + echo "Error, empty output for info: '$1'" > /dev/stderr + exit 2 +fi diff --git a/libpod/boltdb_state.go b/libpod/boltdb_state.go index 176781f07..1de8d80c9 100644 --- a/libpod/boltdb_state.go +++ b/libpod/boltdb_state.go @@ -870,7 +870,7 @@ func (s *BoltState) RewritePodConfig(pod *Pod, newCfg *PodConfig) error { newCfgJSON, err := json.Marshal(newCfg) if err != nil { - return errors.Wrapf(err, "error marshalling new configuration JSON for container %s", pod.ID()) + return errors.Wrapf(err, "error marshalling new configuration JSON for pod %s", pod.ID()) } db, err := s.getDBCon() @@ -900,6 +900,50 @@ func (s *BoltState) RewritePodConfig(pod *Pod, newCfg *PodConfig) error { return err } +// RewriteVolumeConfig rewrites a volume's configuration. +// WARNING: This function is DANGEROUS. Do not use without reading the full +// comment on this function in state.go. +func (s *BoltState) RewriteVolumeConfig(volume *Volume, newCfg *VolumeConfig) error { + if !s.valid { + return define.ErrDBClosed + } + + if !volume.valid { + return define.ErrVolumeRemoved + } + + newCfgJSON, err := json.Marshal(newCfg) + if err != nil { + return errors.Wrapf(err, "error marshalling new configuration JSON for volume %q", volume.Name()) + } + + db, err := s.getDBCon() + if err != nil { + return err + } + defer s.deferredCloseDBCon(db) + + err = db.Update(func(tx *bolt.Tx) error { + volBkt, err := getVolBucket(tx) + if err != nil { + return err + } + + volDB := volBkt.Bucket([]byte(volume.Name())) + if volDB == nil { + volume.valid = false + return errors.Wrapf(define.ErrNoSuchVolume, "no volume with name %q found in DB", volume.Name()) + } + + if err := volDB.Put(configKey, newCfgJSON); err != nil { + return errors.Wrapf(err, "error updating volume %q config JSON", volume.Name()) + } + + return nil + }) + return err +} + // Pod retrieves a pod given its full ID func (s *BoltState) Pod(id string) (*Pod, error) { if id == "" { diff --git a/libpod/boltdb_state_internal.go b/libpod/boltdb_state_internal.go index 408ef7224..6e4179835 100644 --- a/libpod/boltdb_state_internal.go +++ b/libpod/boltdb_state_internal.go @@ -449,6 +449,13 @@ func (s *BoltState) getVolumeFromDB(name []byte, volume *Volume, volBkt *bolt.Bu return errors.Wrapf(err, "error unmarshalling volume %s config from DB", string(name)) } + // Get the lock + lock, err := s.runtime.lockManager.RetrieveLock(volume.config.LockID) + if err != nil { + return errors.Wrapf(err, "error retrieving lock for volume %q", string(name)) + } + volume.lock = lock + volume.runtime = s.runtime volume.valid = true diff --git a/libpod/in_memory_state.go b/libpod/in_memory_state.go index 7c4abd25d..a9b735327 100644 --- a/libpod/in_memory_state.go +++ b/libpod/in_memory_state.go @@ -425,6 +425,26 @@ func (s *InMemoryState) RewritePodConfig(pod *Pod, newCfg *PodConfig) error { return nil } +// RewriteVolumeConfig rewrites a volume's configuration. +// This function is DANGEROUS, even with in-memory state. +// Please read the full comment in state.go before using it. +func (s *InMemoryState) RewriteVolumeConfig(volume *Volume, newCfg *VolumeConfig) error { + if !volume.valid { + return define.ErrVolumeRemoved + } + + // If the volume does not exist, return error + stateVol, ok := s.volumes[volume.Name()] + if !ok { + volume.valid = false + return errors.Wrapf(define.ErrNoSuchVolume, "volume with name %q not found in state", volume.Name()) + } + + stateVol.config = newCfg + + return nil +} + // Volume retrieves a volume from its full name func (s *InMemoryState) Volume(name string) (*Volume, error) { if name == "" { diff --git a/libpod/runtime.go b/libpod/runtime.go index 4d6a80d0b..28774773e 100644 --- a/libpod/runtime.go +++ b/libpod/runtime.go @@ -76,10 +76,6 @@ var ( // place of the configuration file pointed to by ConfigPath. OverrideConfigPath = etcDir + "/containers/libpod.conf" - // DefaultInfraImage to use for infra container - - // DefaultInfraCommand to be run in an infra container - // DefaultSHMLockPath is the default path for SHM locks DefaultSHMLockPath = "/libpod_lock" // DefaultRootlessSHMLockPath is the default path for rootless SHM locks diff --git a/libpod/runtime_ctr.go b/libpod/runtime_ctr.go index 92b2faefb..acd317d20 100644 --- a/libpod/runtime_ctr.go +++ b/libpod/runtime_ctr.go @@ -253,10 +253,13 @@ func (r *Runtime) setupContainer(ctx context.Context, ctr *Container) (c *Contai // Go through named volumes and add them. // If they don't exist they will be created using basic options. + // Maintain an array of them - we need to lock them later. + ctrNamedVolumes := make([]*Volume, 0, len(ctr.config.NamedVolumes)) for _, vol := range ctr.config.NamedVolumes { // Check if it exists already - _, err := r.state.Volume(vol.Name) + dbVol, err := r.state.Volume(vol.Name) if err == nil { + ctrNamedVolumes = append(ctrNamedVolumes, dbVol) // The volume exists, we're good continue } else if errors.Cause(err) != config2.ErrNoSuchVolume { @@ -275,6 +278,8 @@ func (r *Runtime) setupContainer(ctx context.Context, ctr *Container) (c *Contai if err := ctr.copyWithTarFromImage(vol.Dest, newVol.MountPoint()); err != nil && !os.IsNotExist(err) { return nil, errors.Wrapf(err, "Failed to copy content into new volume mount %q", vol.Name) } + + ctrNamedVolumes = append(ctrNamedVolumes, newVol) } if ctr.config.LogPath == "" && ctr.config.LogDriver != JournaldLogging { @@ -291,6 +296,14 @@ func (r *Runtime) setupContainer(ctx context.Context, ctr *Container) (c *Contai ctr.config.Mounts = append(ctr.config.Mounts, ctr.config.ShmDir) } + // Lock all named volumes we are adding ourself to, to ensure we can't + // use a volume being removed. + for _, namedVol := range ctrNamedVolumes { + toLock := namedVol + toLock.lock.Lock() + defer toLock.lock.Unlock() + } + // Add the container to the state // TODO: May be worth looking into recovering from name/ID collisions here if ctr.config.Pod != "" { diff --git a/libpod/runtime_renumber.go b/libpod/runtime_renumber.go index 735ffba34..9de2556b2 100644 --- a/libpod/runtime_renumber.go +++ b/libpod/runtime_renumber.go @@ -53,6 +53,23 @@ func (r *Runtime) renumberLocks() error { return err } } + allVols, err := r.state.AllVolumes() + if err != nil { + return err + } + for _, vol := range allVols { + lock, err := r.lockManager.AllocateLock() + if err != nil { + return errors.Wrapf(err, "error allocating lock for volume %s", vol.Name()) + } + + vol.config.LockID = lock.ID() + + // Write the new lock ID + if err := r.state.RewriteVolumeConfig(vol, vol.config); err != nil { + return err + } + } r.newSystemEvent(events.Renumber) diff --git a/libpod/runtime_volume.go b/libpod/runtime_volume.go index d05db936b..512e778a1 100644 --- a/libpod/runtime_volume.go +++ b/libpod/runtime_volume.go @@ -36,6 +36,10 @@ func (r *Runtime) RemoveVolume(ctx context.Context, v *Volume, force bool) error return nil } } + + v.lock.Lock() + defer v.lock.Unlock() + return r.removeVolume(ctx, v, force) } diff --git a/libpod/runtime_volume_linux.go b/libpod/runtime_volume_linux.go index 84703787d..70296248c 100644 --- a/libpod/runtime_volume_linux.go +++ b/libpod/runtime_volume_linux.go @@ -28,7 +28,7 @@ func (r *Runtime) NewVolume(ctx context.Context, options ...VolumeCreateOption) } // newVolume creates a new empty volume -func (r *Runtime) newVolume(ctx context.Context, options ...VolumeCreateOption) (*Volume, error) { +func (r *Runtime) newVolume(ctx context.Context, options ...VolumeCreateOption) (_ *Volume, Err error) { volume, err := newVolume(r) if err != nil { return nil, errors.Wrapf(err, "error creating volume") @@ -68,6 +68,21 @@ func (r *Runtime) newVolume(ctx context.Context, options ...VolumeCreateOption) } volume.config.MountPoint = fullVolPath + lock, err := r.lockManager.AllocateLock() + if err != nil { + return nil, errors.Wrapf(err, "error allocating lock for new volume") + } + volume.lock = lock + volume.config.LockID = volume.lock.ID() + + defer func() { + if Err != nil { + if err := volume.lock.Free(); err != nil { + logrus.Errorf("Error freeing volume lock after failed creation: %v", err) + } + } + }() + volume.valid = true // Add the volume to state @@ -110,6 +125,8 @@ func (r *Runtime) removeVolume(ctx context.Context, v *Volume, force bool) error return errors.Wrapf(err, "error removing container %s that depends on volume %s", dep, v.Name()) } + logrus.Debugf("Removing container %s (depends on volume %q)", ctr.ID(), v.Name()) + // TODO: do we want to set force here when removing // containers? // I'm inclined to say no, in case someone accidentally @@ -128,12 +145,24 @@ func (r *Runtime) removeVolume(ctx context.Context, v *Volume, force bool) error return errors.Wrapf(err, "error removing volume %s", v.Name()) } - // Delete the mountpoint path of the volume, that is delete the volume from /var/lib/containers/storage/volumes + var removalErr error + + // Free the volume's lock + if err := v.lock.Free(); err != nil { + removalErr = errors.Wrapf(err, "error freeing lock for volume %s", v.Name()) + } + + // Delete the mountpoint path of the volume, that is delete the volume + // from /var/lib/containers/storage/volumes if err := v.teardownStorage(); err != nil { - return errors.Wrapf(err, "error cleaning up volume storage for %q", v.Name()) + if removalErr == nil { + removalErr = errors.Wrapf(err, "error cleaning up volume storage for %q", v.Name()) + } else { + logrus.Errorf("error cleaning up volume storage for volume %q: %v", v.Name(), err) + } } defer v.newVolumeEvent(events.Remove) logrus.Debugf("Removed volume %s", v.Name()) - return nil + return removalErr } diff --git a/libpod/state.go b/libpod/state.go index d0ad1a1f8..5d704e69a 100644 --- a/libpod/state.go +++ b/libpod/state.go @@ -115,12 +115,20 @@ type State interface { // answer is this: use this only very sparingly, and only if you really // know what you're doing. RewriteContainerConfig(ctr *Container, newCfg *ContainerConfig) error - // PLEASE READ THE ABOVE DESCRIPTION BEFORE USING. + // PLEASE READ THE DESCRIPTION FOR RewriteContainerConfig BEFORE USING. // This function is identical to RewriteContainerConfig, save for the // fact that it is used with pods instead. // It is subject to the same conditions as RewriteContainerConfig. // Please do not use this unless you know what you're doing. RewritePodConfig(pod *Pod, newCfg *PodConfig) error + // PLEASE READ THE DESCRIPTION FOR RewriteContainerConfig BEFORE USING. + // This function is identical to RewriteContainerConfig, save for the + // fact that it is used with volumes instead. + // It is subject to the same conditions as RewriteContainerConfig. + // The exception is that volumes do not have IDs, so only volume name + // cannot be altered. + // Please do not use this unless you know what you're doing. + RewriteVolumeConfig(volume *Volume, newCfg *VolumeConfig) error // Accepts full ID of pod. // If the pod given is not in the set namespace, an error will be diff --git a/libpod/volume.go b/libpod/volume.go index 74126b49b..abfa7b3f4 100644 --- a/libpod/volume.go +++ b/libpod/volume.go @@ -2,6 +2,8 @@ package libpod import ( "time" + + "github.com/containers/libpod/libpod/lock" ) // Volume is the type used to create named volumes @@ -11,21 +13,35 @@ type Volume struct { valid bool runtime *Runtime + lock lock.Locker } // VolumeConfig holds the volume's config information type VolumeConfig struct { - // Name of the volume + // Name of the volume. Name string `json:"name"` - - Labels map[string]string `json:"labels"` - Driver string `json:"driver"` - MountPoint string `json:"mountPoint"` - CreatedTime time.Time `json:"createdAt,omitempty"` - Options map[string]string `json:"options"` - IsCtrSpecific bool `json:"ctrSpecific"` - UID int `json:"uid"` - GID int `json:"gid"` + // ID of the volume's lock. + LockID uint32 `json:"lockID"` + // Labels for the volume. + Labels map[string]string `json:"labels"` + // The volume driver. Empty string or local does not activate a volume + // driver, all other volumes will. + Driver string `json:"driver"` + // The location the volume is mounted at. + MountPoint string `json:"mountPoint"` + // Time the volume was created. + CreatedTime time.Time `json:"createdAt,omitempty"` + // Options to pass to the volume driver. For the local driver, this is + // a list of mount options. For other drivers, they are passed to the + // volume driver handling the volume. + Options map[string]string `json:"options"` + // Whether this volume was created for a specific container and will be + // removed with it. + IsCtrSpecific bool `json:"ctrSpecific"` + // UID the volume will be created as. + UID int `json:"uid"` + // GID the volume will be created as. + GID int `json:"gid"` } // Name retrieves the volume's name diff --git a/test/e2e/run_volume_test.go b/test/e2e/run_volume_test.go index 1e0b84310..abb93a149 100644 --- a/test/e2e/run_volume_test.go +++ b/test/e2e/run_volume_test.go @@ -154,4 +154,12 @@ var _ = Describe("Podman run with volumes", func() { session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Not(Equal(0))) }) + + It("podman run with volume flag and multiple named volumes", func() { + session := podmanTest.Podman([]string{"run", "--rm", "-v", "testvol1:/testvol1", "-v", "testvol2:/testvol2", ALPINE, "grep", "/testvol", "/proc/self/mountinfo"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + Expect(session.OutputToString()).To(ContainSubstring("/testvol1")) + Expect(session.OutputToString()).To(ContainSubstring("/testvol2")) + }) }) diff --git a/test/e2e/systemd_test.go b/test/e2e/systemd_test.go index 91604867d..02778d493 100644 --- a/test/e2e/systemd_test.go +++ b/test/e2e/systemd_test.go @@ -5,7 +5,10 @@ package integration import ( "io/ioutil" "os" + "strings" + "time" + "github.com/containers/libpod/pkg/cgroups" . "github.com/containers/libpod/test/utils" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -77,4 +80,49 @@ WantedBy=multi-user.target status := SystemExec("bash", []string{"-c", "systemctl status redis"}) Expect(status.OutputToString()).To(ContainSubstring("active (running)")) }) + + It("podman run container with systemd PID1", func() { + cgroupsv2, err := cgroups.IsCgroup2UnifiedMode() + Expect(err).To(BeNil()) + if cgroupsv2 { + Skip("systemd test does not work in cgroups V2 mode yet") + } + + systemdImage := "fedora" + pull := podmanTest.Podman([]string{"pull", systemdImage}) + pull.WaitWithDefaultTimeout() + Expect(pull.ExitCode()).To(Equal(0)) + + ctrName := "testSystemd" + run := podmanTest.Podman([]string{"run", "--name", ctrName, "-t", "-i", "-d", systemdImage, "init"}) + run.WaitWithDefaultTimeout() + Expect(run.ExitCode()).To(Equal(0)) + ctrID := run.OutputToString() + + logs := podmanTest.Podman([]string{"logs", ctrName}) + logs.WaitWithDefaultTimeout() + Expect(logs.ExitCode()).To(Equal(0)) + + // Give container 10 seconds to start + started := false + for i := 0; i < 10; i++ { + runningCtrs := podmanTest.Podman([]string{"ps", "-q", "--no-trunc"}) + runningCtrs.WaitWithDefaultTimeout() + Expect(runningCtrs.ExitCode()).To(Equal(0)) + + if strings.Contains(runningCtrs.OutputToString(), ctrID) { + started = true + break + } + + time.Sleep(1 * time.Second) + } + + Expect(started).To(BeTrue()) + + systemctl := podmanTest.Podman([]string{"exec", "-t", "-i", ctrName, "systemctl", "status", "--no-pager"}) + systemctl.WaitWithDefaultTimeout() + Expect(systemctl.ExitCode()).To(Equal(0)) + Expect(strings.Contains(systemctl.OutputToString(), "State:")).To(BeTrue()) + }) }) |