diff options
141 files changed, 4496 insertions, 515 deletions
diff --git a/.cirrus.yml b/.cirrus.yml index 8389c638d..dfcd86a5d 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -30,9 +30,9 @@ env: #### #### Cache-image names to test with ### - FEDORA_CACHE_IMAGE_NAME: "fedora-30-libpod-5479475851034624" - PRIOR_FEDORA_CACHE_IMAGE_NAME: "fedora-29-libpod-5479475851034624" - UBUNTU_CACHE_IMAGE_NAME: "ubuntu-18-libpod-5479475851034624" + FEDORA_CACHE_IMAGE_NAME: "fedora-30-libpod-5789386598252544" + PRIOR_FEDORA_CACHE_IMAGE_NAME: "fedora-29-libpod-5789386598252544" + UBUNTU_CACHE_IMAGE_NAME: "ubuntu-18-libpod-5789386598252544" #### #### Variables for composing new cache-images (used in PR testing) from @@ -47,6 +47,7 @@ env: #### SPECIALMODE: "none" # don't do anything special TEST_REMOTE_CLIENT: false # don't test remote client by default + ADD_SECOND_PARTITION: false # will certainly fail inside containers #### #### Credentials and other secret-sauces, decrypted at runtime when authorized. @@ -322,6 +323,7 @@ testing_task: timeout_in: 120m env: + ADD_SECOND_PARTITION: true matrix: TEST_REMOTE_CLIENT: true TEST_REMOTE_CLIENT: false @@ -343,6 +345,49 @@ testing_task: audit_log_script: '$SCRIPT_BASE/logcollector.sh audit' journal_script: '$SCRIPT_BASE/logcollector.sh journal' +# Test crun on last Fedora +testing_crun_task: + + depends_on: + - "gating" + - "vendor" + - "varlink_api" + - "build_each_commit" + - "build_without_cgo" + + # Only test build cache-images, if that's what's requested + only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\*\*\*\s*CIRRUS:\s*TEST\s*IMAGES\s*\*\*\*.*' + + gce_instance: + matrix: + # Images are generated separately, from build_images_task (below) + image_name: "${FEDORA_CACHE_IMAGE_NAME}" + + timeout_in: 120m + + env: + ADD_SECOND_PARTITION: true + OCI_RUNTIME: "/usr/bin/crun" + matrix: + TEST_REMOTE_CLIENT: false + + setup_environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}' + unit_test_script: '$SCRIPT_BASE/unit_test.sh |& ${TIMESTAMP}' + integration_test_script: '$SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP}' + system_test_script: '$SCRIPT_BASE/system_test.sh |& ${TIMESTAMP}' + cache_release_archive_script: >- + [[ "$TEST_REMOTE_CLIENT" == "false" ]] || \ + $SCRIPT_BASE/cache_release_archive.sh |& ${TIMESTAMP} + + on_failure: + failed_branch_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_branch_failure.sh' + + always: &crunstandardlogs + ginkgo_node_logs_script: '$SCRIPT_BASE/logcollector.sh ginkgo' + df_script: '$SCRIPT_BASE/logcollector.sh df' + audit_log_script: '$SCRIPT_BASE/logcollector.sh audit' + journal_script: '$SCRIPT_BASE/logcollector.sh journal' + # This task executes tests under unique environments/conditions special_testing_rootless_task: @@ -357,6 +402,7 @@ special_testing_rootless_task: only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\*\*\*\s*CIRRUS:\s*TEST\s*IMAGES\s*\*\*\*.*' env: + ADD_SECOND_PARTITION: true SPECIALMODE: 'rootless' # See docs matrix: @@ -388,6 +434,7 @@ special_testing_in_podman_task: only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\*\*\*\s*CIRRUS:\s*TEST\s*IMAGES\s*\*\*\*.*' env: + ADD_SECOND_PARTITION: true SPECIALMODE: 'in_podman' # See docs timeout_in: 60m @@ -481,6 +528,7 @@ verify_test_built_images_task: image_name: "ubuntu-18${BUILT_IMAGE_SUFFIX}" env: + ADD_SECOND_PARTITION: true matrix: TEST_REMOTE_CLIENT: true TEST_REMOTE_CLIENT: false @@ -513,6 +561,7 @@ success_task: - "meta" - "image_prune" - "testing" + - "testing_crun" - "special_testing_rootless" - "special_testing_in_podman" - "special_testing_cross" @@ -551,6 +600,7 @@ release_task: - "meta" - "image_prune" - "testing" + - "testing_crun" - "special_testing_rootless" - "special_testing_in_podman" - "special_testing_cross" @@ -2,7 +2,7 @@ export GO111MODULE=off GO ?= go DESTDIR ?= -EPOCH_TEST_COMMIT ?= 55e028a12ee003e057c65e376fe4b723d28ae52e +EPOCH_TEST_COMMIT ?= bb80586e275fe0d3f47700ec54c9718a28b1e59c HEAD ?= HEAD CHANGELOG_BASE ?= HEAD~ CHANGELOG_TARGET ?= HEAD @@ -253,10 +253,34 @@ remoteintegration: varlink_generate test-binaries ginkgo-remote localsystem: # Wipe existing config, database, and cache: start with clean slate. $(RM) -rf ${HOME}/.local/share/containers ${HOME}/.config/containers - if timeout -v 1 true; then PODMAN=./bin/podman bats test/system/; else echo "Skipping localsystem: 'timeout -v' unavailable'"; fi + if timeout -v 1 true; then PODMAN=./bin/podman bats test/system/; else echo "Skipping $@: 'timeout -v' unavailable'"; fi remotesystem: - @echo "remotesystem - unimplemented" + # Wipe existing config, database, and cache: start with clean slate. + $(RM) -rf ${HOME}/.local/share/containers ${HOME}/.config/containers + # Start varlink server using tmp socket; loop-wait for it; + # test podman-remote; kill server, clean up tmp socket file. + # varlink server spews copious unhelpful output; ignore it. + rc=0;\ + if timeout -v 1 true; then \ + SOCK_FILE=$(shell mktemp --dry-run --tmpdir io.podman.XXXXXX);\ + export PODMAN_VARLINK_ADDRESS=unix:$$SOCK_FILE; \ + ./bin/podman varlink --timeout=0 $$PODMAN_VARLINK_ADDRESS &>/dev/null & \ + retry=5;\ + while [[ $$retry -ge 0 ]]; do\ + echo Waiting for varlink server...;\ + sleep 1;\ + ./bin/podman-remote info &>/dev/null && break;\ + retry=$$(expr $$retry - 1);\ + done;\ + env PODMAN=./bin/podman-remote bats test/system/ ;\ + rc=$$?;\ + kill %1;\ + rm -f $$SOCK_FILE;\ + else \ + echo "Skipping $@: 'timeout -v' unavailable'";\ + fi;\ + exit $$rc system.test-binary: .install.ginkgo $(GO) test -c ./test/system @@ -333,12 +357,12 @@ changelog: ## Generate changelog install: .gopathok install.bin install.remote install.man install.cni install.systemd ## Install binaries to system locations -install.remote: +install.remote: podman-remote install ${SELINUXOPT} -d -m 755 $(DESTDIR)$(BINDIR) install ${SELINUXOPT} -m 755 bin/podman-remote $(DESTDIR)$(BINDIR)/podman-remote test -z "${SELINUXOPT}" || chcon --verbose --reference=$(DESTDIR)$(BINDIR)/podman bin/podman-remote -install.bin: +install.bin: podman install ${SELINUXOPT} -d -m 755 $(DESTDIR)$(BINDIR) install ${SELINUXOPT} -m 755 bin/podman $(DESTDIR)$(BINDIR)/podman test -z "${SELINUXOPT}" || chcon --verbose --reference=$(DESTDIR)$(BINDIR)/podman bin/podman @@ -7,6 +7,7 @@ popularized by Kubernetes. Libpod also contains the Pod Manager tool `(Podman)` * [Latest Version: 1.4.4](https://github.com/containers/libpod/releases/latest) * [Continuous Integration:](contrib/cirrus/README.md) [](https://cirrus-ci.com/github/containers/libpod/master) +* [GoDoc: ](https://godoc.org/github.com/containers/libpod/libpod) ## Overview and scope diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 69244bb09..f55fd9b18 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,5 +1,68 @@ # Release Notes +## 1.5.0 +### Features +- Podman containers can now join the user namespaces of other containers with `--userns=container:$ID`, or a user namespace at an arbitary path with `--userns=ns:$PATH` +- Rootless Podman can experimentally squash all UIDs and GIDs in an image to a single UID and GID (which does not require use of the `newuidmap` and `newgidmap` executables) by passing `--storage-opt ignore_chown_errors` +- The `podman generate kube` command now produces YAML for any bind mounts the container has created ([#2303](https://github.com/containers/libpod/issues/2303)) +- The `podman container restore` command now features a new flag, `--ignore-static-ip`, that can be used with `--import` to import a single container with a static IP multiple times on the same host +- Added the ability for `podman events` to output JSON by specifying `--format=json` +- If the OCI runtime or `conmon` binary cannot be found at the paths specified in `libpod.conf`, Podman will now also search for them in the calling user's path +- Added the ability to use `podman import` with URLs ([#3609](https://github.com/containers/libpod/issues/3609)) +- The `podman ps` command now supports filtering names using regular expressions ([#3394](https://github.com/containers/libpod/issues/3394)) +- Rootless Podman containers with `--privileged` set will now mount in all host devices that the user can access +- The `podman create` and `podman run` commands now support the `--env-host` flag to forward all environment variables from the host into the container +- Rootless Podman now supports healthchecks ([#3523](https://github.com/containers/libpod/issues/3523)) +- The format of the `HostConfig` portion of the output of `podman inspect` on containers has been improved and synced with Docker +- Podman containers now support CGroup namespaces, and can create them by passing `--cgroupns=private` to `podman run` or `podman create` +- The `podman create` and `podman run` commands now support the `--ulimit=host` flag, which uses any ulimits currently set on the host for the container +- The `podman rm` and `podman rmi` commands now use different exit codes to indicate 'no such container' and 'container is running' errors +- Support for CGroups V2 through the `crun` OCI runtime has been greatly improved, allowing resource limits to be set for rootless containers when the CGroups V2 hierarchy is in use + +### Bugfixes +- Fixed a bug where a race condition could cause `podman restart` to fail to start containers with ports +- Fixed a bug where containers restored from a checkpoint would not properly report the time they were started at +- Fixed a bug where `podman search` would return at most 25 results, even when the maximum number of results was set higher +- Fixed a bug where `podman play kube` would not honor capabilities set in imported YAML ([#3689](https://github.com/containers/libpod/issues/3689)) +- Fixed a bug where `podman run --env`, when passed a single key (to use the value from the host), would set the environment variable in the container even if it was not set on the host ([#3648](https://github.com/containers/libpod/issues/3648)) +- Fixed a bug where `podman commit --changes` would not properly set environment variables +- Fixed a bug where Podman could segfault while working with images with no history +- Fixed a bug where `podman volume rm` could remove arbitrary volumes if given an ambiguous name ([#3635](https://github.com/containers/libpod/issues/3635)) +- Fixed a bug where `podman exec` invocations leaked memory by not cleaning up files in tmpfs +- Fixed a bug where the `--dns` and `--net=container` flags to `podman run` and `podman create` were not mutually exclusive ([#3553](https://github.com/containers/libpod/issues/3553)) +- Fixed a bug where rootless Podman would be unable to run containers when less than 5 UIDs were available +- Fixed a bug where containers in pods could not be removed without removing the entire pod ([#3556](https://github.com/containers/libpod/issues/3556)) +- Fixed a bug where Podman would not properly clean up all CGroup controllers for created cgroups when using the `cgroupfs` CGroup driver +- Fixed a bug where Podman containers did not properly clean up files in tmpfs, resulting in a memory leak as containers stopped +- Fixed a bug where healthchecks from images would not use default settings for interval, retries, timeout, and start period when they were not provided by the image ([#3525](https://github.com/containers/libpod/issues/3525)) +- Fixed a bug where healthchecks using the `HEALTHCHECK CMD` format where not properly supported ([#3507](https://github.com/containers/libpod/issues/3507)) +- Fixed a bug where volume mounts using relative source paths would not be properly resolved ([#3504](https://github.com/containers/libpod/issues/3504)) +- Fixed a bug where `podman run` did not use authorization credentials when a custom path was specified ([#3524](https://github.com/containers/libpod/issues/3524)) +- Fixed a bug where containers checkpointed with `podman container checkpoint` did not properly set their finished time +- Fixed a bug where running `podman inspect` on any container not created with `podman run` or `podman create` (for example, pod infra containers) would result in a segfault ([#3500](https://github.com/containers/libpod/issues/3500)) +- Fixed a bug where healthcheck flags for `podman create` and `podman run` were incorrectly named ([#3455](https://github.com/containers/libpod/pull/3455)) +- Fixed a bug where Podman commands would fail to find targets if a partial ID was specified that was ambiguous between a container and pod ([#3487](https://github.com/containers/libpod/issues/3487)) +- Fixed a bug where restored containers would not have the correct SELinux label +- Fixed a bug where Varlink endpoints were not working properly if `more` was not correctly specified +- Fixed a bug where the Varlink PullImage endpoint would crash if an error occurred ([#3715](https://github.com/containers/libpod/issues/3715)) +- Fixed a bug where the `--mount` flag to `podman create` and `podman run` did not allow boolean arguments for its `ro` and `rw` options ([#2980](https://github.com/containers/libpod/issues/2980)) +- Fixed a bug where pods did not properly share the UTS namespace, resulting in incorrect behavior from some utilities which rely on hostname ([#3547](https://github.com/containers/libpod/issues/3547)) +- Fixed a bug where Podman would unconditionally append `ENTRYPOINT` to `CMD` during `podman commit` (and when reporting `CMD` in `podman inspect`) ([#3708](https://github.com/containers/libpod/issues/3708)) +- Fixed a bug where `podman events` with the `journald` events backend would incorrectly print 6 previous events when only new events were requested ([#3616](https://github.com/containers/libpod/issues/3616)) +- Fixed a bug where `podman port` would exit prematurely when a port number was specified ([#3747](https://github.com/containers/libpod/issues/3747)) +- Fixed a bug where passing `.` as an argument to the `--dns-search` flag to `podman create` and `podman run` was not properly clearing DNS search domains in the container + +### Misc +- Updated vendored Buildah to v1.10.1 +- Updated vendored containers/image to v3.0.2 +- Updated vendored containers/storage to v1.13.1 +- Podman now requires conmon v2.0.0 or higher +- The `podman info` command now displays the events logger being in use +- The `podman inspect` command on containers now includes the ID of the pod a container has joined and the PID of the container's conmon process +- The `-v` short flag for `podman --version` has been re-added +- Error messages from `podman pull` should be significantly clearer +- The `podman exec` command is now available in the remote client + ## 1.4.4 ### Bugfixes - Fixed a bug where rootless Podman would attempt to use the entire root configuration if no rootless configuration was present for the user, breaking rootless Podman for new installations diff --git a/changelog.txt b/changelog.txt index 51ac92979..beea8dd5c 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,3 +1,251 @@ +- Changelog for v1.5.0 (2019-08-09) + * vendor github.com/containers/storage@v1.13.2 + * Improve dns-search validation, empty domains now return an error + * fix create&run getting --authfile from cli + * Add release notes for v1.5.0 + * Touch up build man page + * podman-container-runlabel(1): drop note + * make rmi messages more compatible with docker + * Add conmon probe to runtime construction + * fix copy change file owner if cp from container + * Vendor Buildah 1.10.1 + * Allow the passing of '.' to --dns-search + * add make to make installs + * namespaces: fix Container() call + * Add a test for verifying ENTRYPOINT and CMD + * fix port early return + * Allow --ro=[true|false] with mount flag + * refer to container whose namespace we share + * add test to verify hostname is shared in a pod + * Properly share UTS namespaces in a pod + * When populating CMD, do not include Entrypoint + * systemd library conflict with seektail and addmatch + * pod top test: reenable + * cgroup: fix regression when running systemd + * Add invalid credentials fix to docs + * Revert "rootless: Rearrange setup of rootless containers" + * restore: correctly set StartedTime + * container stop: kill conmon + * honor libpod.conf in /usr/share/containers + * fix system df crashes on unnamed images + * Don't log errors to the screen when XDG_RUNTIME_DIR is not set + * various fixes for varlink endpoints + * add eventlogger to info + * Add handling for empty LogDriver + * Add rootless NFS and OverlayFS warnings to docs + * podman events format json + * add godoc link to readme + * restore: added --ignore-static-ip option + * System tests: resolve hang in rawhide rootless + * fix search output limit + * Add capability functionality to play kube + * Use "none" instead of "null" for the null eventer + * Deduplicate capabilities in generate kube + * Fix typo + * Pass on events-backend config to cleanup processes + * Print Pod ID in `podman inspect` output + * go build: use `-mod=vendor` for go >= 1.11.x + * Use buildah/pkg/parse volume parsing rather then internal version + * github.com/containers/storage v1.12.13 + * Add new exit codes to rm & rmi for running containers & dependencies + * Add runtime and conmon path discovery + * systemd, cgroupsv2: not bind mount /sys/fs/cgroup/systemd + * Ensure we generate a 'stopped' event on force-remove + * Fix Dockerfile - a dependency's name was changed + * System events are valid, don't error on them + * Do not use an events backend when restoring images + * Expose Null eventer and allow its use in the Podman CLI + * Force tests to use file backend for events + * Add a flag to set events logger type + * Fix test suite + * Retrieve exit codes for containers via events + * podman: fix memleak caused by renaming and not deleting the exit file + * Cirrus: Fix release dependencies + * Cirrus: Fix re-run of release task into no-op. + * e2e test: check exit codes for pull, save, inspect + * rootless: Rearrange setup of rootless containers + * Add comment to describe postConfigureNetNS + * Vendor in buildah 1.9.2 + * Build fix for 32-bit systems. + * Set -env variables as appropriate + * Touch up input argument error on create + * Update libpod.conf to be NixOS friendly + * Allow info test to work with usernames w/dash + * Touch up XDG, add rootless links + * Fix the syntax in the podman export documentation example + * fix `podman -v` regression + * Move random IP code for tests from checkpoint to common + * Fix commit --changes env=X=Y + * Update pause/unpause video links and demo + * Cirrus: Remove fixed clone depth + * podman: support --userns=ns|container + * pods: do not to join a userns if there is not any + * Documenation & build automation for remote darwin + * Cirrus: Bypass release during image-building + * Use systemd cgroups for Ubuntu + * Cirrus: Ubuntu: Set + Test for $RUNC_BINARY + * Cirrus: Simplify evil-unit check in image + * Cirrus: Silence systemd-banish noise + * Cirrus: Fix image build metadata update + * Cirrus: Fix missing -n on CentOS + * Cirrus: Remove disused COMMIT variables + * Improved hooks monitoring + * Fix possible runtime panic if image history len is zero + * When retrieving volumes, only use exact names + * fix import not ignoring url path + * Document SELinux label requirements for the rootfs argument + * Fixes issue #3577. + * refactor to reduce duplicated error parsing + * remove debug prints + * Re-add int64 casts for ctime + * fix build --network=container + * Fix a segfault on Podman no-store commands with refresh + * always send generic error in case io fails + * only use stdin if specified + * buffer errChan + * move handleTerminalAttach to generic build + * remove unnecessary conversions + * add detach keys support for remote + * move editing of exitCode to runtime + * Update e2e tests for remote exec + * Finish up remote exec implementation + * golangci-lint cleanup + * install.md: mention all build tags + * golangci-lint phase 4 + * Change wait to sleep in podmanimage readme + * bump cirrus images to get new conmon + * Implement conmon exec + * bump conmon to 1.0.0-rc2 + * Cirrus: Temp. workaround missing imgprune image + * vendor github.com/containers/image@v2.0.1 + * golangci-lint round #3 + * Remove debug message + * Cleanup Pull Message + * Cirrus: Fix post-merge env. var. not set. + * mkdir -p /etc/cni/net.d requires sudo + * Add support for listing read/only and read/write images + * support podman ps filter regular expressions + * rootless: add rw devices with --privileged + * Cirrus: Minor scripting typo fix + * fix --dns* and --network not set to host conflict + * podman-remote make --size optional in ps + * Remove exec PID files after use to prevent memory leaks + * Add DefaultContent API to retrieve apparmor profile content + * libpod: support for cgroup namespace + * Make GOPATH-related symlinking more precise + * Populate inspect with security-opt settings + * Properly retrieve Conmon PID + * Move the HostConfig portion of Inspect inside libpod + * Fix play kube command + * spec: rework --ulimit host + * Cirrus: Add image-test for locked dpkg + * Cirrus: Use images w/o periodic svcs + * Cirrus: Disable most periodic services/timers + * dependency/analyses: simplify scripts + * dependency-tree analysis: direct and transitive + * analyses: README: consistent code examples + * analyses: README: fix typos + * analyses: add dependency-tree.sh + * analyses: add README.md + * hack/analyses -> dependencies/analyses + * hack/analyses/go-archive-analysis.sh: fix sorting + * add hack/analyses/nm-symbols-analysis.sh + * analyse package sizes + * Completion: complete "--health-start-period" in bash + * Make the healthcheck flags compatible with Docker CLI + * healthcheck: reject empty commands + * create: ignore check if image has HEALTHCHECK NONE + * create: apply defaults on image healthcheck options + * healthcheck: improve command list parser + * Completion: --no-healthcheck is not an option + * Cirrus: Abstract destination branch refs. + * Cirrus: Print images that should be pruned + * create: improve parser for --healthcheck-command + * Improves STD output/readability in combination with debug output. + * Fix the double replySendFile() + * Cirrus: Update to freshly built cache-images + * Cirrus: Execute system-tests during image-validation + * Cirrus: Fix missing removal of packaged podman + * cgroupsv2: do not enable controllers for the last component + * spec: fix userns with less than 5 gids + * Fix spelling mistakes in man pages and other docs + * Add glob parsing for --env flag + * Add support for -env-host + * cgroups: fix a leak when using cgroupfs + * cgroups: attempt a recursive rmdir + * Fix a bug where ctrs could not be removed from pods + * golangci-lint pass number 2 + * Add tests for --ignore-rootfs checkpoint/restore option + * Add --ignore-rootfs option for checkpoint/restore + * Fix typo in checkpoint/restore related texts + * Include root file-system changes in container migration + * Add function to get a filtered tarstream diff + * Correctly set FinishedTime for checkpointed container + * first pass of corrections for golangci-lint + * Cirrus: Fix #3543: Failure in 'release' task + * fix bug convert volume host path to absolute + * Cirrus: Fix 473d06045 / enable build_without_cgo + * account for varlink calls that dont use more + * runtime: drop spurious message log + * Ensure we have a valid store when we refresh + * cgroups: skip not existing cpuacct files + * cgroups: support creating cgroupsv2 paths + * make localsystem: wipe all user config state + * podman: create and run honors auth file location + * healthcheck: support rootless mode + * Use random IP addresses during checkpoint/restore tests + * Fix podman-remote usage message to display `podman-remote` instead of `podman` + * rootless.md: Include GPFS as a parallel filesystem + * speed up rootless tests + * podman: add --ulimit host + * docs: fix --healthcheck-command option + * code cleanup + * fix integration flake tests + * CONTRIBUTING.md: fix project paths + * get last container event + * Do not hardcode podman binary location in generate systemd. + * Move skipping systemd tests to early setup. + * Reload systemd daemon on creation of units location dir in tests. + * Add debug information to "generate systemd" test. + * Use default conmon pidfile location for root containers. + * Use conmon pidfile in generated systemd unit as PIDFile. + * Cirrus: Automate releasing of tested binaries + * trivial cleanups from golang + * ps should use nostore when possible + * libpod: discerne partial IDs between containers and pods + * Added instruction to enable the user namespaces permanenty in Manjaro + * Addressed code review comments + * Updated install.md for Manjaro Linux + * Vendor latest OCICNI version + * Bump current version in README + * Wipe PID and ConmonPID in state after container stops + * Store Conmon's PID in our state and display in inspect + * Restart failed containers in tests + * Improve parsing of mounts + * Add test for generate kube with volumes + * Bump gitvalidation epoch + * Bump to v1.4.5-dev + * Fix rootless detection error for pause & unpause + * Deduplicate volumes + * cirrus: add test for compiling without cgo + * lock: new lock type "file" + * runtime: allow to specify the lock mechanism + * lock: disable without cgo + * spec: move cgo stuff to their own file + * rootless: allow to build without cgo + * attach: move cgo bits to a different file + * vendor: update containers/psgo + * Update the testing documentation with system tests. + * Pass along volumes to pod yaml + * Configure container volumes for generate kube + * configure runtime without store + * Add RUN priv'd test for build + * Cirrus: Use packaged dependencies + * Add exec after checkpoint/restore test + * Provide correct SELinux mount-label for restored container + * Track if a container is restored from an exported checkpoint + * libpod/container_internal: Make all errors loading explicitly configured hook dirs fatal + - Changelog for v1.4.4 (2019-07-02) * Fix release notes * Ensure locks are freed when ctr/pod creation fails diff --git a/cmd/podman/cp.go b/cmd/podman/cp.go index bee7d2199..ad7253ac0 100644 --- a/cmd/podman/cp.go +++ b/cmd/podman/cp.go @@ -140,7 +140,7 @@ func copyBetweenHostAndContainer(runtime *libpod.Runtime, src string, dest strin if err != nil { return errors.Wrapf(err, "error getting IDMappingOptions") } - containerOwner := idtools.IDPair{UID: int(user.UID), GID: int(user.GID)} + destOwner := idtools.IDPair{UID: int(user.UID), GID: int(user.GID)} hostUID, hostGID, err := util.GetHostIDs(convertIDMap(idMappingOpts.UIDMap), convertIDMap(idMappingOpts.GIDMap), user.UID, user.GID) if err != nil { return err @@ -183,6 +183,7 @@ func copyBetweenHostAndContainer(runtime *libpod.Runtime, src string, dest strin destPath = cleanedPath } } else { + destOwner = idtools.IDPair{UID: os.Getuid(), GID: os.Getgid()} if isVol, volDestName, volName := isVolumeDestName(srcPath, ctr); isVol { path, err := pathWithVolumeMount(ctr, runtime, volDestName, volName, srcPath) if err != nil { @@ -230,7 +231,7 @@ func copyBetweenHostAndContainer(runtime *libpod.Runtime, src string, dest strin src = os.Stdin.Name() extract = true } - err := copy(src, destPath, dest, idMappingOpts, &containerOwner, extract, isFromHostToCtr) + err := copy(src, destPath, dest, idMappingOpts, &destOwner, extract, isFromHostToCtr) if lastError != nil { logrus.Error(lastError) } diff --git a/cmd/podman/main_local.go b/cmd/podman/main_local.go index 0f43e0b88..587c8260f 100644 --- a/cmd/podman/main_local.go +++ b/cmd/podman/main_local.go @@ -1,4 +1,5 @@ // +build !remoteclient +// +build linux package main diff --git a/cmd/podman/main_remote.go b/cmd/podman/main_remote.go index d534f5bcb..a005e925c 100644 --- a/cmd/podman/main_remote.go +++ b/cmd/podman/main_remote.go @@ -3,14 +3,8 @@ package main import ( - "fmt" - "os" "os/user" - "path/filepath" - "github.com/containers/libpod/pkg/util" - "github.com/pkg/errors" - "github.com/sirupsen/logrus" "github.com/spf13/cobra" ) @@ -31,49 +25,6 @@ func init() { rootCmd.PersistentFlags().BoolVar(&MainGlobalOpts.Syslog, "syslog", false, "Output logging information to syslog as well as the console") } -func setSyslog() error { - var err error - cfgHomeDir := os.Getenv("XDG_CONFIG_HOME") - if cfgHomeDir == "" { - if cfgHomeDir, err = util.GetRootlessConfigHomeDir(); err != nil { - return err - } - if err = os.Setenv("XDG_CONFIG_HOME", cfgHomeDir); err != nil { - return errors.Wrapf(err, "cannot set XDG_CONFIG_HOME") - } - } - path := filepath.Join(cfgHomeDir, "containers") - - // Log to file if not using syslog - - if _, err := os.Stat(path); os.IsNotExist(err) { - if err := os.MkdirAll(path, 0750); err != nil { - fmt.Fprintf(os.Stderr, "%v", err) - return err - } - } - - // Update path to include file name - path = filepath.Join(path, "podman.log") - - // Create the log file if doesn't exist. And append to it if it already exists. - file, err := os.OpenFile(path, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0640) - if err != nil { - // Cannot open log file. Logging to stderr - fmt.Fprintf(os.Stderr, "%v", err) - return err - } else { - formatter := new(logrus.TextFormatter) - formatter.FullTimestamp = true - logrus.SetFormatter(formatter) - logrus.SetOutput(file) - } - - // Note this message is only logged if --log-level >= Info! - logrus.Infof("Logging level set to %s", logrus.GetLevel().String()) - return nil -} - func profileOn(cmd *cobra.Command) error { return nil } diff --git a/cmd/podman/main_remote_supported.go b/cmd/podman/main_remote_supported.go new file mode 100644 index 000000000..bb567c273 --- /dev/null +++ b/cmd/podman/main_remote_supported.go @@ -0,0 +1,57 @@ +// +build remoteclient +// +build linux darwin + +package main + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/containers/libpod/pkg/util" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +func setSyslog() error { + var err error + cfgHomeDir := os.Getenv("XDG_CONFIG_HOME") + if cfgHomeDir == "" { + if cfgHomeDir, err = util.GetRootlessConfigHomeDir(); err != nil { + return err + } + if err = os.Setenv("XDG_CONFIG_HOME", cfgHomeDir); err != nil { + return errors.Wrapf(err, "cannot set XDG_CONFIG_HOME") + } + } + path := filepath.Join(cfgHomeDir, "containers") + + // Log to file if not using syslog + + if _, err := os.Stat(path); os.IsNotExist(err) { + if err := os.MkdirAll(path, 0750); err != nil { + fmt.Fprintf(os.Stderr, "%v", err) + return err + } + } + + // Update path to include file name + path = filepath.Join(path, "podman.log") + + // Create the log file if doesn't exist. And append to it if it already exists. + file, err := os.OpenFile(path, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0640) + if err != nil { + // Cannot open log file. Logging to stderr + fmt.Fprintf(os.Stderr, "%v", err) + return err + } else { + formatter := new(logrus.TextFormatter) + formatter.FullTimestamp = true + logrus.SetFormatter(formatter) + logrus.SetOutput(file) + } + + // Note this message is only logged if --log-level >= Info! + logrus.Infof("Logging level set to %s", logrus.GetLevel().String()) + return nil +} diff --git a/cmd/podman/main_remote_windows.go b/cmd/podman/main_remote_windows.go new file mode 100644 index 000000000..0ef1370ce --- /dev/null +++ b/cmd/podman/main_remote_windows.go @@ -0,0 +1,7 @@ +// +build remoteclient,windows + +package main + +func setSyslog() error { + return nil +} diff --git a/cmd/podman/pod_create.go b/cmd/podman/pod_create.go index b6154b4db..d04c85dba 100644 --- a/cmd/podman/pod_create.go +++ b/cmd/podman/pod_create.go @@ -78,7 +78,7 @@ func podCreateCmd(c *cliconfig.PodCreateValues) error { if !c.Infra && c.Flag("share").Changed && c.Share != "none" && c.Share != "" { return errors.Errorf("You cannot share kernel namespaces on the pod level without an infra container") } - if c.Flag("pod-id-file").Changed && os.Geteuid() == 0 { + if c.Flag("pod-id-file").Changed { podIdFile, err = util.OpenExclusiveFile(c.PodIDFile) if err != nil && os.IsExist(err) { return errors.Errorf("pod id file exists. Ensure another pod is not using it or delete %s", c.PodIDFile) diff --git a/cmd/podman/pod_stats.go b/cmd/podman/pod_stats.go index 46cacc026..2f1ebd3ac 100644 --- a/cmd/podman/pod_stats.go +++ b/cmd/podman/pod_stats.go @@ -15,6 +15,8 @@ import ( "github.com/containers/libpod/libpod" "github.com/containers/libpod/libpod/define" "github.com/containers/libpod/pkg/adapter" + "github.com/containers/libpod/pkg/cgroups" + "github.com/containers/libpod/pkg/rootless" "github.com/pkg/errors" "github.com/spf13/cobra" ) @@ -53,9 +55,14 @@ func init() { } func podStatsCmd(c *cliconfig.PodStatsValues) error { - - if os.Geteuid() != 0 { - return errors.New("stats is not supported in rootless mode") + if rootless.IsRootless() { + unified, err := cgroups.IsCgroup2UnifiedMode() + if err != nil { + return err + } + if !unified { + return errors.New("stats is not supported in rootless mode without cgroups v2") + } } format := c.Format diff --git a/cmd/podman/port.go b/cmd/podman/port.go index 5753c8e56..4e1f9642c 100644 --- a/cmd/podman/port.go +++ b/cmd/podman/port.go @@ -48,8 +48,8 @@ func init() { func portCmd(c *cliconfig.PortValues) error { var ( - userProto, containerName string - userPort int + userProto string + userPort int ) args := c.InputArgs @@ -106,6 +106,7 @@ func portCmd(c *cliconfig.PortValues) error { if err != nil { return err } + var found bool // Iterate mappings for _, v := range portmappings { hostIP := v.HostIP @@ -125,12 +126,14 @@ func portCmd(c *cliconfig.PortValues) error { if v.ContainerPort == int32(userPort) { if userProto == "" || userProto == v.Protocol { fmt.Printf("%s:%d\n", hostIP, v.HostPort) + found = true break } - } else { - return errors.Errorf("No public port '%d' published for %s", userPort, containerName) } } + if !found && port != "" { + return errors.Errorf("failed to find published port '%d'", userPort) + } } return nil diff --git a/cmd/podman/shared/create.go b/cmd/podman/shared/create.go index 4de68e4bc..e29e6b28e 100644 --- a/cmd/podman/shared/create.go +++ b/cmd/podman/shared/create.go @@ -55,7 +55,7 @@ func CreateContainer(ctx context.Context, c *GenericCLIResults, runtime *libpod. rootfs = c.InputArgs[0] } - if c.IsSet("cidfile") && os.Geteuid() == 0 { + if c.IsSet("cidfile") { cidFile, err = util.OpenExclusiveFile(c.String("cidfile")) if err != nil && os.IsExist(err) { return nil, nil, errors.Errorf("container id file exists. Ensure another container is not using it or delete %s", c.String("cidfile")) @@ -70,8 +70,8 @@ func CreateContainer(ctx context.Context, c *GenericCLIResults, runtime *libpod. imageName := "" var data *inspect.ImageData = nil - // Set the storage if we are running as euid == 0 and there is no rootfs specified - if rootfs == "" && os.Geteuid() == 0 { + // Set the storage if there is no rootfs specified + if rootfs == "" { var writer io.Writer if !c.Bool("quiet") { writer = os.Stderr @@ -83,7 +83,7 @@ func CreateContainer(ctx context.Context, c *GenericCLIResults, runtime *libpod. } else { return nil, nil, errors.Errorf("error, no input arguments were provided") } - newImage, err := runtime.ImageRuntime().New(ctx, name, rtc.SignaturePolicyPath, GetAuthFile(""), writer, nil, image.SigningOptions{}, false, nil) + newImage, err := runtime.ImageRuntime().New(ctx, name, rtc.SignaturePolicyPath, GetAuthFile(c.String("authfile")), writer, nil, image.SigningOptions{}, false, nil) if err != nil { return nil, nil, err } @@ -588,6 +588,7 @@ func ParseCreateOpts(ctx context.Context, c *GenericCLIResults, runtime *libpod. workDir = data.Config.WorkingDir } + userCommand := []string{} entrypoint := configureEntrypoint(c, data) // Build the command // If we have an entry point, it goes first @@ -597,9 +598,11 @@ func ParseCreateOpts(ctx context.Context, c *GenericCLIResults, runtime *libpod. if len(inputCommand) > 0 { // User command overrides data CMD command = append(command, inputCommand...) + userCommand = append(userCommand, inputCommand...) } else if data != nil && len(data.Config.Cmd) > 0 && !c.IsSet("entrypoint") { // If not user command, add CMD command = append(command, data.Config.Cmd...) + userCommand = append(userCommand, data.Config.Cmd...) } if data != nil && len(command) == 0 { @@ -624,8 +627,16 @@ func ParseCreateOpts(ctx context.Context, c *GenericCLIResults, runtime *libpod. return nil, errors.Errorf("cannot pass additional search domains when also specifying '.'") } + // Check for explicit dns-search domain of '' + if c.Changed("dns-search") && len(c.StringSlice("dns-search")) == 0 { + return nil, errors.Errorf("'' is not a valid domain") + } + // Validate domains are good for _, dom := range c.StringSlice("dns-search") { + if dom == "." { + continue + } if _, err := parse.ValidateDomain(dom); err != nil { return nil, err } @@ -680,6 +691,7 @@ func ParseCreateOpts(ctx context.Context, c *GenericCLIResults, runtime *libpod. Cgroupns: c.String("cgroupns"), CgroupParent: c.String("cgroup-parent"), Command: command, + UserCommand: userCommand, Detach: c.Bool("detach"), Devices: c.StringSlice("device"), DNSOpt: c.StringSlice("dns-opt"), diff --git a/cmd/podman/shared/intermediate.go b/cmd/podman/shared/intermediate.go index 4062ac48a..3479876b4 100644 --- a/cmd/podman/shared/intermediate.go +++ b/cmd/podman/shared/intermediate.go @@ -366,6 +366,7 @@ func NewIntermediateLayer(c *cliconfig.PodmanCommand, remote bool) GenericCLIRes m["add-host"] = newCRStringSlice(c, "add-host") m["annotation"] = newCRStringSlice(c, "annotation") m["attach"] = newCRStringSlice(c, "attach") + m["authfile"] = newCRString(c, "authfile") m["blkio-weight"] = newCRString(c, "blkio-weight") m["blkio-weight-device"] = newCRStringSlice(c, "blkio-weight-device") m["cap-add"] = newCRStringSlice(c, "cap-add") diff --git a/cmd/podman/stats.go b/cmd/podman/stats.go index 3accae1b6..2f696445e 100644 --- a/cmd/podman/stats.go +++ b/cmd/podman/stats.go @@ -2,7 +2,6 @@ package main import ( "fmt" - "os" "reflect" "strings" "time" @@ -13,6 +12,8 @@ import ( "github.com/containers/libpod/cmd/podman/libpodruntime" "github.com/containers/libpod/libpod" "github.com/containers/libpod/libpod/define" + "github.com/containers/libpod/pkg/cgroups" + "github.com/containers/libpod/pkg/rootless" "github.com/docker/go-units" "github.com/pkg/errors" "github.com/spf13/cobra" @@ -66,8 +67,14 @@ func init() { } func statsCmd(c *cliconfig.StatsValues) error { - if os.Geteuid() != 0 { - return errors.New("stats is not supported for rootless containers") + if rootless.IsRootless() { + unified, err := cgroups.IsCgroup2UnifiedMode() + if err != nil { + return err + } + if !unified { + return errors.New("stats is not supported in rootless mode without cgroups v2") + } } all := c.All diff --git a/contrib/cirrus/add_second_partition.sh b/contrib/cirrus/add_second_partition.sh new file mode 100644 index 000000000..73db192c5 --- /dev/null +++ b/contrib/cirrus/add_second_partition.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +# N/B: This script could mega f*!@up your disks if run by mistake. +# it is left without the execute-bit on purpose! + +# $SLASH_DEVICE is the disk device to be f*xtuP +SLASH_DEVICE="/dev/sda" # Always the case on GCP + +# The unallocated space results from the difference in disk-size between VM Image +# and runtime request. The check_image.sh test includes a minimum-space check, +# with the Image size set initially lower by contrib/cirrus/packer/libpod_images.yml +NEW_PART_START="50%" +NEW_PART_END="100%" + +set -eo pipefail + +source $(dirname $0)/lib.sh + +if [[ ! -r "/root" ]] || [[ -r "/root/second_partition_ready" ]] +then + echo "Warning: Ignoring attempted execution of $(basename $0)" + exit 0 +fi + +[[ -n "type -P parted" ]] || \ + die 2 "The parted command is required." + +[[ ! -b ${SLASH_DEVICE}2 ]] || \ + die 5 "Found unexpected block device ${SLASH_DEVICE}2" + +PPRINTCMD="parted --script ${SLASH_DEVICE} print" +FINDMNTCMD="findmnt --source=${SLASH_DEVICE}1 --mountpoint=/ --canonicalize --evaluate --first-only --noheadings" +TMPF=$(mktemp -p '' $(basename $0)_XXXX) +trap "rm -f $TMPF" EXIT + +if $FINDMNTCMD | tee $TMPF | egrep -q "^/\s+${SLASH_DEVICE}1" +then + echo "Repartitioning original partition table:" + $PPRINTCMD +else + die 6 "Unexpected output from '$FINDMNTCMD': $(<$TMPF)" +fi + +echo "Adding partition offset within unpartitioned space." +parted --script --align optimal /dev/sda unit % mkpart primary "" "" "$NEW_PART_START" "$NEW_PART_END" + +echo "New partition table:" +$PPRINTCMD + +echo "Growing ${SLASH_DEVICE}1 meet start of ${SLASH_DEVICE}2" +growpart ${SLASH_DEVICE} 1 + +FSTYPE=$(findmnt --first-only --noheadings --output FSTYPE ${SLASH_DEVICE}1) +echo "Expanding $FSTYPE filesystem on ${SLASH_DEVICE}1" +case $FSTYPE in + ext*) resize2fs ${SLASH_DEVICE}1 ;; + *) die 11 "Script $(basename $0) doesn't know how to resize a $FSTYPE filesystem." ;; +esac + +# Must happen last - signals completion to other tooling +echo "Recording newly available disk partition device into /root/second_partition_ready" +echo "${SLASH_DEVICE}2" > /root/second_partition_ready diff --git a/contrib/cirrus/build_vm_images.sh b/contrib/cirrus/build_vm_images.sh index dd5182c37..6230610cb 100755 --- a/contrib/cirrus/build_vm_images.sh +++ b/contrib/cirrus/build_vm_images.sh @@ -62,6 +62,6 @@ URI="gs://packer-import${POST_MERGE_BUCKET_SUFFIX}/manifest${BUILT_IMAGE_SUFFIX} gsutil cp packer-manifest.json "$URI" # Ensure any background 'gcloud compute images update' processes finish -wait # CentOS has no -n option :( +wait # No -n option in CentOS, this is the best that can be done :( echo "Finished. A JSON manifest of produced images is available at $URI" diff --git a/contrib/cirrus/check_image.sh b/contrib/cirrus/check_image.sh index 22ed1ddc4..8a9fbae1d 100755 --- a/contrib/cirrus/check_image.sh +++ b/contrib/cirrus/check_image.sh @@ -7,7 +7,7 @@ source $(dirname $0)/lib.sh NFAILS=0 echo "Validating VM image" -MIN_SLASH_GIGS=50 +MIN_SLASH_GIGS=30 read SLASH_DEVICE SLASH_FSTYPE SLASH_SIZE JUNK <<<$(findmnt --df --first-only --noheadings / | cut -d '.' -f 1) SLASH_SIZE_GIGS=$(echo "$SLASH_SIZE" | sed -r -e 's/G|g//') item_test "Minimum available disk space" $SLASH_SIZE_GIGS -gt $MIN_SLASH_GIGS || let "NFAILS+=1" diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh index 737ca3c01..ffb7cd45b 100644 --- a/contrib/cirrus/lib.sh +++ b/contrib/cirrus/lib.sh @@ -100,6 +100,9 @@ OS_RELEASE_VER="$(source /etc/os-release; echo $VERSION_ID | cut -d '.' -f 1)" # Combined to ease soe usage OS_REL_VER="${OS_RELEASE_ID}-${OS_RELEASE_VER}" +# Installed into cache-images, supports overrides +# by user-data in case of breakage or for debugging. +CUSTOM_CLOUD_CONFIG_DEFAULTS="$GOSRC/$PACKER_BASE/cloud-init/$OS_RELEASE_ID/cloud.cfg.d" # Pass in a list of one or more envariable names; exit non-zero with # helpful error message if any value is empty req_env_var() { @@ -354,7 +357,7 @@ remove_packaged_podman_files(){ } systemd_banish(){ - echo "Disabling periodic services that could destabilize testing:" + echo "Disabling periodic services that could destabilize testing (ignoring errors):" set +e # Not all of these exist on every platform for unit in $EVIL_UNITS do @@ -372,11 +375,20 @@ systemd_banish(){ _finalize(){ set +e # Don't fail at the very end - set +e # make errors non-fatal - echo "Removing leftover giblets from cloud-init" + if [[ -d "$CUSTOM_CLOUD_CONFIG_DEFAULTS" ]] + then + echo "Installing custom cloud-init defaults" + sudo cp -v "$CUSTOM_CLOUD_CONFIG_DEFAULTS"/* /etc/cloud/cloud.cfg.d/ + else + echo "Could not find any files in $CUSTOM_CLOUD_CONFIG_DEFAULTS" + fi + echo "Re-initializing so next boot does 'first-boot' setup again." + sudo history -c cd / sudo rm -rf /var/lib/cloud/instanc* sudo rm -rf /root/.ssh/* + sudo rm -rf /etc/ssh/*key* + sudo rm -rf /etc/ssh/moduli sudo rm -rf /home/* sudo rm -rf /tmp/* sudo rm -rf /tmp/.??* @@ -386,11 +398,6 @@ _finalize(){ rh_finalize(){ set +e # Don't fail at the very end - # Allow root ssh-logins - if [[ -r /etc/cloud/cloud.cfg ]] - then - sudo sed -re 's/^disable_root:.*/disable_root: 0/g' -i /etc/cloud/cloud.cfg - fi echo "Resetting to fresh-state for usage as cloud-image." PKG=$(type -P dnf || type -P yum || echo "") sudo $PKG clean all diff --git a/contrib/cirrus/packer/cloud-init/fedora/cloud.cfg.d/40_enable_root.cfg b/contrib/cirrus/packer/cloud-init/fedora/cloud.cfg.d/40_enable_root.cfg new file mode 100644 index 000000000..672d1907b --- /dev/null +++ b/contrib/cirrus/packer/cloud-init/fedora/cloud.cfg.d/40_enable_root.cfg @@ -0,0 +1 @@ +disable_root: 0 diff --git a/contrib/cirrus/packer/cloud-init/fedora/cloud.cfg.d/50_custom_disk_setup.cfg b/contrib/cirrus/packer/cloud-init/fedora/cloud.cfg.d/50_custom_disk_setup.cfg new file mode 100644 index 000000000..c0fdf0e23 --- /dev/null +++ b/contrib/cirrus/packer/cloud-init/fedora/cloud.cfg.d/50_custom_disk_setup.cfg @@ -0,0 +1,4 @@ +#cloud-config +growpart: + mode: false +resize_rootfs: false diff --git a/contrib/cirrus/packer/cloud-init/ubuntu/cloud.cfg.d/40_enable_root.cfg b/contrib/cirrus/packer/cloud-init/ubuntu/cloud.cfg.d/40_enable_root.cfg new file mode 120000 index 000000000..98a0e3918 --- /dev/null +++ b/contrib/cirrus/packer/cloud-init/ubuntu/cloud.cfg.d/40_enable_root.cfg @@ -0,0 +1 @@ +../../fedora/cloud.cfg.d/40_enable_root.cfg
\ No newline at end of file diff --git a/contrib/cirrus/packer/fedora_setup.sh b/contrib/cirrus/packer/fedora_setup.sh index 1e25a1a3c..e9b145391 100644 --- a/contrib/cirrus/packer/fedora_setup.sh +++ b/contrib/cirrus/packer/fedora_setup.sh @@ -17,6 +17,10 @@ trap "sudo rm -rf $GOPATH" EXIT ooe.sh sudo dnf update -y +echo "Enabling updates-testing repository" +ooe.sh sudo dnf install -y 'dnf-command(config-manager)' +ooe.sh sudo dnf config-manager --set-enabled updates-testing + echo "Installing general build/test dependencies" ooe.sh sudo dnf install -y \ atomic-registries \ diff --git a/contrib/cirrus/packer/libpod_images.yml b/contrib/cirrus/packer/libpod_images.yml index 91ed3b474..2e2b21426 100644 --- a/contrib/cirrus/packer/libpod_images.yml +++ b/contrib/cirrus/packer/libpod_images.yml @@ -33,7 +33,7 @@ builders: image_name: '{{build_name}}{{user `BUILT_IMAGE_SUFFIX`}}' image_family: '{{build_name}}-libpod' source_image: '{{user `UBUNTU_BASE_IMAGE`}}' - disk_size: 20 + disk_size: 20 # REQUIRED: Runtime allocation > this value project_id: '{{user `GCP_PROJECT_ID`}}' service_account_email: '{{user `SERVICE_ACCOUNT`}}' communicator: 'ssh' diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh index 2230684ac..ab7279b11 100755 --- a/contrib/cirrus/setup_environment.sh +++ b/contrib/cirrus/setup_environment.sh @@ -42,8 +42,16 @@ case "${OS_REL_VER}" in ln -f "$CRIO_RUNC_PATH" "/usr/bin/runc" fi ;; - fedora-30) ;; - fedora-29) ;; + fedora-30) ;& # continue to next item + fedora-29) + # There is no crun package on Fedora29 + if test "${OS_REL_VER}" != "fedora-29"; then + yum install -y crun + fi + + if [[ "$ADD_SECOND_PARTITION" == "true" ]]; then + bash "$SCRIPT_BASE/add_second_partition.sh"; fi + ;; centos-7) # Current VM is an image-builder-image no local podman/testing echo "No further setup required for VM image building" exit 0 diff --git a/contrib/spec/podman.spec.in b/contrib/spec/podman.spec.in index 9d8467783..0de797f2b 100644 --- a/contrib/spec/podman.spec.in +++ b/contrib/spec/podman.spec.in @@ -39,7 +39,7 @@ %global shortcommit_conmon %(c=%{commit_conmon}; echo ${c:0:7}) Name: podman -Version: 1.4.5 +Version: 1.5.1 Release: #COMMITDATE#.git%{shortcommit0}%{?dist} Summary: Manage Pods, Containers and Container Images License: ASL 2.0 @@ -354,9 +354,13 @@ providing packages with %{import_path} prefix. %prep %autosetup -Sgit -n %{repo}-%{shortcommit0} -# untar cri-o +# untar conmon tar zxf %{SOURCE1} +sed -i 's/install.remote: podman-remote/install.remote:/' Makefile +sed -i 's/install.bin: podman/install.bin:/' Makefile +sed -i 's/install.man: docs/install.man:/' Makefile + %build mkdir _build pushd _build diff --git a/docs/podman-build.1.md b/docs/podman-build.1.md index c4667070d..878b31080 100644 --- a/docs/podman-build.1.md +++ b/docs/podman-build.1.md @@ -168,6 +168,15 @@ The [username[:password]] to use to authenticate with the registry if required. If one or both values are not supplied, a command line prompt will appear and the value can be entered. The password is entered without echo. +**--disable-compression, -D** + +Don't compress filesystem layers when building the image unless it is required +by the location where the image is being written. This is the default setting, +because image layers are compressed automatically when they are pushed to +registries, and images being written to local storage would only need to be +decompressed again to be stored. Compression can be forced in all cases by +specifying **--disable-compression=false**. + **--disable-content-trust** This is a Docker specific option to disable image verification to a Docker @@ -178,6 +187,10 @@ solely for scripting compatibility. Set custom DNS servers +This option can be used to override the DNS configuration passed to the container. Typically this is necessary when the host DNS configuration is invalid for the container (e.g., 127.0.0.1). When this is the case the `--dns` flag is necessary for every run. + +The special value **none** can be specified to disable creation of /etc/resolv.conf in the container by Podman. The /etc/resolv.conf file in the image will be used without changes. + **--dns-option**=*option* Set custom DNS options @@ -259,6 +272,12 @@ environment variable. `export BUILDAH_LAYERS=true` Log output which would be sent to standard output and standard error to the specified file instead of to standard output and standard error. +**--loglevel** *number* + +Adjust the logging level up or down. Valid option values range from -2 to 3, +with 3 being roughly equivalent to using the global *--debug* option, and +values below 0 omitting even error messages which accompany fatal errors. + **--memory**, **-m**=*LIMIT* Memory limit (format: <number>[<unit>], where unit = b, k, m or g) @@ -301,6 +320,12 @@ that the PID namespace in which `podman` itself is being run should be reused, or it can be the path to a PID namespace which is already in use by another process. +**--platform**="Linux" + +This option has no effect on the build. Other container engines use this option +to control the execution platform for the build (e.g., Windows, Linux) which is +not required for Buildah as it supports only Linux. + **--pull** When the flag is enabled, attempt to pull the latest image from the registries diff --git a/docs/podman-container-runlabel.1.md b/docs/podman-container-runlabel.1.md index 9b74a3410..c16d8c3f4 100644 --- a/docs/podman-container-runlabel.1.md +++ b/docs/podman-container-runlabel.1.md @@ -20,8 +20,6 @@ If the container image has a LABEL INSTALL instruction like the following: If the container image does not have the desired label, an error message will be displayed along with a non-zero return code. If the image is not found in local storage, Podman will attempt to pull it first. -Note: Podman will always ensure that `podman` is the first argument of the command being executed. - **LABEL** The label name specified via the command. @@ -13,11 +13,11 @@ require ( github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc // indirect github.com/containernetworking/cni v0.7.1 github.com/containernetworking/plugins v0.8.1 - github.com/containers/buildah v1.9.2 + github.com/containers/buildah v1.10.1 github.com/containers/conmon v0.3.0 // indirect - github.com/containers/image v2.0.1+incompatible + github.com/containers/image v3.0.2+incompatible github.com/containers/psgo v1.3.1 - github.com/containers/storage v1.12.16 + github.com/containers/storage v1.13.2 github.com/coreos/bbolt v1.3.3 // indirect github.com/coreos/etcd v3.3.13+incompatible // indirect github.com/coreos/go-iptables v0.4.1 @@ -91,7 +91,6 @@ require ( github.com/ugorji/go v1.1.5-pre // indirect github.com/ulikunitz/xz v0.5.6 // indirect github.com/varlink/go v0.0.0-20190502142041-0f1d566d194b - github.com/vbauerster/mpb v3.4.0+incompatible // indirect github.com/vishvananda/netlink v1.0.0 github.com/vishvananda/netns v0.0.0-20190625233234-7109fa855b0f // indirect go.etcd.io/bbolt v1.3.3 // indirect @@ -71,12 +71,16 @@ github.com/containers/buildah v1.9.0 h1:ktVRCGNoVfW8PlTuCKUeh+zGdqn1Nik80DSWvGX+ github.com/containers/buildah v1.9.0/go.mod h1:1CsiLJvyU+h+wOjnqJJOWuJCVcMxZOr5HN/gHGdzJxY= github.com/containers/buildah v1.9.2 h1:dg87r1W1poWVQE0lTmP3BzaqgEI5IRudZ3jKjNIZ3xQ= github.com/containers/buildah v1.9.2/go.mod h1:UFq7EQtnDEEZv42AE7ZbmQMN+mSWSg1JIMwjYW1bn48= +github.com/containers/buildah v1.10.1 h1:YBFHZkpbWCxUR/gjRAZrRzs2E0DfdUe3+/8OA9filWY= +github.com/containers/buildah v1.10.1/go.mod h1:ZTyMFo3IQlu9tYndtnAf0Tjf2NdeUL6bY2/TpP9uIuU= github.com/containers/conmon v0.3.0 h1:NDkYcQAu1BDZSVLh6xrY9jh/WmiDaUloKzRM16237XM= github.com/containers/conmon v0.3.0/go.mod h1:hgwZ2mtuDrppv78a/cOBNiCm6O0UMWGx1mu7P00nu5I= github.com/containers/image v2.0.0+incompatible h1:FTr6Br7jlIKNCKMjSOMbAxKp2keQ0//jzJaYNTVhauk= github.com/containers/image v2.0.0+incompatible/go.mod h1:8Vtij257IWSanUQKe1tAeNOm2sRVkSqQTVQ1IlwI3+M= github.com/containers/image v2.0.1+incompatible h1:w39mlElA/aSFZ6moFa5N+A4MWu9c8hgdMiMMYnH94Hs= github.com/containers/image v2.0.1+incompatible/go.mod h1:8Vtij257IWSanUQKe1tAeNOm2sRVkSqQTVQ1IlwI3+M= +github.com/containers/image v3.0.2+incompatible h1:B1lqAE8MUPCrsBLE86J0gnXleeRq8zJnQryhiiGQNyE= +github.com/containers/image v3.0.2+incompatible/go.mod h1:8Vtij257IWSanUQKe1tAeNOm2sRVkSqQTVQ1IlwI3+M= github.com/containers/psgo v1.3.0 h1:kDhiA4gNNyJ2qCzmOuBf6AmrF/Pp+6Jo98P68R7fB8I= github.com/containers/psgo v1.3.0/go.mod h1:7MELvPTW1fj6yMrwD9I1Iasx1vU+hKlRkHXAJ51sFtU= github.com/containers/psgo v1.3.1-0.20190626112706-fbef66e4ce92 h1:aVJs/Av0Yc9uNoWnIwmG+6Z+XozuRXFwvLwAOVmwlvI= @@ -93,6 +97,10 @@ github.com/containers/storage v1.12.13 h1:GtaLCY8p1Drlk1Oew581jGvB137UaO+kpz0HII github.com/containers/storage v1.12.13/go.mod h1:+RirK6VQAqskQlaTBrOG6ulDvn4si2QjFE1NZCn06MM= github.com/containers/storage v1.12.16 h1:zePYS1GiG8CuRqLCeA0ufx4X27K06HcJLV50DdojL+Y= github.com/containers/storage v1.12.16/go.mod h1:QsZp4XMJjyPNNbQHZeyNW3OmhwsWviI+7S6iOcu6a4c= +github.com/containers/storage v1.13.1 h1:rjVirLS9fCGkUFlLDZEoGDDUugtIf46DufWvJu08wxQ= +github.com/containers/storage v1.13.1/go.mod h1:6D8nK2sU9V7nEmAraINRs88ZEscM5C5DK+8Npp27GeA= +github.com/containers/storage v1.13.2 h1:UXZ0Ckmk6+6+4vj2M2ywruVtH97pnRoAhTG8ctd+yQI= +github.com/containers/storage v1.13.2/go.mod h1:6D8nK2sU9V7nEmAraINRs88ZEscM5C5DK+8Npp27GeA= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/bbolt v1.3.3/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= diff --git a/libpod/define/errors.go b/libpod/define/errors.go index a4368a9aa..9d532263c 100644 --- a/libpod/define/errors.go +++ b/libpod/define/errors.go @@ -107,4 +107,8 @@ var ( // ErrOCIRuntimeNotFound indicates the OCI runtime attempted to invoke a command // that was not found ErrOCIRuntimeNotFound = errors.New("OCI runtime command not found error") + + // ErrConmonOutdated indicates the version of conmon found (whether via the configuration or $PATH) + // is out of date for the current podman version + ErrConmonOutdated = errors.New("outdated conmon version") ) diff --git a/libpod/events/journal_linux.go b/libpod/events/journal_linux.go index ae96e3b3b..7d195dc79 100644 --- a/libpod/events/journal_linux.go +++ b/libpod/events/journal_linux.go @@ -54,14 +54,17 @@ func (e EventJournalD) Read(options ReadOptions) error { if err != nil { return errors.Wrapf(err, "failed to generate event options") } - podmanJournal := sdjournal.Match{Field: "SYSLOG_IDENTIFIER", Value: "podman"} //nolint - j, err := sdjournal.NewJournal() //nolint + j, err := sdjournal.NewJournal() //nolint if err != nil { return err } - if err := j.AddMatch(podmanJournal.String()); err != nil { - return errors.Wrap(err, "failed to add filter for event log") - } + // TODO AddMatch and Seek seem to conflict + // Issue filed upstream -> https://github.com/coreos/go-systemd/issues/315 + // Leaving commented code in case upstream fixes things + //podmanJournal := sdjournal.Match{Field: "SYSLOG_IDENTIFIER", Value: "podman"} //nolint + //if err := j.AddMatch(podmanJournal.String()); err != nil { + // return errors.Wrap(err, "failed to add filter for event log") + //} if len(options.Since) == 0 && len(options.Until) == 0 && options.Stream { if err := j.SeekTail(); err != nil { return errors.Wrap(err, "failed to seek end of journal") @@ -96,6 +99,12 @@ func (e EventJournalD) Read(options ReadOptions) error { if err != nil { return err } + // TODO this keeps us from feeding the podman event parser with + // with regular journal content; it can be removed if the above + // problem with AddMatch is resolved. + if entry.Fields["PODMAN_EVENT"] == "" { + continue + } newEvent, err := newEventFromJournalEntry(entry) if err != nil { // We can't decode this event. diff --git a/libpod/image/docker_registry_options.go b/libpod/image/docker_registry_options.go index c191a3ca2..60bb3c33f 100644 --- a/libpod/image/docker_registry_options.go +++ b/libpod/image/docker_registry_options.go @@ -1,8 +1,12 @@ package image import ( + "fmt" + "github.com/containers/image/docker/reference" "github.com/containers/image/types" + + podmanVersion "github.com/containers/libpod/version" ) // DockerRegistryOptions encapsulates settings that affect how we connect or @@ -36,6 +40,7 @@ func (o DockerRegistryOptions) GetSystemContext(parent *types.SystemContext, add sc.SignaturePolicyPath = parent.SignaturePolicyPath sc.AuthFilePath = parent.AuthFilePath sc.DirForceCompress = parent.DirForceCompress + sc.DockerRegistryUserAgent = parent.DockerRegistryUserAgent } return sc } @@ -48,5 +53,7 @@ func GetSystemContext(signaturePolicyPath, authFilePath string, forceCompress bo } sc.AuthFilePath = authFilePath sc.DirForceCompress = forceCompress + sc.DockerRegistryUserAgent = fmt.Sprintf("libpod/%s", podmanVersion.Version) + return sc } diff --git a/libpod/oci_internal_linux.go b/libpod/oci_internal_linux.go index 52cebefab..e2c73f5ed 100644 --- a/libpod/oci_internal_linux.go +++ b/libpod/oci_internal_linux.go @@ -352,31 +352,29 @@ func startCommandGivenSelinux(cmd *exec.Cmd) error { // it then signals for conmon to start by sending nonse data down the start fd func (r *OCIRuntime) moveConmonToCgroupAndSignal(ctr *Container, cmd *exec.Cmd, startFd *os.File, uuid string) error { cgroupParent := ctr.CgroupParent() - if os.Geteuid() == 0 { - if r.cgroupManager == SystemdCgroupsManager { - unitName := createUnitName("libpod-conmon", ctr.ID()) - - realCgroupParent := cgroupParent - splitParent := strings.Split(cgroupParent, "/") - if strings.HasSuffix(cgroupParent, ".slice") && len(splitParent) > 1 { - realCgroupParent = splitParent[len(splitParent)-1] - } + if r.cgroupManager == SystemdCgroupsManager { + unitName := createUnitName("libpod-conmon", ctr.ID()) - logrus.Infof("Running conmon under slice %s and unitName %s", realCgroupParent, unitName) - if err := utils.RunUnderSystemdScope(cmd.Process.Pid, realCgroupParent, unitName); err != nil { - logrus.Warnf("Failed to add conmon to systemd sandbox cgroup: %v", err) - } + realCgroupParent := cgroupParent + splitParent := strings.Split(cgroupParent, "/") + if strings.HasSuffix(cgroupParent, ".slice") && len(splitParent) > 1 { + realCgroupParent = splitParent[len(splitParent)-1] + } + + logrus.Infof("Running conmon under slice %s and unitName %s", realCgroupParent, unitName) + if err := utils.RunUnderSystemdScope(cmd.Process.Pid, realCgroupParent, unitName); err != nil { + logrus.Warnf("Failed to add conmon to systemd sandbox cgroup: %v", err) + } + } else { + cgroupPath := filepath.Join(ctr.config.CgroupParent, "conmon") + control, err := cgroups.New(cgroupPath, &spec.LinuxResources{}) + if err != nil { + logrus.Warnf("Failed to add conmon to cgroupfs sandbox cgroup: %v", err) } else { - cgroupPath := filepath.Join(ctr.config.CgroupParent, "conmon") - control, err := cgroups.New(cgroupPath, &spec.LinuxResources{}) - if err != nil { + // we need to remove this defer and delete the cgroup once conmon exits + // maybe need a conmon monitor? + if err := control.AddPid(cmd.Process.Pid); err != nil { logrus.Warnf("Failed to add conmon to cgroupfs sandbox cgroup: %v", err) - } else { - // we need to remove this defer and delete the cgroup once conmon exits - // maybe need a conmon monitor? - if err := control.AddPid(cmd.Process.Pid); err != nil { - logrus.Warnf("Failed to add conmon to cgroupfs sandbox cgroup: %v", err) - } } } } diff --git a/libpod/runtime.go b/libpod/runtime.go index 38bfac8ba..8a4eee081 100644 --- a/libpod/runtime.go +++ b/libpod/runtime.go @@ -1,6 +1,7 @@ package libpod import ( + "bytes" "context" "fmt" "io/ioutil" @@ -8,6 +9,8 @@ import ( "os/exec" "os/user" "path/filepath" + "regexp" + "strconv" "strings" "sync" "syscall" @@ -271,6 +274,8 @@ type runtimeConfiguredFrom struct { runtimePath bool cniPluginDir bool noPivotRoot bool + runtimeSupportsJSON bool + ociRuntime bool } func defaultRuntimeConfig() (RuntimeConfig, error) { @@ -590,6 +595,12 @@ func newRuntimeFromConfig(ctx context.Context, userConfigPath string, options .. if tmpConfig.NoPivotRoot { runtime.configuredFrom.noPivotRoot = true } + if tmpConfig.RuntimeSupportsJSON != nil { + runtime.configuredFrom.runtimeSupportsJSON = true + } + if tmpConfig.OCIRuntime != "" { + runtime.configuredFrom.ociRuntime = true + } if _, err := toml.Decode(string(contents), runtime.config); err != nil { return nil, errors.Wrapf(err, "error decoding configuration file %s", configPath) @@ -630,6 +641,13 @@ func newRuntimeFromConfig(ctx context.Context, userConfigPath string, options .. if !runtime.configuredFrom.noPivotRoot { runtime.config.NoPivotRoot = tmpConfig.NoPivotRoot } + if !runtime.configuredFrom.runtimeSupportsJSON { + runtime.config.RuntimeSupportsJSON = tmpConfig.RuntimeSupportsJSON + } + if !runtime.configuredFrom.ociRuntime { + runtime.config.OCIRuntime = tmpConfig.OCIRuntime + } + break } } @@ -739,11 +757,43 @@ func getLockManager(runtime *Runtime) (lock.Manager, error) { return manager, nil } +// probeConmon calls conmon --version and verifies it is a new enough version for +// the runtime expectations podman currently has +func probeConmon(conmonBinary string) error { + cmd := exec.Command(conmonBinary, "--version") + var out bytes.Buffer + cmd.Stdout = &out + err := cmd.Run() + if err != nil { + return err + } + r := regexp.MustCompile(`^conmon version (?P<Major>\d+).(?P<Minor>\d+).(?P<Patch>\d+)`) + + matches := r.FindStringSubmatch(out.String()) + if len(matches) != 4 { + return errors.Wrapf(err, "conmon version changed format") + } + major, err := strconv.Atoi(matches[1]) + if err != nil || major < 1 { + return define.ErrConmonOutdated + } + // conmon used to be shipped with CRI-O, and was versioned along with it. + // even though the conmon that came with crio-1.9 to crio-1.15 has a higher + // version number than conmon 1.0.0, 1.0.0 is newer, so we need this check + minor, err := strconv.Atoi(matches[2]) + if err != nil || minor > 9 { + return define.ErrConmonOutdated + } + + return nil +} + // Make a new runtime based on the given configuration // Sets up containers/storage, state store, OCI runtime func makeRuntime(ctx context.Context, runtime *Runtime) (err error) { // Find a working conmon binary foundConmon := false + foundOutdatedConmon := false for _, path := range runtime.config.ConmonPath { stat, err := os.Stat(path) if err != nil { @@ -752,6 +802,11 @@ func makeRuntime(ctx context.Context, runtime *Runtime) (err error) { if stat.IsDir() { continue } + if err := probeConmon(path); err != nil { + logrus.Warnf("conmon at %s invalid: %v", path, err) + foundOutdatedConmon = true + continue + } foundConmon = true runtime.conmonPath = path logrus.Debugf("using conmon: %q", path) @@ -761,13 +816,21 @@ func makeRuntime(ctx context.Context, runtime *Runtime) (err error) { // Search the $PATH as last fallback if !foundConmon { if conmon, err := exec.LookPath("conmon"); err == nil { - foundConmon = true - runtime.conmonPath = conmon - logrus.Debugf("using conmon from $PATH: %q", conmon) + if err := probeConmon(conmon); err != nil { + logrus.Warnf("conmon at %s is invalid: %v", conmon, err) + foundOutdatedConmon = true + } else { + foundConmon = true + runtime.conmonPath = conmon + logrus.Debugf("using conmon from $PATH: %q", conmon) + } } } if !foundConmon { + if foundOutdatedConmon { + return errors.Wrapf(define.ErrConmonOutdated, "please update to v1.0.0 or later") + } return errors.Wrapf(define.ErrInvalidArg, "could not find a working conmon binary (configured options: %v)", runtime.config.ConmonPath) diff --git a/libpod/runtime_ctr.go b/libpod/runtime_ctr.go index 61a871b28..92b2faefb 100644 --- a/libpod/runtime_ctr.go +++ b/libpod/runtime_ctr.go @@ -54,6 +54,14 @@ func (r *Runtime) RestoreContainer(ctx context.Context, rSpec *spec.Spec, config } // For an imported checkpoint no one has ever set the StartedTime. Set it now. ctr.state.StartedTime = time.Now() + + // If the path to ConmonPidFile starts with the default value (RunRoot), then + // the user has not specified '--conmon-pidfile' during run or create (probably). + // In that case reset ConmonPidFile to be set to the default value later. + if strings.HasPrefix(ctr.config.ConmonPidFile, r.config.StorageConfig.RunRoot) { + ctr.config.ConmonPidFile = "" + } + return r.setupContainer(ctx, ctr) } diff --git a/libpod/runtime_img.go b/libpod/runtime_img.go index 4055734eb..20dee4080 100644 --- a/libpod/runtime_img.go +++ b/libpod/runtime_img.go @@ -28,6 +28,7 @@ import ( // RemoveImage deletes an image from local storage // Images being used by running containers can only be removed if force=true func (r *Runtime) RemoveImage(ctx context.Context, img *image.Image, force bool) (string, error) { + var returnMessage string r.lock.Lock() defer r.lock.Unlock() @@ -93,7 +94,11 @@ func (r *Runtime) RemoveImage(ctx context.Context, img *image.Image, force bool) err = errStorage } } - return img.ID(), err + for _, name := range img.Names() { + returnMessage = returnMessage + fmt.Sprintf("Untagged: %s\n", name) + } + returnMessage = returnMessage + fmt.Sprintf("Deleted: %s", img.ID()) + return returnMessage, err } // Remove containers that are in storage rather than Podman. diff --git a/pkg/adapter/containers.go b/pkg/adapter/containers.go index b712bd9aa..45a9a54a3 100644 --- a/pkg/adapter/containers.go +++ b/pkg/adapter/containers.go @@ -342,7 +342,7 @@ func (r *LocalRuntime) Run(ctx context.Context, c *cliconfig.RunValues, exitCode if err := ctr.Start(ctx, c.IsSet("pod")); err != nil { // This means the command did not exist exitCode = 127 - if strings.Contains(err.Error(), "permission denied") { + if strings.Contains(err.Error(), "permission denied") || strings.Contains(err.Error(), "file not found") { exitCode = 126 } return exitCode, err diff --git a/pkg/adapter/runtime.go b/pkg/adapter/runtime.go index 4a3b41297..03419c0bd 100644 --- a/pkg/adapter/runtime.go +++ b/pkg/adapter/runtime.go @@ -288,9 +288,8 @@ func (r *LocalRuntime) Build(ctx context.Context, c *cliconfig.BuildValues, opti options.CommonBuildOpts = commonOpts options.SystemContext = systemContext - if c.Flag("runtime").Changed { - options.Runtime = r.GetOCIRuntimePath() - } + options.Runtime = r.GetOCIRuntimePath() + if c.Quiet { options.ReportWriter = ioutil.Discard } diff --git a/pkg/adapter/runtime_remote.go b/pkg/adapter/runtime_remote.go index 828838bde..0cafbb2aa 100644 --- a/pkg/adapter/runtime_remote.go +++ b/pkg/adapter/runtime_remote.go @@ -10,6 +10,7 @@ import ( "io" "io/ioutil" "os" + "path/filepath" "strings" "text/template" "time" @@ -68,6 +69,12 @@ func GetRuntime(ctx context.Context, c *cliconfig.PodmanCommand) (*LocalRuntime, cmd: c.GlobalFlags, } configPath := remoteclientconfig.GetConfigFilePath() + // Check if the basedir for configPath exists and if not, create it. + if _, err := os.Stat(filepath.Dir(configPath)); os.IsNotExist(err) { + if mkdirErr := os.MkdirAll(filepath.Dir(configPath), 0750); mkdirErr != nil { + return nil, mkdirErr + } + } if len(c.GlobalFlags.RemoteConfigFilePath) > 0 { configPath = c.GlobalFlags.RemoteConfigFilePath customConfig = true diff --git a/pkg/namespaces/namespaces.go b/pkg/namespaces/namespaces.go index 35298796f..9d1033b93 100644 --- a/pkg/namespaces/namespaces.go +++ b/pkg/namespaces/namespaces.go @@ -4,17 +4,30 @@ import ( "strings" ) +const ( + bridgeType = "bridge" + containerType = "container" + defaultType = "default" + hostType = "host" + noneType = "none" + nsType = "ns" + podType = "pod" + privateType = "private" + shareableType = "shareable" + slirpType = "slirp4netns" +) + // CgroupMode represents cgroup mode in the container. type CgroupMode string // IsHost indicates whether the container uses the host's cgroup. func (n CgroupMode) IsHost() bool { - return n == "host" + return n == hostType } // IsNS indicates a cgroup namespace passed in by path (ns:<path>) func (n CgroupMode) IsNS() bool { - return strings.HasPrefix(string(n), "ns:") + return strings.HasPrefix(string(n), nsType) } // NS gets the path associated with a ns:<path> cgroup ns @@ -29,13 +42,13 @@ func (n CgroupMode) NS() string { // IsContainer indicates whether the container uses a new cgroup namespace. func (n CgroupMode) IsContainer() bool { parts := strings.SplitN(string(n), ":", 2) - return len(parts) > 1 && parts[0] == "container" + return len(parts) > 1 && parts[0] == containerType } // Container returns the name of the container whose cgroup namespace is going to be used. func (n CgroupMode) Container() string { parts := strings.SplitN(string(n), ":", 2) - if len(parts) > 1 { + if len(parts) > 1 && parts[0] == containerType { return parts[1] } return "" @@ -43,15 +56,15 @@ func (n CgroupMode) Container() string { // IsPrivate indicates whether the container uses the a private cgroup. func (n CgroupMode) IsPrivate() bool { - return n == "private" + return n == privateType } // Valid indicates whether the Cgroup namespace is valid. func (n CgroupMode) Valid() bool { parts := strings.Split(string(n), ":") switch mode := parts[0]; mode { - case "", "host", "private", "ns": - case "container": + case "", hostType, privateType, nsType: + case containerType: if len(parts) != 2 || parts[1] == "" { return false } @@ -66,7 +79,7 @@ type UsernsMode string // IsHost indicates whether the container uses the host's userns. func (n UsernsMode) IsHost() bool { - return n == "host" + return n == hostType } // IsKeepID indicates whether container uses a mapping where the (uid, gid) on the host is lept inside of the namespace. @@ -83,8 +96,8 @@ func (n UsernsMode) IsPrivate() bool { func (n UsernsMode) Valid() bool { parts := strings.Split(string(n), ":") switch mode := parts[0]; mode { - case "", "host", "keep-id", "ns": - case "container": + case "", hostType, "keep-id", nsType: + case containerType: if len(parts) != 2 || parts[1] == "" { return false } @@ -111,13 +124,13 @@ func (n UsernsMode) NS() string { // IsContainer indicates whether container uses a container userns. func (n UsernsMode) IsContainer() bool { parts := strings.SplitN(string(n), ":", 2) - return len(parts) > 1 && parts[0] == "container" + return len(parts) > 1 && parts[0] == containerType } // Container is the id of the container which network this container is connected to. func (n UsernsMode) Container() string { parts := strings.SplitN(string(n), ":", 2) - if len(parts) > 1 { + if len(parts) > 1 && parts[0] == containerType { return parts[1] } return "" @@ -133,19 +146,19 @@ func (n UTSMode) IsPrivate() bool { // IsHost indicates whether the container uses the host's UTS namespace. func (n UTSMode) IsHost() bool { - return n == "host" + return n == hostType } // IsContainer indicates whether the container uses a container's UTS namespace. func (n UTSMode) IsContainer() bool { parts := strings.SplitN(string(n), ":", 2) - return len(parts) > 1 && parts[0] == "container" + return len(parts) > 1 && parts[0] == containerType } // Container returns the name of the container whose uts namespace is going to be used. func (n UTSMode) Container() string { parts := strings.SplitN(string(n), ":", 2) - if len(parts) > 1 { + if len(parts) > 1 && parts[0] == containerType { return parts[1] } return "" @@ -155,8 +168,8 @@ func (n UTSMode) Container() string { func (n UTSMode) Valid() bool { parts := strings.Split(string(n), ":") switch mode := parts[0]; mode { - case "", "host": - case "container": + case "", hostType: + case containerType: if len(parts) != 2 || parts[1] == "" { return false } @@ -171,28 +184,28 @@ type IpcMode string // IsPrivate indicates whether the container uses its own private ipc namespace which cannot be shared. func (n IpcMode) IsPrivate() bool { - return n == "private" + return n == privateType } // IsHost indicates whether the container shares the host's ipc namespace. func (n IpcMode) IsHost() bool { - return n == "host" + return n == hostType } // IsShareable indicates whether the container's ipc namespace can be shared with another container. func (n IpcMode) IsShareable() bool { - return n == "shareable" + return n == shareableType } // IsContainer indicates whether the container uses another container's ipc namespace. func (n IpcMode) IsContainer() bool { parts := strings.SplitN(string(n), ":", 2) - return len(parts) > 1 && parts[0] == "container" + return len(parts) > 1 && parts[0] == containerType } // IsNone indicates whether container IpcMode is set to "none". func (n IpcMode) IsNone() bool { - return n == "none" + return n == noneType } // IsEmpty indicates whether container IpcMode is empty @@ -208,7 +221,7 @@ func (n IpcMode) Valid() bool { // Container returns the name of the container ipc stack is going to be used. func (n IpcMode) Container() string { parts := strings.SplitN(string(n), ":", 2) - if len(parts) > 1 && parts[0] == "container" { + if len(parts) > 1 && parts[0] == containerType { return parts[1] } return "" @@ -224,21 +237,21 @@ func (n PidMode) IsPrivate() bool { // IsHost indicates whether the container uses the host's pid namespace. func (n PidMode) IsHost() bool { - return n == "host" + return n == hostType } // IsContainer indicates whether the container uses a container's pid namespace. func (n PidMode) IsContainer() bool { parts := strings.SplitN(string(n), ":", 2) - return len(parts) > 1 && parts[0] == "container" + return len(parts) > 1 && parts[0] == containerType } // Valid indicates whether the pid namespace is valid. func (n PidMode) Valid() bool { parts := strings.Split(string(n), ":") switch mode := parts[0]; mode { - case "", "host": - case "container": + case "", hostType: + case containerType: if len(parts) != 2 || parts[1] == "" { return false } @@ -251,7 +264,7 @@ func (n PidMode) Valid() bool { // Container returns the name of the container whose pid namespace is going to be used. func (n PidMode) Container() string { parts := strings.SplitN(string(n), ":", 2) - if len(parts) > 1 { + if len(parts) > 1 && parts[0] == containerType { return parts[1] } return "" @@ -262,17 +275,17 @@ type NetworkMode string // IsNone indicates whether container isn't using a network stack. func (n NetworkMode) IsNone() bool { - return n == "none" + return n == noneType } // IsHost indicates whether the container uses the host's network stack. func (n NetworkMode) IsHost() bool { - return n == "host" + return n == hostType } // IsDefault indicates whether container uses the default network stack. func (n NetworkMode) IsDefault() bool { - return n == "default" + return n == defaultType } // IsPrivate indicates whether container uses its private network stack. @@ -283,13 +296,13 @@ func (n NetworkMode) IsPrivate() bool { // IsContainer indicates whether container uses a container network stack. func (n NetworkMode) IsContainer() bool { parts := strings.SplitN(string(n), ":", 2) - return len(parts) > 1 && parts[0] == "container" + return len(parts) > 1 && parts[0] == containerType } // Container is the id of the container which network this container is connected to. func (n NetworkMode) Container() string { parts := strings.SplitN(string(n), ":", 2) - if len(parts) > 1 { + if len(parts) > 1 && parts[0] == containerType { return parts[1] } return "" @@ -305,17 +318,17 @@ func (n NetworkMode) UserDefined() string { // IsBridge indicates whether container uses the bridge network stack func (n NetworkMode) IsBridge() bool { - return n == "bridge" + return n == bridgeType } // IsSlirp4netns indicates if we are running a rootless network stack func (n NetworkMode) IsSlirp4netns() bool { - return n == "slirp4netns" + return n == slirpType } // IsNS indicates a network namespace passed in by path (ns:<path>) func (n NetworkMode) IsNS() bool { - return strings.HasPrefix(string(n), "ns:") + return strings.HasPrefix(string(n), nsType) } // NS gets the path associated with a ns:<path> network ns @@ -329,7 +342,7 @@ func (n NetworkMode) NS() string { // IsPod returns whether the network refers to pod networking func (n NetworkMode) IsPod() bool { - return n == "pod" + return n == podType } // IsUserDefined indicates user-created network diff --git a/pkg/rootless/rootless_linux.c b/pkg/rootless/rootless_linux.c index 19b76f387..94933ddd0 100644 --- a/pkg/rootless/rootless_linux.c +++ b/pkg/rootless/rootless_linux.c @@ -137,7 +137,7 @@ get_cmd_line_args (pid_t pid) { allocated += 512; char *tmp = realloc (buffer, allocated); - if (buffer == NULL) + if (tmp == NULL) { free (buffer); return NULL; @@ -276,7 +276,7 @@ static void __attribute__((constructor)) init() return; } - r = TEMP_FAILURE_RETRY (read (fd, buf, sizeof (buf))); + r = TEMP_FAILURE_RETRY (read (fd, buf, sizeof (buf) - 1)); close (fd); if (r < 0) { @@ -457,6 +457,11 @@ create_pause_process (const char *pause_pid_file_path, char **argv) } r = TEMP_FAILURE_RETRY (write (p[1], "0", 1)); + if (r < 0) + { + fprintf (stderr, "cannot write to pipe: %s\n", strerror (errno)); + _exit (EXIT_FAILURE); + } close (p[1]); _exit (EXIT_SUCCESS); @@ -811,6 +816,11 @@ reexec_in_user_namespace (int ready, char *pause_pid_file_path, char *file_to_re } ret = TEMP_FAILURE_RETRY (write (ready, "0", 1)); + if (ret < 0) + { + fprintf (stderr, "cannot write to ready pipe: %s\n", strerror (errno)); + _exit (EXIT_FAILURE); + } close (ready); if (sigprocmask (SIG_SETMASK, &oldsigset, NULL) < 0) diff --git a/pkg/spec/createconfig.go b/pkg/spec/createconfig.go index f21ae2831..3f70e5935 100644 --- a/pkg/spec/createconfig.go +++ b/pkg/spec/createconfig.go @@ -64,8 +64,9 @@ type CreateConfig struct { CidFile string ConmonPidFile string Cgroupns string - CgroupParent string // cgroup-parent - Command []string + CgroupParent string // cgroup-parent + Command []string // Full command that will be used + UserCommand []string // User-entered command (or image CMD) Detach bool // detach Devices []string // device DNSOpt []string //dns-opt @@ -230,8 +231,8 @@ func (c *CreateConfig) getContainerCreateOptions(runtime *libpod.Runtime, pod *l options = append(options, libpod.WithNamedVolumes(namedVolumes)) } - if len(c.Command) != 0 { - options = append(options, libpod.WithCommand(c.Command)) + if len(c.UserCommand) != 0 { + options = append(options, libpod.WithCommand(c.UserCommand)) } // Add entrypoint unconditionally diff --git a/pkg/spec/spec.go b/pkg/spec/spec.go index c94746767..156d6849d 100644 --- a/pkg/spec/spec.go +++ b/pkg/spec/spec.go @@ -174,10 +174,20 @@ func (config *CreateConfig) createConfigToOCISpec(runtime *libpod.Runtime, userM } hostname := config.Hostname - if hostname == "" && (config.NetMode.IsHost() || config.UtsMode.IsHost()) { - hostname, err = os.Hostname() - if err != nil { - return nil, errors.Wrap(err, "unable to retrieve hostname") + if hostname == "" { + if utsCtrID := config.UtsMode.Container(); utsCtrID != "" { + utsCtr, err := runtime.GetContainer(utsCtrID) + if err != nil { + return nil, errors.Wrapf(err, "unable to retrieve hostname from dependency container %s", utsCtrID) + } + hostname = utsCtr.Hostname() + } else if config.NetMode.IsHost() || config.UtsMode.IsHost() { + hostname, err = os.Hostname() + if err != nil { + return nil, errors.Wrap(err, "unable to retrieve hostname of the host") + } + } else { + logrus.Debug("No hostname set; container's hostname will default to runtime default") } } g.RemoveHostname() @@ -541,8 +551,8 @@ func addPidNS(config *CreateConfig, g *generate.Generator) error { if pidMode.IsHost() { return g.RemoveLinuxNamespace(string(spec.PIDNamespace)) } - if pidMode.IsContainer() { - logrus.Debug("using container pidmode") + if pidCtr := pidMode.Container(); pidCtr != "" { + logrus.Debugf("using container %s pidmode", pidCtr) } if IsPod(string(pidMode)) { logrus.Debug("using pod pidmode") @@ -579,8 +589,8 @@ func addNetNS(config *CreateConfig, g *generate.Generator) error { } else if netMode.IsBridge() { logrus.Debug("Using bridge netmode") return nil - } else if netMode.IsContainer() { - logrus.Debug("Using container netmode") + } else if netCtr := netMode.Container(); netCtr != "" { + logrus.Debugf("using container %s netmode", netCtr) return nil } else if IsNS(string(netMode)) { logrus.Debug("Using ns netmode") @@ -606,6 +616,9 @@ func addUTSNS(config *CreateConfig, g *generate.Generator) error { if utsMode.IsHost() { return g.RemoveLinuxNamespace(string(spec.UTSNamespace)) } + if utsCtr := utsMode.Container(); utsCtr != "" { + logrus.Debugf("using container %s utsmode", utsCtr) + } return nil } @@ -617,8 +630,8 @@ func addIpcNS(config *CreateConfig, g *generate.Generator) error { if ipcMode.IsHost() { return g.RemoveLinuxNamespace(string(spec.IPCNamespace)) } - if ipcMode.IsContainer() { - logrus.Debug("Using container ipcmode") + if ipcCtr := ipcMode.Container(); ipcCtr != "" { + logrus.Debugf("Using container %s ipcmode", ipcCtr) } return nil @@ -635,8 +648,8 @@ func addCgroupNS(config *CreateConfig, g *generate.Generator) error { if cgroupMode.IsPrivate() { return g.AddOrReplaceLinuxNamespace(string(spec.CgroupNamespace), "") } - if cgroupMode.IsContainer() { - logrus.Debug("Using container cgroup mode") + if cgCtr := cgroupMode.Container(); cgCtr != "" { + logrus.Debugf("Using container %s cgroup mode", cgCtr) } return nil } diff --git a/pkg/spec/storage.go b/pkg/spec/storage.go index ac7a2c30f..a8dc7f4a8 100644 --- a/pkg/spec/storage.go +++ b/pkg/spec/storage.go @@ -238,11 +238,6 @@ func (config *CreateConfig) parseVolumes(runtime *libpod.Runtime) ([]spec.Mount, // Conflicts are resolved simply - the last container specified wins. // Container names may be suffixed by mount options after a colon. func (config *CreateConfig) getVolumesFrom(runtime *libpod.Runtime) (map[string]spec.Mount, map[string]*libpod.ContainerNamedVolume, error) { - // TODO: This can probably be disabled now - if os.Geteuid() != 0 { - return nil, nil, nil - } - // Both of these are maps of mount destination to mount type. // We ensure that each destination is only mounted to once in this way. finalMounts := make(map[string]spec.Mount) @@ -410,13 +405,42 @@ func getBindMount(args []string) (spec.Mount, error) { setSource := false setDest := false + setRORW := false for _, val := range args { kv := strings.Split(val, "=") switch kv[0] { case "bind-nonrecursive": newMount.Options = append(newMount.Options, "bind") - case "ro", "nosuid", "nodev", "noexec": + case "ro", "rw": + if setRORW { + return newMount, errors.Wrapf(optionArgError, "cannot pass 'ro' or 'rw' options more than once") + } + setRORW = true + // Can be formatted as one of: + // ro + // ro=[true|false] + // rw + // rw=[true|false] + if len(kv) == 1 { + newMount.Options = append(newMount.Options, kv[0]) + } else if len(kv) == 2 { + switch strings.ToLower(kv[1]) { + case "true": + newMount.Options = append(newMount.Options, kv[0]) + case "false": + // Set the opposite only for rw + // ro's opposite is the default + if kv[0] == "rw" { + newMount.Options = append(newMount.Options, "ro") + } + default: + return newMount, errors.Wrapf(optionArgError, "%s must be set to true or false, instead received %q", kv[0], kv[1]) + } + } else { + return newMount, errors.Wrapf(optionArgError, "badly formatted option %q", val) + } + case "nosuid", "nodev", "noexec": // TODO: detect duplication of these options. // (Is this necessary?) newMount.Options = append(newMount.Options, kv[0]) diff --git a/pkg/util/utils_windows.go b/pkg/util/utils_windows.go index e7b2a272e..e781e6717 100644 --- a/pkg/util/utils_windows.go +++ b/pkg/util/utils_windows.go @@ -6,29 +6,31 @@ import ( "github.com/pkg/errors" ) +var errNotImplemented = errors.New("not yet implemented") + // IsCgroup2UnifiedMode returns whether we are running in cgroup 2 unified mode. func IsCgroup2UnifiedMode() (bool, error) { - return false, errors.New("this function is not implemented for windows") + return false, errors.Wrap(errNotImplemented, "IsCgroup2Unified") } // GetContainerPidInformationDescriptors returns a string slice of all supported // format descriptors of GetContainerPidInformation. func GetContainerPidInformationDescriptors() ([]string, error) { - return nil, errors.New("this function is not implemented for windows") + return nil, errors.Wrap(errNotImplemented, "GetContainerPidInformationDescriptors") } // GetRootlessPauseProcessPidPath returns the path to the file that holds the pid for // the pause process func GetRootlessPauseProcessPidPath() (string, error) { - return "", errors.New("this function is not implemented for windows") + return "", errors.Wrap(errNotImplemented, "GetRootlessPauseProcessPidPath") } // GetRootlessRuntimeDir returns the runtime directory when running as non root func GetRootlessRuntimeDir() (string, error) { - return "", errors.New("this function is not implemented for windows") + return "", errors.Wrap(errNotImplemented, "GetRootlessRuntimeDir") } // GetRootlessConfigHomeDir returns the config home directory when running as non root func GetRootlessConfigHomeDir() (string, error) { - return "", errors.New("this function is not implemented for windows") + return "", errors.Wrap(errNotImplemented, "GetRootlessConfigHomeDir") } diff --git a/test/e2e/checkpoint_test.go b/test/e2e/checkpoint_test.go index d37d7c7cc..1caefd299 100644 --- a/test/e2e/checkpoint_test.go +++ b/test/e2e/checkpoint_test.go @@ -375,23 +375,28 @@ var _ = Describe("Podman checkpoint", func() { result := podmanTest.Podman([]string{"container", "checkpoint", "-l", "-e", fileName}) result.WaitWithDefaultTimeout() + // As the container has been started with '--rm' it will be completely + // cleaned up after checkpointing. Expect(result.ExitCode()).To(Equal(0)) Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0)) Expect(podmanTest.NumberOfContainers()).To(Equal(0)) - result = podmanTest.Podman([]string{"container", "restore", "-i", fileName}) + // Restore container the first time with different name. + // Using '--ignore-static-ip' as for parallel test runs + // each containers gets a random IP address via '--ip'. + // '--ignore-static-ip' tells the restore to use the next + // available IP address. + // First restore the container with a new name/ID to make + // sure nothing in the restored container depends on the + // original container. + result = podmanTest.Podman([]string{"container", "restore", "-i", fileName, "-n", "restore_again", "--ignore-static-ip"}) result.WaitWithDefaultTimeout() Expect(result.ExitCode()).To(Equal(0)) Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1)) Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Up")) - // Restore container a second time with different name. - // Using '--ignore-static-ip' as for parallel test runs - // each containers gets a random IP address via '--ip'. - // '--ignore-static-ip' tells the restore to use the next - // available IP address. - result = podmanTest.Podman([]string{"container", "restore", "-i", fileName, "-n", "restore_again", "--ignore-static-ip"}) + result = podmanTest.Podman([]string{"container", "restore", "-i", fileName}) result.WaitWithDefaultTimeout() Expect(result.ExitCode()).To(Equal(0)) diff --git a/test/e2e/cp_test.go b/test/e2e/cp_test.go index 5e98e73eb..edd9c70c6 100644 --- a/test/e2e/cp_test.go +++ b/test/e2e/cp_test.go @@ -209,4 +209,40 @@ var _ = Describe("Podman cp", func() { session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) }) + + It("podman cp from ctr chown ", func() { + setup := podmanTest.RunTopContainer("testctr") + setup.WaitWithDefaultTimeout() + Expect(setup.ExitCode()).To(Equal(0)) + + session := podmanTest.Podman([]string{"exec", "testctr", "adduser", "-S", "testuser"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + + session = podmanTest.Podman([]string{"exec", "-u", "testuser", "testctr", "touch", "testfile"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + + session = podmanTest.Podman([]string{"cp", "testctr:testfile", "testfile1"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + + // owner of the file copied to local machine is not testuser + cmd := exec.Command("ls", "-l", "testfile1") + cmdRet, err := cmd.Output() + Expect(err).To(BeNil()) + Expect(strings.Contains(string(cmdRet), "testuser")).To(BeFalse()) + + session = podmanTest.Podman([]string{"cp", "testfile1", "testctr:testfile2"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + + // owner of the file copied to a container is the root user + session = podmanTest.Podman([]string{"exec", "-it", "testctr", "ls", "-l", "testfile2"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + Expect(session.OutputToString()).To(ContainSubstring("root")) + + os.Remove("testfile1") + }) }) diff --git a/test/e2e/create_test.go b/test/e2e/create_test.go index e2b4a7cf4..25d0c3390 100644 --- a/test/e2e/create_test.go +++ b/test/e2e/create_test.go @@ -218,4 +218,17 @@ var _ = Describe("Podman create", func() { match, _ := check.GrepString("foobar") Expect(match).To(BeTrue()) }) + + It("podman run entrypoint and cmd test", func() { + name := "test101" + create := podmanTest.Podman([]string{"create", "--name", name, redis}) + create.WaitWithDefaultTimeout() + Expect(create.ExitCode()).To(Equal(0)) + + ctrJSON := podmanTest.InspectContainer(name) + Expect(len(ctrJSON)).To(Equal(1)) + Expect(len(ctrJSON[0].Config.Cmd)).To(Equal(1)) + Expect(ctrJSON[0].Config.Cmd[0]).To(Equal("redis-server")) + Expect(ctrJSON[0].Config.Entrypoint).To(Equal("docker-entrypoint.sh")) + }) }) diff --git a/test/e2e/exec_test.go b/test/e2e/exec_test.go index 6cf78a25c..3f9639fda 100644 --- a/test/e2e/exec_test.go +++ b/test/e2e/exec_test.go @@ -179,6 +179,8 @@ var _ = Describe("Podman exec", func() { }) It("podman exec cannot be invoked", func() { + SkipIfNotRunc() + setup := podmanTest.RunTopContainer("test1") setup.WaitWithDefaultTimeout() Expect(setup.ExitCode()).To(Equal(0)) @@ -189,6 +191,8 @@ var _ = Describe("Podman exec", func() { }) It("podman exec command not found", func() { + SkipIfNotRunc() + setup := podmanTest.RunTopContainer("test1") setup.WaitWithDefaultTimeout() Expect(setup.ExitCode()).To(Equal(0)) diff --git a/test/e2e/libpod_suite_remoteclient_test.go b/test/e2e/libpod_suite_remoteclient_test.go index 7f33fec87..a6cedfc58 100644 --- a/test/e2e/libpod_suite_remoteclient_test.go +++ b/test/e2e/libpod_suite_remoteclient_test.go @@ -28,6 +28,13 @@ func SkipIfRootless() { } } +func SkipIfNotRunc() { + runtime := os.Getenv("OCI_RUNTIME") + if runtime != "" && filepath.Base(runtime) != "runc" { + ginkgo.Skip("Not using runc as runtime") + } +} + // Podman is the exec call to podman on the filesystem func (p *PodmanTestIntegration) Podman(args []string) *PodmanSessionIntegration { podmanSession := p.PodmanBase(args, false, false) diff --git a/test/e2e/libpod_suite_test.go b/test/e2e/libpod_suite_test.go index 1df59dbe3..22cc14d6b 100644 --- a/test/e2e/libpod_suite_test.go +++ b/test/e2e/libpod_suite_test.go @@ -21,6 +21,13 @@ func SkipIfRootless() { } } +func SkipIfNotRunc() { + runtime := os.Getenv("OCI_RUNTIME") + if runtime != "" && filepath.Base(runtime) != "runc" { + ginkgo.Skip("Not using runc as runtime") + } +} + // Podman is the exec call to podman on the filesystem func (p *PodmanTestIntegration) Podman(args []string) *PodmanSessionIntegration { podmanSession := p.PodmanBase(args, false, false) diff --git a/test/e2e/login_logout_test.go b/test/e2e/login_logout_test.go index d64340248..4d476e05f 100644 --- a/test/e2e/login_logout_test.go +++ b/test/e2e/login_logout_test.go @@ -127,6 +127,10 @@ var _ = Describe("Podman login and logout", func() { session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) + session = podmanTest.Podman([]string{"run", "--authfile", authFile, testImg}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + session = podmanTest.Podman([]string{"logout", "--authfile", authFile, server}) }) diff --git a/test/e2e/pod_infra_container_test.go b/test/e2e/pod_infra_container_test.go index c8763de9f..3897aa851 100644 --- a/test/e2e/pod_infra_container_test.go +++ b/test/e2e/pod_infra_container_test.go @@ -383,4 +383,24 @@ var _ = Describe("Podman pod create", func() { session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) }) + + It("podman run hostname is shared", func() { + session := podmanTest.Podman([]string{"pod", "create"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + podID := session.OutputToString() + + // verify we can add a host to the infra's /etc/hosts + session = podmanTest.Podman([]string{"run", "--pod", podID, ALPINE, "hostname"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + hostname := session.OutputToString() + + infraName := podID[:12] + "-infra" + // verify we can see the other hosts of infra's /etc/hosts + session = podmanTest.Podman([]string{"inspect", infraName}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + Expect(session.OutputToString()).To(ContainSubstring(hostname)) + }) }) diff --git a/test/e2e/port_test.go b/test/e2e/port_test.go index 26c5fd7d0..b15d8e133 100644 --- a/test/e2e/port_test.go +++ b/test/e2e/port_test.go @@ -105,4 +105,42 @@ var _ = Describe("Podman port", func() { result.WaitWithDefaultTimeout() Expect(result.ExitCode()).To(Equal(0)) }) + + It("podman port nginx by name", func() { + session, cid := podmanTest.RunNginxWithHealthCheck("portcheck") + Expect(session.ExitCode()).To(Equal(0)) + + if err := podmanTest.RunHealthCheck(cid); err != nil { + Fail(err.Error()) + } + + result := podmanTest.Podman([]string{"port", "portcheck"}) + result.WaitWithDefaultTimeout() + Expect(result.ExitCode()).To(Equal(0)) + result.LineInOuputStartsWith("80/tcp -> 0.0.0.0:") + }) + + It("podman port multiple ports", func() { + // Acquire and release locks + lock1 := GetPortLock("5000") + defer lock1.Unlock() + lock2 := GetPortLock("5001") + defer lock2.Unlock() + + setup := podmanTest.Podman([]string{"run", "-dt", "-p", "5000:5000", "-p", "5001:5001", ALPINE, "top"}) + setup.WaitWithDefaultTimeout() + Expect(setup.ExitCode()).To(BeZero()) + + // Check that the first port was honored + result1 := podmanTest.Podman([]string{"port", "-l", "5000"}) + result1.WaitWithDefaultTimeout() + Expect(result1.ExitCode()).To(BeZero()) + Expect(result1.LineInOuputStartsWith("0.0.0.0:5000")) + + // Check that the second port was honored + result2 := podmanTest.Podman([]string{"port", "-l", "5001"}) + result2.WaitWithDefaultTimeout() + Expect(result2.ExitCode()).To(BeZero()) + Expect(result2.LineInOuputStartsWith("0.0.0.0:5001")) + }) }) diff --git a/test/e2e/run_dns_test.go b/test/e2e/run_dns_test.go index 081fab3fd..dc0f4a8fb 100644 --- a/test/e2e/run_dns_test.go +++ b/test/e2e/run_dns_test.go @@ -41,6 +41,13 @@ var _ = Describe("Podman run dns", func() { session.LineInOuputStartsWith("search foobar.com") }) + It("podman run remove all search domain", func() { + session := podmanTest.Podman([]string{"run", "--dns-search=.", ALPINE, "cat", "/etc/resolv.conf"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + Expect(session.LineInOuputStartsWith("search")).To(BeFalse()) + }) + It("podman run add bad dns server", func() { session := podmanTest.Podman([]string{"run", "--dns=foobar", ALPINE, "ls"}) session.WaitWithDefaultTimeout() diff --git a/test/e2e/run_exit_test.go b/test/e2e/run_exit_test.go index 861d6b3b7..b05849ddb 100644 --- a/test/e2e/run_exit_test.go +++ b/test/e2e/run_exit_test.go @@ -41,12 +41,16 @@ var _ = Describe("Podman run exit", func() { }) It("podman run exit 126", func() { + SkipIfNotRunc() + result := podmanTest.Podman([]string{"run", ALPINE, "/etc"}) result.WaitWithDefaultTimeout() Expect(result.ExitCode()).To(Equal(126)) }) It("podman run exit 127", func() { + SkipIfNotRunc() + result := podmanTest.Podman([]string{"run", ALPINE, "foobar"}) result.WaitWithDefaultTimeout() Expect(result.ExitCode()).To(Equal(127)) diff --git a/test/e2e/run_test.go b/test/e2e/run_test.go index f66d1d2fa..1420a8403 100644 --- a/test/e2e/run_test.go +++ b/test/e2e/run_test.go @@ -353,6 +353,8 @@ var _ = Describe("Podman run", func() { It("podman run notify_socket", func() { SkipIfRemote() + SkipIfNotRunc() + host := GetHostDistributionInfo() if host.Distribution != "rhel" && host.Distribution != "centos" && host.Distribution != "fedora" { Skip("this test requires a working runc") @@ -563,6 +565,7 @@ var _ = Describe("Podman run", func() { }) It("podman run exit code on failure to exec", func() { + SkipIfNotRunc() session := podmanTest.Podman([]string{"run", ALPINE, "/etc"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(126)) diff --git a/test/e2e/run_volume_test.go b/test/e2e/run_volume_test.go index 9e160e73c..1e0b84310 100644 --- a/test/e2e/run_volume_test.go +++ b/test/e2e/run_volume_test.go @@ -136,4 +136,22 @@ var _ = Describe("Podman run with volumes", func() { session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) }) + + It("podman run with mount flag and boolean options", func() { + mountPath := filepath.Join(podmanTest.TempDir, "secrets") + os.Mkdir(mountPath, 0755) + session := podmanTest.Podman([]string{"run", "--rm", "--mount", fmt.Sprintf("type=bind,src=%s,target=/run/test,ro=false", mountPath), ALPINE, "grep", "/run/test", "/proc/self/mountinfo"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + Expect(session.OutputToString()).To(ContainSubstring("/run/test rw")) + + session = podmanTest.Podman([]string{"run", "--rm", "--mount", fmt.Sprintf("type=bind,src=%s,target=/run/test,ro=true", mountPath), ALPINE, "grep", "/run/test", "/proc/self/mountinfo"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + Expect(session.OutputToString()).To(ContainSubstring("/run/test ro")) + + session = podmanTest.Podman([]string{"run", "--rm", "--mount", fmt.Sprintf("type=bind,src=%s,target=/run/test,ro=true,rw=false", mountPath), ALPINE, "grep", "/run/test", "/proc/self/mountinfo"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Not(Equal(0))) + }) }) diff --git a/test/e2e/start_test.go b/test/e2e/start_test.go index fc1203ed1..2dbb9545b 100644 --- a/test/e2e/start_test.go +++ b/test/e2e/start_test.go @@ -101,6 +101,8 @@ var _ = Describe("Podman start", func() { }) It("podman failed to start with --rm should delete the container", func() { + SkipIfNotRunc() + session := podmanTest.Podman([]string{"create", "-it", "--rm", ALPINE, "foo"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) @@ -114,6 +116,8 @@ var _ = Describe("Podman start", func() { }) It("podman failed to start without --rm should NOT delete the container", func() { + SkipIfNotRunc() + session := podmanTest.Podman([]string{"create", "-it", ALPINE, "foo"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) diff --git a/test/system/001-basic.bats b/test/system/001-basic.bats index 85b9bc1ca..5fc07acfb 100644 --- a/test/system/001-basic.bats +++ b/test/system/001-basic.bats @@ -13,6 +13,14 @@ function setup() { @test "podman version emits reasonable output" { run_podman version + # First line of podman-remote is "Client:<blank>". + # Just delete it (i.e. remove the first entry from the 'lines' array) + if is_remote; then + if expr "${lines[0]}" : "Client:" >/dev/null; then + lines=("${lines[@]:1}") + fi + fi + is "${lines[0]}" "Version:[ ]\+[1-9][0-9.]\+" "Version line 1" is "$output" ".*Go Version: \+" "'Go Version' in output" is "$output" ".*RemoteAPI Version: \+" "API version in output" diff --git a/test/system/070-build.bats b/test/system/070-build.bats index c1e7c7ec4..5ef84e9b8 100644 --- a/test/system/070-build.bats +++ b/test/system/070-build.bats @@ -6,10 +6,8 @@ load helpers @test "podman build - basic test" { - if [[ "$PODMAN" =~ -remote ]]; then - if [ "$(id -u)" -ne 0 ]; then - skip "unreliable with podman-remote and rootless; #2972" - fi + if is_remote && is_rootless; then + skip "unreliable with podman-remote and rootless; #2972" fi rand_filename=$(random_string 20) diff --git a/test/system/200-pod-top.bats b/test/system/200-pod-top.bats index 10808ddb2..bba1e8d14 100644 --- a/test/system/200-pod-top.bats +++ b/test/system/200-pod-top.bats @@ -3,6 +3,8 @@ load helpers @test "podman pod top - containers in different PID namespaces" { + skip_if_remote "podman-pod does not work with podman-remote" + # With infra=false, we don't get a /pause container (we also # don't pull k8s.gcr.io/pause ) no_infra='--infra=false' diff --git a/test/system/README.md b/test/system/README.md index d98b1c0fe..fe6d1ed52 100644 --- a/test/system/README.md +++ b/test/system/README.md @@ -28,6 +28,8 @@ on failure. * `skip_if_rootless` - if rootless, skip this test with a helpful message. +* `skip_if_remote` - like the above, but skip if testing `podman-remote` + * `random_string` - returns a pseudorandom alphanumeric string Test files are of the form `NNN-name.bats` where NNN is a three-digit diff --git a/test/system/helpers.bash b/test/system/helpers.bash index fe0a25b37..3d607f4bd 100644 --- a/test/system/helpers.bash +++ b/test/system/helpers.bash @@ -216,26 +216,31 @@ function wait_for_ready { ############################################################################### # BEGIN miscellaneous tools +# Shortcuts for common needs: +function is_rootless() { + [ "$(id -u)" -ne 0 ] +} + +function is_remote() { + [[ "$PODMAN" =~ -remote ]] +} + ###################### # skip_if_rootless # ...with an optional message ###################### function skip_if_rootless() { - if [ "$(id -u)" -eq 0 ]; then - return + if is_rootless; then + skip "${1:-not applicable under rootless podman}" fi - - skip "${1:-not applicable under rootless podman}" } #################### # skip_if_remote # ...with an optional message #################### function skip_if_remote() { - if [[ ! "$PODMAN" =~ -remote ]]; then - return + if is_remote; then + skip "${1:-test does not work with podman-remote}" fi - - skip "${1:-test does not work with podman-remote}" } ######################### diff --git a/troubleshooting.md b/troubleshooting.md index 798d9cec4..b88940dc8 100644 --- a/troubleshooting.md +++ b/troubleshooting.md @@ -298,7 +298,33 @@ tells SELinux to apply the labels to the actual content. Now all new content created in these directories will automatically be created with the correct label. -### 12) Running Podman inside a container causes container crashes and inconsistent states +### 12) Anonymous image pull fails with 'invalid username/password' + +Pulling an anonymous image that doesn't require authentication can result in an +`invalid username/password` error. + +#### Symptom + +If you pull an anonymous image, one that should not require credentials, you can receive +and `invalid username/password` error if you have credentials established in the +authentication file for the target container registry that are no longer valid. + +``` +podman run -it --rm docker://docker.io/library/alpine:latest ls +Trying to pull docker://docker.io/library/alpine:latest...ERRO[0000] Error pulling image ref //alpine:latest: Error determining manifest MIME type for docker://alpine:latest: unable to retrieve auth token: invalid username/password +Failed +Error: unable to pull docker://docker.io/library/alpine:latest: unable to pull image: Error determining manifest MIME type for docker://alpine:latest: unable to retrieve auth token: invalid username/password +``` + +This can happen if the authentication file is modified 'by hand' or if the credentials +are established locally and then the password is updated later in the container registry. + +#### Solution + +Depending upon which container tool was used to establish the credentials, use `podman logout` +or `docker logout` to remove the credentials from the authentication file. + +### 13) Running Podman inside a container causes container crashes and inconsistent states Running Podman in a container and forwarding some, but not all, of the required host directories can cause inconsistent container behavior. @@ -316,7 +342,7 @@ This can cause Podman to reset container states and lose track of running contai For running containers on the host from inside a container, we also recommend the [Podman remote client](remote_client.md), which only requires a single socket to be mounted into the container. -### 13) Rootless 'podman build' fails EPERM on NFS: +### 14) Rootless 'podman build' fails EPERM on NFS: NFS enforces file creation on different UIDs on the server side and does not understand user namespace, which rootless Podman requires. When a container root process like YUM attempts to create a file owned by a different UID, NFS Server denies the creation. @@ -336,7 +362,7 @@ Choose one of the following: * Edit `~/.config/containers/libpod.conf` and point the `volume_path` option to that local directory. * Otherwise just run podman as root, via `sudo podman` -### 14) Rootless 'podman build' fails when using OverlayFS: +### 15) Rootless 'podman build' fails when using OverlayFS: The Overlay file system (OverlayFS) requires the ability to call the `mknod` command when creating whiteout files when extracting an image. However, a rootless user does not have the privileges to use `mknod` in this capacity. diff --git a/vendor/github.com/containers/buildah/.golangci.yml b/vendor/github.com/containers/buildah/.golangci.yml index 9a34d03e6..52e9990ed 100644 --- a/vendor/github.com/containers/buildah/.golangci.yml +++ b/vendor/github.com/containers/buildah/.golangci.yml @@ -17,6 +17,7 @@ linters: - errcheck - gofmt - goimports + - golint - gosimple - govet - ineffassign @@ -35,7 +36,6 @@ linters: # - goconst # - gocritic # - gocyclo - # - golint # - gosec # - interfacer # - lll diff --git a/vendor/github.com/containers/buildah/CHANGELOG.md b/vendor/github.com/containers/buildah/CHANGELOG.md index 1c59a67ad..ddf6fb1d4 100644 --- a/vendor/github.com/containers/buildah/CHANGELOG.md +++ b/vendor/github.com/containers/buildah/CHANGELOG.md @@ -2,6 +2,111 @@ # Changelog +## v1.10.0 (2019-08-02) + vendor github.com/containers/image@v3.0.0 + Remove GO111MODULE in favor of `-mod=vendor` + Vendor in containers/storage v1.12.16 + Add '-' minus syntax for removal of config values + tests: enable overlay tests for rootless + rootless, overlay: use fuse-overlayfs + vendor github.com/containers/image@v2.0.1 + Added '-' syntax to remove volume config option + delete `successfully pushed` message + Add golint linter and apply fixes + vendor github.com/containers/storage@v1.12.15 + Change wait to sleep in buildahimage readme + Handle ReadOnly images when deleting images + Add support for listing read/only images + +## v1.9.2 (2019-07-19) + from/import: record the base image's digest, if it has one + Fix CNI version retrieval to not require network connection + Add misspell linter and apply fixes + Add goimports linter and apply fixes + Add stylecheck linter and apply fixes + Add unconvert linter and apply fixes + image: make sure we don't try to use zstd compression + run.bats: skip the "z" flag when testing --mount + Update to runc v1.0.0-rc8 + Update to match updated runtime-tools API + bump github.com/opencontainers/runtime-tools to v0.9.0 + Build e2e tests using the proper build tags + Add unparam linter and apply fixes + Run: correct a typo in the --cap-add help text + unshare: add a --mount flag + fix push check image name is not empty + Bump to v1.9.2-dev + +## v1.9.1 (2019-07-12) + add: fix slow copy with no excludes + Add errcheck linter and fix missing error check + Improve tests/tools/Makefile parallelism and abstraction + Fix response body not closed resource leak + Switch to golangci-lint + Add gomod instructions and mailing list links + On Masked path, check if /dev/null already mounted before mounting + Update to containers/storage v1.12.13 + Refactor code in package imagebuildah + Add rootless podman with NFS issue in documentation + Add --mount for buildah run + import method ValidateVolumeOpts from libpod + Fix typo + Makefile: set GO111MODULE=off + rootless: add the built-in slirp DNS server + Update docker/libnetwork to get rid of outdated sctp package + Update buildah-login.md + migrate to go modules + install.md: mention go modules + tests/tools: go module for test binaries + fix --volume splits comma delimited option + Add bud test for RUN with a priv'd command + vendor logrus v1.4.2 + pkg/cli: panic when flags can't be hidden + pkg/unshare: check all errors + pull: check error during report write + run_linux.go: ignore unchecked errors + conformance test: catch copy error + chroot/run_test.go: export funcs to actually be executed + tests/imgtype: ignore error when shutting down the store + testreport: check json error + bind/util.go: remove unused func + rm chroot/util.go + imagebuildah: remove unused `dedupeStringSlice` + StageExecutor: EnsureContainerPath: catch error from SecureJoin() + imagebuildah/build.go: return <expr> instead of branching + rmi: avoid redundant branching + conformance tests: nilness: allocate map + imagebuildah/build.go: avoid redundant `filepath.Join()` + imagebuildah/build.go: avoid redundant `os.Stat()` + imagebuildah: omit comparison to bool + fix "ineffectual assignment" lint errors + docker: ignore "repeats json tag" lint error + pkg/unshare: use `...` instead of iterating a slice + conformance: bud test: use raw strings for regexes + conformance suite: remove unused func/var + buildah test suite: remove unused vars/funcs + testreport: fix golangci-lint errors + util: remove redundant `return` statement + chroot: only log clean-up errors + images_test: ignore golangci-lint error + blobcache: log error when draining the pipe + imagebuildah: check errors in deferred calls + chroot: fix error handling in deferred funcs + cmd: check all errors + chroot/run_test.go: check errors + chroot/run.go: check errors in deferred calls + imagebuildah.Executor: remove unused onbuild field + docker/types.go: remove unused struct fields + util: use strings.ContainsRune instead of index check + Cirrus: Initial implementation + Bump to v1.9.1-dev + +## v1.9.0 (2019-06-15) + buildah-run: fix-out-of-range panic (2) + Bump back to v1.9.0-dev + + + ## v1.8.4 (2019-06-13) Update containers/image to v2.0.0 run: fix hang with run and --isolation=chroot @@ -32,49 +137,49 @@ Cleanup Overlay Mounts content ## v1.8.3 (2019-06-04) - * Add support for file secret mounts - * Add ability to skip secrets in mounts file - * allow 32bit builds - * fix tutorial instructions - * imagebuilder: pass the right contextDir to Add() - * add: use fileutils.PatternMatcher for .dockerignore - * bud.bats: add another .dockerignore test - * unshare: fallback to single usermapping - * addHelperSymlink: clear the destination on os.IsExist errors - * bud.bats: test replacing symbolic links - * imagebuildah: fix handling of destinations that end with '/' - * bud.bats: test COPY with a final "/" in the destination - * linux: add check for sysctl before using it - * unshare: set _CONTAINERS_ROOTLESS_GID - * Rework buildahimamges - * build context: support https git repos - * Add a test for ENV special chars behaviour - * Check in new Dockerfiles - * Apply custom SHELL during build time - * config: expand variables only at the command line - * SetEnv: we only need to expand v once - * Add default /root if empty on chroot iso - * Add support for Overlay volumes into the container. - * Export buildah validate volume functions so it can share code with libpod - * Bump baseline test to F30 - * Fix rootless handling of /dev/shm size - * Avoid fmt.Printf() in the library - * imagebuildah: tighten cache checking back up - * Handle WORKDIR with dangling target - * Default Authfile to proper path - * Make buildah run --isolation follow BUILDAH_ISOLATION environment - * Vendor in latest containers/storage and containers/image - * getParent/getChildren: handle layerless images - * imagebuildah: recognize cache images for layerless images - * bud.bats: test scratch images with --layers caching - * Get CHANGELOG.md updates - * Add some symlinks to test our .dockerignore logic - * imagebuildah: addHelper: handle symbolic links - * commit/push: use an everything-allowed policy - * Correct manpage formatting in files section - * Remove must be root statement from buildah doc - * Change image names to stable, testing and upstream - * Bump back to v1.9.0-dev + Add support for file secret mounts + Add ability to skip secrets in mounts file + allow 32bit builds + fix tutorial instructions + imagebuilder: pass the right contextDir to Add() + add: use fileutils.PatternMatcher for .dockerignore + bud.bats: add another .dockerignore test + unshare: fallback to single usermapping + addHelperSymlink: clear the destination on os.IsExist errors + bud.bats: test replacing symbolic links + imagebuildah: fix handling of destinations that end with '/' + bud.bats: test COPY with a final "/" in the destination + linux: add check for sysctl before using it + unshare: set _CONTAINERS_ROOTLESS_GID + Rework buildahimamges + build context: support https git repos + Add a test for ENV special chars behaviour + Check in new Dockerfiles + Apply custom SHELL during build time + config: expand variables only at the command line + SetEnv: we only need to expand v once + Add default /root if empty on chroot iso + Add support for Overlay volumes into the container. + Export buildah validate volume functions so it can share code with libpod + Bump baseline test to F30 + Fix rootless handling of /dev/shm size + Avoid fmt.Printf() in the library + imagebuildah: tighten cache checking back up + Handle WORKDIR with dangling target + Default Authfile to proper path + Make buildah run --isolation follow BUILDAH_ISOLATION environment + Vendor in latest containers/storage and containers/image + getParent/getChildren: handle layerless images + imagebuildah: recognize cache images for layerless images + bud.bats: test scratch images with --layers caching + Get CHANGELOG.md updates + Add some symlinks to test our .dockerignore logic + imagebuildah: addHelper: handle symbolic links + commit/push: use an everything-allowed policy + Correct manpage formatting in files section + Remove must be root statement from buildah doc + Change image names to stable, testing and upstream + Bump back to v1.9.0-dev ## v1.8.2 (2019-05-02) Vendor Storage 1.12.6 diff --git a/vendor/github.com/containers/buildah/Makefile b/vendor/github.com/containers/buildah/Makefile index 42ab36a3c..f8e079cbf 100644 --- a/vendor/github.com/containers/buildah/Makefile +++ b/vendor/github.com/containers/buildah/Makefile @@ -1,8 +1,7 @@ -export GO111MODULE=off - SELINUXTAG := $(shell ./selinux_tag.sh) +APPARMORTAG := $(shell hack/apparmor_tag.sh) STORAGETAGS := $(shell ./btrfs_tag.sh) $(shell ./btrfs_installed_tag.sh) $(shell ./libdm_tag.sh) $(shell ./ostree_tag.sh) -SECURITYTAGS ?= seccomp $(SELINUXTAG) +SECURITYTAGS ?= seccomp $(SELINUXTAG) $(APPARMORTAG) TAGS ?= $(SECURITYTAGS) $(STORAGETAGS) BUILDTAGS += $(TAGS) PREFIX := /usr/local @@ -10,9 +9,17 @@ BINDIR := $(PREFIX)/bin BASHINSTALLDIR = $(PREFIX)/share/bash-completion/completions BUILDFLAGS := -tags "$(BUILDTAGS)" BUILDAH := buildah + GO := go GO110 := 1.10 GOVERSION := $(findstring $(GO110),$(shell go version)) +# test for go module support +ifeq ($(shell go help mod >/dev/null 2>&1 && echo true), true) +export GO_BUILD=GO111MODULE=on $(GO) build -mod=vendor +else +export GO_BUILD=$(GO) build +endif + GIT_COMMIT ?= $(if $(shell git rev-parse --short HEAD),$(shell git rev-parse --short HEAD),$(error "git failed")) BUILD_INFO := $(if $(shell date +%s),$(shell date +%s),$(error "date failed")) STATIC_STORAGETAGS = "containers_image_ostree_stub containers_image_openpgp exclude_graphdriver_devicemapper $(STORAGE_TAGS)" @@ -33,15 +40,15 @@ static: $(SOURCES) .PHONY: binary binary: $(SOURCES) - $(GO) build $(LDFLAGS) -o $(BUILDAH) $(BUILDFLAGS) ./cmd/buildah + $(GO_BUILD) $(LDFLAGS) -o $(BUILDAH) $(BUILDFLAGS) ./cmd/buildah buildah: binary darwin: - GOOS=darwin $(GO) build $(LDFLAGS) -o buildah.darwin -tags "containers_image_openpgp" ./cmd/buildah + GOOS=darwin $(GO_BUILD) $(LDFLAGS) -o buildah.darwin -tags "containers_image_openpgp" ./cmd/buildah imgtype: *.go docker/*.go util/*.go tests/imgtype/imgtype.go - $(GO) build $(LDFLAGS) -o imgtype $(BUILDFLAGS) ./tests/imgtype/imgtype.go + $(GO_BUILD) $(LDFLAGS) -o imgtype $(BUILDFLAGS) ./tests/imgtype/imgtype.go .PHONY: clean clean: @@ -121,7 +128,7 @@ test-integration: install.tools cd tests; ./test_runner.sh tests/testreport/testreport: tests/testreport/testreport.go - $(GO) build -ldflags "-linkmode external -extldflags -static" -tags "$(STORAGETAGS) $(SECURITYTAGS)" -o tests/testreport/testreport ./tests/testreport + $(GO_BUILD) -ldflags "-linkmode external -extldflags -static" -tags "$(STORAGETAGS) $(SECURITYTAGS)" -o tests/testreport/testreport ./tests/testreport .PHONY: test-unit test-unit: tests/testreport/testreport diff --git a/vendor/github.com/containers/buildah/buildah.go b/vendor/github.com/containers/buildah/buildah.go index ba5a1c77e..cd0d48566 100644 --- a/vendor/github.com/containers/buildah/buildah.go +++ b/vendor/github.com/containers/buildah/buildah.go @@ -26,7 +26,7 @@ const ( Package = "buildah" // Version for the Package. Bump version in contrib/rpm/buildah.spec // too. - Version = "1.9.2" + Version = "1.10.1" // The value we use to identify what type of information, currently a // serialized Builder structure, we are using as per-container state. // This should only be changed when we make incompatible changes to diff --git a/vendor/github.com/containers/buildah/changelog.txt b/vendor/github.com/containers/buildah/changelog.txt index 9f64f903b..48c67d842 100644 --- a/vendor/github.com/containers/buildah/changelog.txt +++ b/vendor/github.com/containers/buildah/changelog.txt @@ -1,3 +1,29 @@ +- Changelog for v1.10.1 (2019-08-08) + * Bump containers/image to v3.0.2 to fix keyring issue + * Bug fix for volume minus syntax + * Bump container/storage v1.13.1 and containers/image v3.0.1 + * bump github.com/containernetworking/cni to v0.7.1 + * Add overlayfs to fuse-overlayfs tip + * Add automatic apparmor tag discovery + * Fix bug whereby --get-login has no effect + * Bump to v1.11.0-dev + +- Changelog for v1.10.0 (2019-08-02) + * vendor github.com/containers/image@v3.0.0 + * Remove GO111MODULE in favor of `-mod=vendor` + * Vendor in containers/storage v1.12.16 + * Add '-' minus syntax for removal of config values + * tests: enable overlay tests for rootless + * rootless, overlay: use fuse-overlayfs + * vendor github.com/containers/image@v2.0.1 + * Added '-' syntax to remove volume config option + * delete `successfully pushed` message + * Add golint linter and apply fixes + * vendor github.com/containers/storage@v1.12.15 + * Change wait to sleep in buildahimage readme + * Handle ReadOnly images when deleting images + * Add support for listing read/only images + - Changelog for v1.9.2 (2019-07-19) * from/import: record the base image's digest, if it has one * Fix CNI version retrieval to not require network connection diff --git a/vendor/github.com/containers/buildah/chroot/run.go b/vendor/github.com/containers/buildah/chroot/run.go index db8a4ff06..b224fb367 100644 --- a/vendor/github.com/containers/buildah/chroot/run.go +++ b/vendor/github.com/containers/buildah/chroot/run.go @@ -205,13 +205,13 @@ func runUsingChrootMain() { } // Prepare to shuttle stdio back and forth. - rootUid32, rootGid32, err := util.GetHostRootIDs(options.Spec) + rootUID32, rootGID32, err := util.GetHostRootIDs(options.Spec) if err != nil { logrus.Errorf("error determining ownership for container stdio") os.Exit(1) } - rootUid := int(rootUid32) - rootGid := int(rootGid32) + rootUID := int(rootUID32) + rootGID := int(rootGID32) relays := make(map[int]int) closeOnceRunning := []*os.File{} var ctty *os.File @@ -288,7 +288,7 @@ func runUsingChrootMain() { // Open an *os.File object that we can pass to our child. ctty = os.NewFile(ptyFd, "/dev/tty") // Set ownership for the PTY. - if err = ctty.Chown(rootUid, rootGid); err != nil { + if err = ctty.Chown(rootUID, rootGID); err != nil { var cttyInfo unix.Stat_t err2 := unix.Fstat(int(ptyFd), &cttyInfo) from := "" @@ -297,7 +297,7 @@ func runUsingChrootMain() { op = "changing" from = fmt.Sprintf("from %d/%d ", cttyInfo.Uid, cttyInfo.Gid) } - logrus.Warnf("error %s ownership of container PTY %sto %d/%d: %v", op, from, rootUid, rootGid, err) + logrus.Warnf("error %s ownership of container PTY %sto %d/%d: %v", op, from, rootUID, rootGID, err) } // Set permissions on the PTY. if err = ctty.Chmod(0620); err != nil { @@ -336,15 +336,15 @@ func runUsingChrootMain() { fdDesc[unix.Stdout] = "stdout" fdDesc[unix.Stderr] = "stderr" // Set ownership for the pipes. - if err = stdinRead.Chown(rootUid, rootGid); err != nil { + if err = stdinRead.Chown(rootUID, rootGID); err != nil { logrus.Errorf("error setting ownership of container stdin pipe: %v", err) os.Exit(1) } - if err = stdoutWrite.Chown(rootUid, rootGid); err != nil { + if err = stdoutWrite.Chown(rootUID, rootGID); err != nil { logrus.Errorf("error setting ownership of container stdout pipe: %v", err) os.Exit(1) } - if err = stderrWrite.Chown(rootUid, rootGid); err != nil { + if err = stderrWrite.Chown(rootUID, rootGID); err != nil { logrus.Errorf("error setting ownership of container stderr pipe: %v", err) os.Exit(1) } diff --git a/vendor/github.com/containers/buildah/config.go b/vendor/github.com/containers/buildah/config.go index 8665e4143..0292ea43c 100644 --- a/vendor/github.com/containers/buildah/config.go +++ b/vendor/github.com/containers/buildah/config.go @@ -422,6 +422,16 @@ func (b *Builder) Volumes() []string { return nil } +// CheckVolume returns True if the location exists in the image's list of locations +// which should be mounted from outside of the container when a container +// based on an image built from this container is run + +func (b *Builder) CheckVolume(v string) bool { + _, OCIv1Volume := b.OCIv1.Config.Volumes[v] + _, DockerVolume := b.Docker.Config.Volumes[v] + return OCIv1Volume || DockerVolume +} + // AddVolume adds a location to the image's list of locations which should be // mounted from outside of the container when a container based on an image // built from this container is run. diff --git a/vendor/github.com/containers/buildah/go.mod b/vendor/github.com/containers/buildah/go.mod index 7e80b0a83..197db35da 100644 --- a/vendor/github.com/containers/buildah/go.mod +++ b/vendor/github.com/containers/buildah/go.mod @@ -3,19 +3,16 @@ module github.com/containers/buildah go 1.12 require ( - github.com/BurntSushi/toml v0.2.0 // indirect - github.com/DataDog/zstd v1.4.0 // indirect - github.com/Microsoft/hcsshim v0.8.3 // indirect github.com/VividCortex/ewma v1.1.1 // indirect github.com/blang/semver v3.5.0+incompatible // indirect github.com/containerd/continuity v0.0.0-20181203112020-004b46473808 // indirect - github.com/containernetworking/cni v0.7.0-rc2 - github.com/containers/image v2.0.0+incompatible - github.com/containers/storage v1.12.13 + github.com/containernetworking/cni v0.7.1 + github.com/containers/image v3.0.2+incompatible + github.com/containers/storage v1.13.1 github.com/cyphar/filepath-securejoin v0.2.1 github.com/docker/distribution v0.0.0-20170817175659-5f6282db7d65 github.com/docker/docker-credential-helpers v0.6.1 // indirect - github.com/docker/go-units v0.3.3 + github.com/docker/go-units v0.4.0 github.com/docker/libnetwork v0.8.0-dev.2.0.20190625141545-5a177b73e316 github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect github.com/etcd-io/bbolt v1.3.2 @@ -26,12 +23,8 @@ require ( github.com/imdario/mergo v0.3.6 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/ishidawataru/sctp v0.0.0-20180918013207-6e2cb1366111 // indirect - github.com/klauspost/compress v1.4.1 // indirect - github.com/klauspost/cpuid v1.2.0 // indirect - github.com/klauspost/pgzip v1.2.1 // indirect github.com/mattn/go-isatty v0.0.4 // indirect - github.com/mattn/go-shellwords v1.0.3 - github.com/mistifyio/go-zfs v2.1.1+incompatible // indirect + github.com/mattn/go-shellwords v1.0.5 github.com/moby/moby v0.0.0-20171005181806-f8806b18b4b9 // indirect github.com/mtrmac/gpgme v0.0.0-20170102180018-b2432428689c // indirect github.com/onsi/ginkgo v1.6.0 @@ -43,26 +36,21 @@ require ( github.com/opencontainers/runtime-tools v0.9.0 github.com/opencontainers/selinux v1.2.2 github.com/openshift/imagebuilder v1.1.0 - github.com/ostreedev/ostree-go v0.0.0-20181112201119-9ab99253d365 // indirect github.com/pkg/errors v0.8.1 - github.com/pquerna/ffjson v0.0.0-20171002144729-d49c2bc1aa13 // indirect github.com/seccomp/containers-golang v0.0.0-20180629143253-cdfdaa7543f4 github.com/seccomp/libseccomp-golang v0.9.0 github.com/sirupsen/logrus v1.4.2 github.com/spf13/cobra v0.0.3 github.com/spf13/pflag v1.0.3 github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2 - github.com/tchap/go-patricia v2.2.6+incompatible // indirect github.com/ulikunitz/xz v0.5.5 // indirect - github.com/vbatts/tar-split v0.10.2 // indirect - github.com/vbauerster/mpb v3.3.4+incompatible // indirect + github.com/vbauerster/mpb v3.4.0+incompatible // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xeipuuv/gojsonschema v1.1.0 // indirect - golang.org/x/crypto v0.0.0-20190103213133-ff983b9c42bc - golang.org/x/net v0.0.0-20190107210223-45ffb0cd1ba0 // indirect + golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 // indirect - golang.org/x/sys v0.0.0-20190422165155-953cdadca894 + golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb gopkg.in/yaml.v2 v2.2.2 // indirect k8s.io/client-go v0.0.0-20181219152756-3dd551c0f083 // indirect ) diff --git a/vendor/github.com/containers/buildah/go.sum b/vendor/github.com/containers/buildah/go.sum index d8e8f983f..1016bcbea 100644 --- a/vendor/github.com/containers/buildah/go.sum +++ b/vendor/github.com/containers/buildah/go.sum @@ -2,12 +2,18 @@ github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7O github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/BurntSushi/toml v0.2.0 h1:OthAm9ZSUx4uAmn3WbPwc06nowWrByRwBsYRhbmFjBs= github.com/BurntSushi/toml v0.2.0/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DataDog/zstd v1.4.0 h1:vhoV+DUHnRZdKW1i5UMjAk2G4JY8wN4ayRfYDNdEhwo= github.com/DataDog/zstd v1.4.0/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/Microsoft/go-winio v0.4.11 h1:zoIOcVf0xPN1tnMVbTtEdI+P8OofVk3NObnwOQ6nK2Q= github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= +github.com/Microsoft/go-winio v0.4.12 h1:xAfWHN1IrQ0NJ9TBC0KBZoqLjzDTr1ML+4MywiUOryc= +github.com/Microsoft/go-winio v0.4.12/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= github.com/Microsoft/hcsshim v0.8.3 h1:KWCdVGOju81E0RL4ndn9/E6I4qMBi6kuPw1W4yBYlCw= github.com/Microsoft/hcsshim v0.8.3/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= +github.com/Microsoft/hcsshim v0.8.6 h1:ZfF0+zZeYdzMIVMZHKtDKJvLHj76XCuVae/jNkjj0IA= +github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM= @@ -19,15 +25,37 @@ github.com/containerd/continuity v0.0.0-20181203112020-004b46473808 h1:4BX8f882b github.com/containerd/continuity v0.0.0-20181203112020-004b46473808/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containernetworking/cni v0.7.0-rc2 h1:2GGDhbwdWPY53iT7LXy+LBP76Ch2D/hnw1U2zVFfGbk= github.com/containernetworking/cni v0.7.0-rc2/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= +github.com/containernetworking/cni v0.7.1 h1:fE3r16wpSEyaqY4Z4oFrLMmIGfBYIKpPrHK31EJ9FzE= +github.com/containernetworking/cni v0.7.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= github.com/containers/image v2.0.0+incompatible h1:FTr6Br7jlIKNCKMjSOMbAxKp2keQ0//jzJaYNTVhauk= github.com/containers/image v2.0.0+incompatible/go.mod h1:8Vtij257IWSanUQKe1tAeNOm2sRVkSqQTVQ1IlwI3+M= +github.com/containers/image v2.0.1+incompatible h1:w39mlElA/aSFZ6moFa5N+A4MWu9c8hgdMiMMYnH94Hs= +github.com/containers/image v2.0.1+incompatible/go.mod h1:8Vtij257IWSanUQKe1tAeNOm2sRVkSqQTVQ1IlwI3+M= +github.com/containers/image v3.0.0+incompatible h1:pdUHY//H+3jYNnoTt+rqY8NsStX4ZBLKzPTlMC+XvnU= +github.com/containers/image v3.0.0+incompatible/go.mod h1:8Vtij257IWSanUQKe1tAeNOm2sRVkSqQTVQ1IlwI3+M= +github.com/containers/image v3.0.1+incompatible h1:VlNEQUI1JHa1SJfJ4jz/GBt7gpk+aRYGR6TUKsxXMkU= +github.com/containers/image v3.0.1+incompatible/go.mod h1:8Vtij257IWSanUQKe1tAeNOm2sRVkSqQTVQ1IlwI3+M= +github.com/containers/image v3.0.2+incompatible h1:B1lqAE8MUPCrsBLE86J0gnXleeRq8zJnQryhiiGQNyE= +github.com/containers/image v3.0.2+incompatible/go.mod h1:8Vtij257IWSanUQKe1tAeNOm2sRVkSqQTVQ1IlwI3+M= +github.com/containers/storage v1.12.10-0.20190725063046-8038df61d6f6 h1:c7Fq9bbRl0Ua6swRHAH8rkrK2fSt6K+ZBrXHD50kDR4= +github.com/containers/storage v1.12.10-0.20190725063046-8038df61d6f6/go.mod h1:QsZp4XMJjyPNNbQHZeyNW3OmhwsWviI+7S6iOcu6a4c= github.com/containers/storage v1.12.13 h1:GtaLCY8p1Drlk1Oew581jGvB137UaO+kpz0HII67T0A= github.com/containers/storage v1.12.13/go.mod h1:+RirK6VQAqskQlaTBrOG6ulDvn4si2QjFE1NZCn06MM= +github.com/containers/storage v1.12.14 h1:S1QGlC15gj5JOvB73W5tpVBApS4I7b/6rvxfflBAg+Q= +github.com/containers/storage v1.12.14/go.mod h1:QsZp4XMJjyPNNbQHZeyNW3OmhwsWviI+7S6iOcu6a4c= +github.com/containers/storage v1.12.15 h1:nN/RxtEe4ejasGVJqzy+y5++pIYp54XPXzRO46xXnns= +github.com/containers/storage v1.12.15/go.mod h1:QsZp4XMJjyPNNbQHZeyNW3OmhwsWviI+7S6iOcu6a4c= +github.com/containers/storage v1.12.16 h1:zePYS1GiG8CuRqLCeA0ufx4X27K06HcJLV50DdojL+Y= +github.com/containers/storage v1.12.16/go.mod h1:QsZp4XMJjyPNNbQHZeyNW3OmhwsWviI+7S6iOcu6a4c= +github.com/containers/storage v1.13.1 h1:rjVirLS9fCGkUFlLDZEoGDDUugtIf46DufWvJu08wxQ= +github.com/containers/storage v1.13.1/go.mod h1:6D8nK2sU9V7nEmAraINRs88ZEscM5C5DK+8Npp27GeA= github.com/cyphar/filepath-securejoin v0.2.1 h1:5DPkzz/0MwUpvR4fxASKzgApeq2OMFY5FfYtrX28Coo= github.com/cyphar/filepath-securejoin v0.2.1/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/docker/distribution v0.0.0-20170817175659-5f6282db7d65 h1:4zlOyrJUbYnrvlzChJ+jP2J3i77Jbhm336NEuCv7kZo= github.com/docker/distribution v0.0.0-20170817175659-5f6282db7d65/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/docker v0.0.0-20171019062838-86f080cff091/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v0.7.3-0.20180827131323-0c5f8d2b9b23 h1:mJtkfC9RUrUWHMk0cFDNhVoc9U3k2FRAzEZ+5pqSIHo= github.com/docker/docker v0.7.3-0.20180827131323-0c5f8d2b9b23/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker-credential-helpers v0.6.1 h1:Dq4iIfcM7cNtddhLVWe9h4QDjsi4OER3Z8voPu/I52g= @@ -36,6 +64,8 @@ github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKoh github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.3.3 h1:Xk8S3Xj5sLGlG5g67hJmYMmUgXv5N4PhkjJHHqrwnTk= github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= +github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/libnetwork v0.8.0-dev.2.0.20180608203834-19279f049241/go.mod h1:93m0aTqz6z+g32wla4l4WxTrdtvBRmVzYRkYvasA5Z8= github.com/docker/libnetwork v0.8.0-dev.2.0.20190625141545-5a177b73e316 h1:moehPjPiGUaWdwgOl92xRyFHJyaqXDHcCyW9M6nmCK4= github.com/docker/libnetwork v0.8.0-dev.2.0.20190625141545-5a177b73e316/go.mod h1:93m0aTqz6z+g32wla4l4WxTrdtvBRmVzYRkYvasA5Z8= @@ -71,8 +101,12 @@ github.com/ishidawataru/sctp v0.0.0-20180918013207-6e2cb1366111 h1:NAAiV9ass6VRe github.com/ishidawataru/sctp v0.0.0-20180918013207-6e2cb1366111/go.mod h1:DM4VvS+hD/kDi1U1QsX2fnZowwBhqD0Dk3bRPKF/Oc8= github.com/klauspost/compress v1.4.1 h1:8VMb5+0wMgdBykOV96DwNwKFQ+WTI4pzYURP99CcB9E= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.7.2 h1:liMOoeIvFpr9kEvalrZ7VVBA4wGf7zfOgwBjzz/5g2Y= +github.com/klauspost/compress v1.7.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/cpuid v1.2.0 h1:NMpwD2G9JSFOE1/TJjGSo5zG7Yb2bTe7eq1jH+irmeE= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/cpuid v1.2.1 h1:vJi+O/nMdFt0vqm8NZBI6wzALWdA2X+egi0ogNyrC/w= +github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/pgzip v1.2.1 h1:oIPZROsWuPHpOdMVWLuJZXwgjhrW8r1yEX8UqMyeNHM= github.com/klauspost/pgzip v1.2.1/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= @@ -81,6 +115,8 @@ github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-shellwords v1.0.3 h1:K/VxK7SZ+cvuPgFSLKi5QPI9Vr/ipOf4C1gN+ntueUk= github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= +github.com/mattn/go-shellwords v1.0.5 h1:JhhFTIOslh5ZsPrpa3Wdg8bF0WI3b44EMblmU9wIsXc= +github.com/mattn/go-shellwords v1.0.5/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= github.com/mistifyio/go-zfs v2.1.1+incompatible h1:gAMO1HM9xBRONLHHYnu5iFsOJUiJdNZo6oqSENd4eW8= github.com/mistifyio/go-zfs v2.1.1+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= github.com/moby/moby v0.0.0-20171005181806-f8806b18b4b9/go.mod h1:fDXVQ6+S340veQPv35CzDahGBmHsiclFwfEygB/TWMc= @@ -108,12 +144,16 @@ github.com/openshift/imagebuilder v1.1.0 h1:oT704SkwMEzmIMU/+Uv1Wmvt+p10q3v2WuYM github.com/openshift/imagebuilder v1.1.0/go.mod h1:9aJRczxCH0mvT6XQ+5STAQaPWz7OsWcU5/mRkt8IWeo= github.com/ostreedev/ostree-go v0.0.0-20181112201119-9ab99253d365 h1:5DKEDlc/DLftia3h4tk5K0KBiqBXogCc6EarWTlD3fM= github.com/ostreedev/ostree-go v0.0.0-20181112201119-9ab99253d365/go.mod h1:J6OG6YJVEWopen4avK3VNQSnALmmjvniMmni/YFYAwc= +github.com/ostreedev/ostree-go v0.0.0-20190702140239-759a8c1ac913 h1:TnbXhKzrTOyuvWrjI8W6pcoI9XPbLHFXCdN2dtUw7Rw= +github.com/ostreedev/ostree-go v0.0.0-20190702140239-759a8c1ac913/go.mod h1:J6OG6YJVEWopen4avK3VNQSnALmmjvniMmni/YFYAwc= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pquerna/ffjson v0.0.0-20171002144729-d49c2bc1aa13 h1:AUK/hm/tPsiNNASdb3J8fySVRZoI7fnK5mlOvdFD43o= github.com/pquerna/ffjson v0.0.0-20171002144729-d49c2bc1aa13/go.mod h1:YARuvh7BUWHNhzDq2OM5tzR2RiCcN2D7sapiKyCel/M= +github.com/pquerna/ffjson v0.0.0-20181028064349-e517b90714f7 h1:gGBSHPOU7g8YjTbhwn+lvFm2VDEhhA+PwDIlstkgSxE= +github.com/pquerna/ffjson v0.0.0-20181028064349-e517b90714f7/go.mod h1:YARuvh7BUWHNhzDq2OM5tzR2RiCcN2D7sapiKyCel/M= github.com/seccomp/containers-golang v0.0.0-20180629143253-cdfdaa7543f4 h1:rOG9oHVIndNR14f3HRyBy9UPQYmIPniWqTU1TDdHhq4= github.com/seccomp/containers-golang v0.0.0-20180629143253-cdfdaa7543f4/go.mod h1:f/98/SnvAzhAEFQJ3u836FePXvcbE8BS0YGMQNn4mhA= github.com/seccomp/libseccomp-golang v0.9.0 h1:S1pmhdFh5spQtVojA+4GUdWBqvI8ydYHxrx8iR6xN8o= @@ -125,20 +165,30 @@ github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2 h1:b6uOv7YOFK0TYG7HtkIgExQo+2RdLuwRft63jn2HWj8= github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/tchap/go-patricia v2.2.6+incompatible h1:JvoDL7JSoIP2HDE8AbDH3zC8QBPxmzYe32HHy5yQ+Ck= github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I= +github.com/tchap/go-patricia v2.3.0+incompatible h1:GkY4dP3cEfEASBPPkWd+AmjYxhmDkqO9/zg7R0lSQRs= +github.com/tchap/go-patricia v2.3.0+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I= github.com/ulikunitz/xz v0.5.5 h1:pFrO0lVpTBXLpYw+pnLj6TbvHuyjXMfjGeCwSqCVwok= github.com/ulikunitz/xz v0.5.5/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= github.com/vbatts/tar-split v0.10.2 h1:CXd7HEKGkTLjBMinpObcJZU5Hm8EKlor2a1JtX6msXQ= github.com/vbatts/tar-split v0.10.2/go.mod h1:LEuURwDEiWjRjwu46yU3KVGuUdVv/dcnpcEPSzR8z6g= +github.com/vbatts/tar-split v0.11.1 h1:0Odu65rhcZ3JZaPHxl7tCI3V/C/Q9Zf82UFravl02dE= +github.com/vbatts/tar-split v0.11.1/go.mod h1:LEuURwDEiWjRjwu46yU3KVGuUdVv/dcnpcEPSzR8z6g= github.com/vbauerster/mpb v3.3.4+incompatible h1:DDIhnwmgTQIDZo+SWlEr5d6mJBxkOLBwCXPzunhEfJ4= github.com/vbauerster/mpb v3.3.4+incompatible/go.mod h1:zAHG26FUhVKETRu+MWqYXcI70POlC6N8up9p1dID7SU= +github.com/vbauerster/mpb v3.4.0+incompatible h1:mfiiYw87ARaeRW6x5gWwYRUawxaW1tLAD8IceomUCNw= +github.com/vbauerster/mpb v3.4.0+incompatible/go.mod h1:zAHG26FUhVKETRu+MWqYXcI70POlC6N8up9p1dID7SU= github.com/vishvananda/netlink v1.0.0/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI= +github.com/vrothberg/storage v0.0.0-20190724065215-a1e42fd78930 h1:/LeIxi2kj5UYTJR9W35t5Pq2gqz03ZNoTURchTH3vc0= +github.com/vrothberg/storage v0.0.0-20190724065215-a1e42fd78930/go.mod h1:QsZp4XMJjyPNNbQHZeyNW3OmhwsWviI+7S6iOcu6a4c= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= @@ -148,19 +198,27 @@ github.com/xeipuuv/gojsonschema v1.1.0/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4m golang.org/x/crypto v0.0.0-20180820150726-614d502a4dac/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190103213133-ff983b9c42bc h1:F5tKCVGp+MUAHhKp5MZtGqAlGX3+oCsiL1Q629FL90M= golang.org/x/crypto v0.0.0-20190103213133-ff983b9c42bc/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190107210223-45ffb0cd1ba0 h1:1DW40AJQ7AP4nY6ORUGUdkpXyEC9W2GAXcOPaMZK0K8= golang.org/x/net v0.0.0-20190107210223-45ffb0cd1ba0/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7 h1:rTIdg5QFRR7XCaK4LCjBiPbx8j4DQRpdYMnGn/bJUEU= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180824143301-4910a1d54f87/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb h1:fgwFCsaw9buMuxNd6+DQfAuSFqbNiQZpcgJQAgJsK6k= +golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/tools v0.0.0-20180810170437-e96c4e24768d/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= @@ -171,6 +229,7 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gotest.tools v0.0.0-20190624233834-05ebafbffc79/go.mod h1:R//lfYlUuTOTfblYI3lGoAAAebUdzjvbmQsuB7Ykd90= gotest.tools v2.1.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= k8s.io/client-go v0.0.0-20181219152756-3dd551c0f083 h1:+Qf/nITucAbm09aIdxvoA+7X0BwaXmQGVoR8k7Ynk9o= k8s.io/client-go v0.0.0-20181219152756-3dd551c0f083/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s= diff --git a/vendor/github.com/containers/buildah/install.md b/vendor/github.com/containers/buildah/install.md index 6ff2f327a..463f4ebc9 100644 --- a/vendor/github.com/containers/buildah/install.md +++ b/vendor/github.com/containers/buildah/install.md @@ -378,7 +378,7 @@ cat /etc/containers/policy.json ## Vendoring Buildah uses Go Modules for vendoring purposes. If you need to update or add a vendored package into Buildah, please follow this proceedure: - * Enter into your sandbox `src/github.com/containers/buildah` and ensure that he GOPATH variable is set to the directory prior as noted above. + * Enter into your sandbox `src/github.com/containers/buildah` and ensure that the GOPATH variable is set to the directory prior as noted above. * `export GO111MODULE=on` * Assuming you want to 'bump' the `github.com/containers/storage` package to version 1.12.13, use this command: `go get github.com/containers/storage@v1.12.13` * `make vendor` diff --git a/vendor/github.com/containers/buildah/new.go b/vendor/github.com/containers/buildah/new.go index 72d53dd99..5642ef916 100644 --- a/vendor/github.com/containers/buildah/new.go +++ b/vendor/github.com/containers/buildah/new.go @@ -8,7 +8,7 @@ import ( "github.com/containers/buildah/util" "github.com/containers/image/manifest" - "github.com/containers/image/pkg/sysregistries" + "github.com/containers/image/pkg/sysregistriesv2" is "github.com/containers/image/storage" "github.com/containers/image/transports" "github.com/containers/image/transports/alltransports" @@ -186,7 +186,7 @@ func resolveImage(ctx context.Context, systemContext *types.SystemContext, store return nil, "", nil, fmt.Errorf("internal error: %d candidates (%#v) vs. %d failures (%#v)", len(candidates), candidates, len(failures), failures) } - registriesConfPath := sysregistries.RegistriesConfPath(systemContext) + registriesConfPath := sysregistriesv2.ConfigPath(systemContext) switch len(failures) { case 0: if searchRegistriesWereUsedButEmpty { diff --git a/vendor/github.com/containers/buildah/ostree_tag.sh b/vendor/github.com/containers/buildah/ostree_tag.sh index bae9d5108..6a2f2e38b 100644 --- a/vendor/github.com/containers/buildah/ostree_tag.sh +++ b/vendor/github.com/containers/buildah/ostree_tag.sh @@ -1,6 +1,6 @@ #!/bin/bash if pkg-config ostree-1 2> /dev/null ; then - echo ostree + echo containers_image_ostree else echo containers_image_ostree_stub fi diff --git a/vendor/github.com/containers/buildah/pkg/blobcache/blobcache.go b/vendor/github.com/containers/buildah/pkg/blobcache/blobcache.go index f2f2cac79..53e6ec44b 100644 --- a/vendor/github.com/containers/buildah/pkg/blobcache/blobcache.go +++ b/vendor/github.com/containers/buildah/pkg/blobcache/blobcache.go @@ -414,8 +414,8 @@ func saveStream(wg *sync.WaitGroup, decompressReader io.ReadCloser, tempFile *os } } -func (s *blobCacheDestination) HasThreadSafePutBlob() bool { - return s.destination.HasThreadSafePutBlob() +func (d *blobCacheDestination) HasThreadSafePutBlob() bool { + return d.destination.HasThreadSafePutBlob() } func (d *blobCacheDestination) PutBlob(ctx context.Context, stream io.Reader, inputInfo types.BlobInfo, cache types.BlobInfoCache, isConfig bool) (types.BlobInfo, error) { diff --git a/vendor/github.com/containers/buildah/pkg/cli/common.go b/vendor/github.com/containers/buildah/pkg/cli/common.go index 81f3e8f32..d3c071555 100644 --- a/vendor/github.com/containers/buildah/pkg/cli/common.go +++ b/vendor/github.com/containers/buildah/pkg/cli/common.go @@ -71,7 +71,7 @@ type BudResults struct { Squash bool Tag []string Target string - TlsVerify bool + TLSVerify bool } // FromAndBugResults represents the results for common flags @@ -90,7 +90,7 @@ type FromAndBudResults struct { DNSSearch []string DNSServers []string DNSOptions []string - HttpProxy bool + HTTPProxy bool Isolation string Memory string MemorySwap string @@ -166,7 +166,7 @@ func GetBudFlags(flags *BudResults) pflag.FlagSet { fs.BoolVar(&flags.Squash, "squash", false, "Squash newly built layers into a single new layer.") fs.StringArrayVarP(&flags.Tag, "tag", "t", []string{}, "tagged `name` to apply to the built image") fs.StringVar(&flags.Target, "target", "", "set the target build stage to build") - fs.BoolVar(&flags.TlsVerify, "tls-verify", true, "require HTTPS and verify certificates when accessing the registry") + fs.BoolVar(&flags.TLSVerify, "tls-verify", true, "require HTTPS and verify certificates when accessing the registry") return fs } @@ -188,7 +188,7 @@ func GetFromAndBudFlags(flags *FromAndBudResults, usernsResults *UserNSResults, fs.StringSliceVar(&flags.DNSSearch, "dns-search", []string{}, "Set custom DNS search domains") fs.StringSliceVar(&flags.DNSServers, "dns", []string{}, "Set custom DNS servers") fs.StringSliceVar(&flags.DNSOptions, "dns-option", []string{}, "Set custom DNS options") - fs.BoolVar(&flags.HttpProxy, "http-proxy", true, "pass thru HTTP Proxy environment variables") + fs.BoolVar(&flags.HTTPProxy, "http-proxy", true, "pass thru HTTP Proxy environment variables") fs.StringVar(&flags.Isolation, "isolation", DefaultIsolation(), "`type` of process isolation to use. Use BUILDAH_ISOLATION environment variable to override.") fs.StringVarP(&flags.Memory, "memory", "m", "", "memory limit (format: <number>[<unit>], where unit = b, k, m or g)") fs.StringVar(&flags.MemorySwap, "memory-swap", "", "swap limit equal to memory plus swap: '-1' to enable unlimited swap") diff --git a/vendor/github.com/containers/buildah/pkg/overlay/overlay.go b/vendor/github.com/containers/buildah/pkg/overlay/overlay.go index 14d29a25b..ae1c63148 100644 --- a/vendor/github.com/containers/buildah/pkg/overlay/overlay.go +++ b/vendor/github.com/containers/buildah/pkg/overlay/overlay.go @@ -4,21 +4,24 @@ import ( "fmt" "io/ioutil" "os" + "os/exec" "path/filepath" "strings" + "github.com/containers/buildah/pkg/unshare" "github.com/containers/storage" "github.com/containers/storage/pkg/idtools" "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" + "golang.org/x/sys/unix" ) // MountTemp creates a subdir of the contentDir based on the source directory -// from the source system. It then mounds up the source directory on to the +// from the source system. It then mounts up the source directory on to the // generated mount point and returns the mount point to the caller. -func MountTemp(store storage.Store, containerId, source, dest string, rootUID, rootGID int) (mount specs.Mount, contentDir string, Err error) { +func MountTemp(store storage.Store, containerID, source, dest string, rootUID, rootGID int) (mount specs.Mount, contentDir string, Err error) { - containerDir, err := store.ContainerDirectory(containerId) + containerDir, err := store.ContainerDirectory(containerID) if err != nil { return mount, "", err } @@ -46,10 +49,55 @@ func MountTemp(store storage.Store, containerId, source, dest string, rootUID, r return mount, "", errors.Wrapf(err, "failed to create the overlay %s directory", workDir) } + overlayOptions := fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s,private", source, upperDir, workDir) + + if unshare.IsRootless() { + mountProgram := "" + + mountMap := map[string]bool{ + ".mount_program": true, + "overlay.mount_program": true, + "overlay2.mount_program": true, + } + + for _, i := range store.GraphOptions() { + s := strings.SplitN(i, "=", 2) + if len(s) != 2 { + continue + } + k := s[0] + v := s[1] + if mountMap[k] { + mountProgram = v + break + } + } + if mountProgram != "" { + mergeDir := filepath.Join(contentDir, "merge") + + if err := idtools.MkdirAllAs(mergeDir, 0700, rootUID, rootGID); err != nil { + return mount, "", errors.Wrapf(err, "failed to create the overlay %s directory", mergeDir) + } + + cmd := exec.Command(mountProgram, "-o", overlayOptions, mergeDir) + + if err := cmd.Run(); err != nil { + return mount, "", errors.Wrapf(err, "exec %s", mountProgram) + } + + mount.Source = mergeDir + mount.Destination = dest + mount.Type = "bind" + mount.Options = []string{"bind", "slave"} + return mount, contentDir, nil + } + /* If a mount_program is not specified, fallback to try mount native overlay. */ + } + mount.Source = "overlay" mount.Destination = dest mount.Type = "overlay" - mount.Options = strings.Split(fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s,private", source, upperDir, workDir), ",") + mount.Options = strings.Split(overlayOptions, ",") return mount, contentDir, nil } @@ -57,6 +105,14 @@ func MountTemp(store storage.Store, containerId, source, dest string, rootUID, r // RemoveTemp removes temporary mountpoint and all content from its parent // directory func RemoveTemp(contentDir string) error { + if unshare.IsRootless() { + mergeDir := filepath.Join(contentDir, "merge") + if err := unix.Unmount(mergeDir, 0); err != nil { + if !os.IsNotExist(err) { + return errors.Wrapf(err, "unmount overlay %s", mergeDir) + } + } + } return os.RemoveAll(contentDir) } @@ -64,6 +120,15 @@ func RemoveTemp(contentDir string) error { // directory func CleanupContent(containerDir string) (Err error) { contentDir := filepath.Join(containerDir, "overlay") + + if unshare.IsRootless() { + mergeDir := filepath.Join(contentDir, "merge") + if err := unix.Unmount(mergeDir, 0); err != nil { + if !os.IsNotExist(err) { + return errors.Wrapf(err, "unmount overlay %s", mergeDir) + } + } + } if err := os.RemoveAll(contentDir); err != nil && !os.IsNotExist(err) { return errors.Wrapf(err, "failed to cleanup overlay %s directory", contentDir) } diff --git a/vendor/github.com/containers/buildah/pkg/parse/parse.go b/vendor/github.com/containers/buildah/pkg/parse/parse.go index 78cbee746..0ab449c1e 100644 --- a/vendor/github.com/containers/buildah/pkg/parse/parse.go +++ b/vendor/github.com/containers/buildah/pkg/parse/parse.go @@ -14,7 +14,6 @@ import ( "unicode" "github.com/containers/buildah" - "github.com/containers/buildah/pkg/unshare" "github.com/containers/image/types" "github.com/containers/storage/pkg/idtools" "github.com/docker/go-units" @@ -104,7 +103,7 @@ func CommonBuildOptions(c *cobra.Command) (*buildah.CommonBuildOptions, error) { return nil, errors.Wrapf(err, "invalid --shm-size") } volumes, _ := c.Flags().GetStringSlice("volume") - if err := ParseVolumes(volumes); err != nil { + if err := Volumes(volumes); err != nil { return nil, err } cpuPeriod, _ := c.Flags().GetUint64("cpu-period") @@ -179,8 +178,8 @@ func parseSecurityOpts(securityOpts []string, commonOpts *buildah.CommonBuildOpt return nil } -// ParseVolume parses the input of --volume -func ParseVolume(volume string) (specs.Mount, error) { +// Volume parses the input of --volume +func Volume(volume string) (specs.Mount, error) { mount := specs.Mount{} arr := strings.SplitN(volume, ":", 3) if len(arr) < 2 { @@ -207,13 +206,13 @@ func ParseVolume(volume string) (specs.Mount, error) { return mount, nil } -// ParseVolumes validates the host and container paths passed in to the --volume flag -func ParseVolumes(volumes []string) error { +// Volumes validates the host and container paths passed in to the --volume flag +func Volumes(volumes []string) error { if len(volumes) == 0 { return nil } for _, volume := range volumes { - if _, err := ParseVolume(volume); err != nil { + if _, err := Volume(volume); err != nil { return err } } @@ -224,7 +223,7 @@ func getVolumeMounts(volumes []string) (map[string]specs.Mount, error) { finalVolumeMounts := make(map[string]specs.Mount) for _, volume := range volumes { - volumeMount, err := ParseVolume(volume) + volumeMount, err := Volume(volume) if err != nil { return nil, err } @@ -473,9 +472,6 @@ func ValidateVolumeOpts(options []string) ([]string, error) { } foundRWRO++ case "z", "Z", "O": - if opt == "O" && unshare.IsRootless() { - return nil, errors.Errorf("invalid options %q, overlay mounts not supported in rootless mode", strings.Join(options, ", ")) - } if foundLabelChange > 1 { return nil, errors.Errorf("invalid options %q, can only specify 1 'z', 'Z', or 'O' option", strings.Join(options, ", ")) } diff --git a/vendor/github.com/containers/buildah/troubleshooting.md b/vendor/github.com/containers/buildah/troubleshooting.md index 3e292a6b2..96fa403f0 100644 --- a/vendor/github.com/containers/buildah/troubleshooting.md +++ b/vendor/github.com/containers/buildah/troubleshooting.md @@ -112,9 +112,7 @@ lstat /home/myusername/~: no such file or directory --- ### 5) Rootless buildah bud fails EPERM on NFS: -NFS enforces file creation on different UIDs on the server side and does not understand User Namespace. -When a container root process like YUM attempts to create a file owned by a different UID, NFS Server denies the creation. -NFS is also a problem for the file locks when the storage is on it. +NFS enforces file creation on different UIDs on the server side and does not understand user namespace, which rootless Podman requires. When a container root process like YUM attempts to create a file owned by a different UID, NFS Server denies the creation. NFS is also a problem for the file locks when the storage is on it. Other distributed file systems (for example: Lustre, Spectrum Scale, the General Parallel File System (GPFS)) are also not supported when running in rootless mode as these file systems do not understand user namespace. #### Symptom ```console @@ -127,4 +125,34 @@ error creating build container: Error committing the finished image: error addin Choose one of the following: * Setup containers/storage in a different directory, not on an NFS share. * Otherwise just run buildah as root, via `sudo buildah` ----
\ No newline at end of file +--- +### 6) Rootless buildah bud fails when using OverlayFS: + +The Overlay file system (OverlayFS) requires the ability to call the `mknod` command when creating whiteout files +when extracting an image. However, a rootless user does not have the privileges to use `mknod` in this capacity. + +#### Symptom +```console +buildah bud --storage-driver overlay . +STEP 1: FROM docker.io/ubuntu:xenial +Getting image source signatures +Copying blob edf72af6d627 done +Copying blob 3e4f86211d23 done +Copying blob 8d3eac894db4 done +Copying blob f7277927d38a done +Copying config 5e13f8dd4c done +Writing manifest to image destination +Storing signatures +Error: error creating build container: Error committing the finished image: error adding layer with blob "sha256:8d3eac894db4dc4154377ad28643dfe6625ff0e54bcfa63e0d04921f1a8ef7f8": Error processing tar file(exit status 1): operation not permitted +$ buildah bud . +ERRO[0014] Error while applying layer: ApplyLayer exit status 1 stdout: stderr: open /root/.bash_logout: permission denied +error creating build container: Error committing the finished image: error adding layer with blob "sha256:a02a4930cb5d36f3290eb84f4bfa30668ef2e9fe3a1fb73ec015fc58b9958b17": ApplyLayer exit status 1 stdout: stderr: open /root/.bash_logout: permission denied +``` + +#### Solution +Choose one of the following: + * Complete the build operation as a privileged user. + * Install and configure fuse-overlayfs. + * Install the fuse-overlayfs package for your Linux Distribution. + * Add `mount_program = "/usr/bin/fuse-overlayfs` under `[storage.options]` in your `~/.config/containers/storage.conf` file. +--- diff --git a/vendor/github.com/containers/buildah/util.go b/vendor/github.com/containers/buildah/util.go index 71505f19e..a44165921 100644 --- a/vendor/github.com/containers/buildah/util.go +++ b/vendor/github.com/containers/buildah/util.go @@ -8,7 +8,6 @@ import ( "github.com/containers/buildah/util" "github.com/containers/image/docker/reference" - "github.com/containers/image/pkg/sysregistries" "github.com/containers/image/pkg/sysregistriesv2" "github.com/containers/image/types" "github.com/containers/storage" @@ -279,17 +278,17 @@ func (b *Builder) untar(chownOpts *idtools.IDPair, hasher io.Writer) func(tarArc func isRegistryBlocked(registry string, sc *types.SystemContext) (bool, error) { reginfo, err := sysregistriesv2.FindRegistry(sc, registry) if err != nil { - return false, errors.Wrapf(err, "unable to parse the registries configuration (%s)", sysregistries.RegistriesConfPath(sc)) + return false, errors.Wrapf(err, "unable to parse the registries configuration (%s)", sysregistriesv2.ConfigPath(sc)) } if reginfo != nil { if reginfo.Blocked { - logrus.Debugf("registry %q is marked as blocked in registries configuration %q", registry, sysregistries.RegistriesConfPath(sc)) + logrus.Debugf("registry %q is marked as blocked in registries configuration %q", registry, sysregistriesv2.ConfigPath(sc)) } else { - logrus.Debugf("registry %q is not marked as blocked in registries configuration %q", registry, sysregistries.RegistriesConfPath(sc)) + logrus.Debugf("registry %q is not marked as blocked in registries configuration %q", registry, sysregistriesv2.ConfigPath(sc)) } return reginfo.Blocked, nil } - logrus.Debugf("registry %q is not listed in registries configuration %q, assuming it's not blocked", registry, sysregistries.RegistriesConfPath(sc)) + logrus.Debugf("registry %q is not listed in registries configuration %q, assuming it's not blocked", registry, sysregistriesv2.ConfigPath(sc)) return false, nil } diff --git a/vendor/github.com/containers/image/docker/docker_client.go b/vendor/github.com/containers/image/docker/docker_client.go index 81a43e0fa..48427f3d3 100644 --- a/vendor/github.com/containers/image/docker/docker_client.go +++ b/vendor/github.com/containers/image/docker/docker_client.go @@ -254,6 +254,9 @@ func newDockerClient(sys *types.SystemContext, registry, reference string) (*doc return nil, errors.Wrapf(err, "error loading registries") } if reg != nil { + if reg.Blocked { + return nil, fmt.Errorf("registry %s is blocked in %s", reg.Prefix, sysregistriesv2.ConfigPath(sys)) + } skipVerify = reg.Insecure } tlsClientConfig.InsecureSkipVerify = skipVerify @@ -523,11 +526,7 @@ func (c *dockerClient) getBearerToken(ctx context.Context, challenge challenge, authReq.SetBasicAuth(c.username, c.password) } logrus.Debugf("%s %s", authReq.Method, authReq.URL.String()) - tr := tlsclientconfig.NewTransport() - // TODO(runcom): insecure for now to contact the external token service - tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} - client := &http.Client{Transport: tr} - res, err := client.Do(authReq) + res, err := c.client.Do(authReq) if err != nil { return nil, err } diff --git a/vendor/github.com/containers/image/ostree/ostree_src.go b/vendor/github.com/containers/image/ostree/ostree_src.go index dc52ccb6e..43d8f6837 100644 --- a/vendor/github.com/containers/image/ostree/ostree_src.go +++ b/vendor/github.com/containers/image/ostree/ostree_src.go @@ -59,9 +59,15 @@ func (s *ostreeImageSource) Close() error { return nil } -func (s *ostreeImageSource) getLayerSize(blob string) (int64, error) { +func (s *ostreeImageSource) getBlobUncompressedSize(blob string, isCompressed bool) (int64, error) { + var metadataKey string + if isCompressed { + metadataKey = "docker.uncompressed_size" + } else { + metadataKey = "docker.size" + } b := fmt.Sprintf("ociimage/%s", blob) - found, data, err := readMetadata(s.repo, b, "docker.size") + found, data, err := readMetadata(s.repo, b, metadataKey) if err != nil || !found { return 0, err } @@ -275,8 +281,8 @@ func (s *ostreeImageSource) GetBlob(ctx context.Context, info types.BlobInfo, ca } } - compressedBlob, found := s.compressed[info.Digest] - if found { + compressedBlob, isCompressed := s.compressed[info.Digest] + if isCompressed { blob = compressedBlob.Hex() } branch := fmt.Sprintf("ociimage/%s", blob) @@ -289,7 +295,7 @@ func (s *ostreeImageSource) GetBlob(ctx context.Context, info types.BlobInfo, ca s.repo = repo } - layerSize, err := s.getLayerSize(blob) + layerSize, err := s.getBlobUncompressedSize(blob, isCompressed) if err != nil { return nil, 0, err } diff --git a/vendor/github.com/containers/image/pkg/docker/config/config.go b/vendor/github.com/containers/image/pkg/docker/config/config.go index 2e6bb378f..eef629d5c 100644 --- a/vendor/github.com/containers/image/pkg/docker/config/config.go +++ b/vendor/github.com/containers/image/pkg/docker/config/config.go @@ -32,9 +32,13 @@ var ( dockerHomePath = filepath.FromSlash(".docker/config.json") dockerLegacyHomePath = ".dockercfg" + enableKeyring = false + // ErrNotLoggedIn is returned for users not logged into a registry // that they are trying to logout of ErrNotLoggedIn = errors.New("not logged in") + // ErrNotSupported is returned for unsupported methods + ErrNotSupported = errors.New("not supported") ) // SetAuthentication stores the username and password in the auth.json file @@ -44,6 +48,18 @@ func SetAuthentication(sys *types.SystemContext, registry, username, password st return false, setAuthToCredHelper(ch, registry, username, password) } + // Set the credentials to kernel keyring if enableKeyring is true. + // The keyring might not work in all environments (e.g., missing capability) and isn't supported on all platforms. + // Hence, we want to fall-back to using the authfile in case the keyring failed. + // However, if the enableKeyring is false, we want adhere to the user specification and not use the keyring. + if enableKeyring { + err := setAuthToKernelKeyring(registry, username, password) + if err == nil { + logrus.Debugf("credentials for (%s, %s) were stored in the kernel keyring\n", registry, username) + return false, nil + } + logrus.Debugf("failed to authenticate with the kernel keyring, falling back to authfiles. %v", err) + } creds := base64.StdEncoding.EncodeToString([]byte(username + ":" + password)) newCreds := dockerAuthConfig{Auth: creds} auths.AuthConfigs[registry] = newCreds @@ -60,6 +76,14 @@ func GetAuthentication(sys *types.SystemContext, registry string) (string, strin return sys.DockerAuthConfig.Username, sys.DockerAuthConfig.Password, nil } + if enableKeyring { + username, password, err := getAuthFromKernelKeyring(registry) + if err == nil { + logrus.Debug("returning credentials from kernel keyring") + return username, password, nil + } + } + dockerLegacyPath := filepath.Join(homedir.Get(), dockerLegacyHomePath) var paths []string pathToAuth, err := getPathToAuth(sys) @@ -97,6 +121,16 @@ func RemoveAuthentication(sys *types.SystemContext, registry string) error { return false, deleteAuthFromCredHelper(ch, registry) } + // Next if keyring is enabled try kernel keyring + if enableKeyring { + err := deleteAuthFromKernelKeyring(registry) + if err == nil { + logrus.Debugf("credentials for %s were deleted from the kernel keyring", registry) + return false, nil + } + logrus.Debugf("failed to delete credentials from the kernel keyring, falling back to authfiles") + } + if _, ok := auths.AuthConfigs[registry]; ok { delete(auths.AuthConfigs, registry) } else if _, ok := auths.AuthConfigs[normalizeRegistry(registry)]; ok { diff --git a/vendor/github.com/containers/image/pkg/docker/config/config_linux.go b/vendor/github.com/containers/image/pkg/docker/config/config_linux.go new file mode 100644 index 000000000..4d66a50df --- /dev/null +++ b/vendor/github.com/containers/image/pkg/docker/config/config_linux.go @@ -0,0 +1,79 @@ +package config + +import ( + "fmt" + "strings" + + "github.com/containers/image/pkg/keyctl" + "github.com/pkg/errors" +) + +func getAuthFromKernelKeyring(registry string) (string, string, error) { + userkeyring, err := keyctl.UserKeyring() + if err != nil { + return "", "", err + } + key, err := userkeyring.Search(genDescription(registry)) + if err != nil { + return "", "", err + } + authData, err := key.Get() + if err != nil { + return "", "", err + } + parts := strings.SplitN(string(authData), "\x00", 2) + if len(parts) != 2 { + return "", "", nil + } + return parts[0], parts[1], nil +} + +func deleteAuthFromKernelKeyring(registry string) error { + userkeyring, err := keyctl.UserKeyring() + + if err != nil { + return err + } + key, err := userkeyring.Search(genDescription(registry)) + if err != nil { + return err + } + return key.Unlink() +} + +func setAuthToKernelKeyring(registry, username, password string) error { + keyring, err := keyctl.SessionKeyring() + if err != nil { + return err + } + id, err := keyring.Add(genDescription(registry), []byte(fmt.Sprintf("%s\x00%s", username, password))) + if err != nil { + return err + } + + // sets all permission(view,read,write,search,link,set attribute) for current user + // it enables the user to search the key after it linked to user keyring and unlinked from session keyring + err = keyctl.SetPerm(id, keyctl.PermUserAll) + if err != nil { + return err + } + // link the key to userKeyring + userKeyring, err := keyctl.UserKeyring() + if err != nil { + return errors.Wrapf(err, "error getting user keyring") + } + err = keyctl.Link(userKeyring, id) + if err != nil { + return errors.Wrapf(err, "error linking the key to user keyring") + } + // unlink the key from session keyring + err = keyctl.Unlink(keyring, id) + if err != nil { + return errors.Wrapf(err, "error unlinking the key from session keyring") + } + return nil +} + +func genDescription(registry string) string { + return fmt.Sprintf("container-registry-login:%s", registry) +} diff --git a/vendor/github.com/containers/image/pkg/docker/config/config_unsupported.go b/vendor/github.com/containers/image/pkg/docker/config/config_unsupported.go new file mode 100644 index 000000000..1c1a02511 --- /dev/null +++ b/vendor/github.com/containers/image/pkg/docker/config/config_unsupported.go @@ -0,0 +1,16 @@ +// +build !linux +// +build !386 !amd64 + +package config + +func getAuthFromKernelKeyring(registry string) (string, string, error) { + return "", "", ErrNotSupported +} + +func deleteAuthFromKernelKeyring(registry string) error { + return ErrNotSupported +} + +func setAuthToKernelKeyring(registry, username, password string) error { + return ErrNotSupported +} diff --git a/vendor/github.com/containers/image/pkg/keyctl/key.go b/vendor/github.com/containers/image/pkg/keyctl/key.go new file mode 100644 index 000000000..e4396a9df --- /dev/null +++ b/vendor/github.com/containers/image/pkg/keyctl/key.go @@ -0,0 +1,64 @@ +// Copyright 2015 Jesse Sipprell. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build linux + +package keyctl + +import ( + "golang.org/x/sys/unix" +) + +// Key represents a single key linked to one or more kernel keyrings. +type Key struct { + Name string + + id, ring keyID + size int +} + +// ID returns the 32-bit kernel identifier for a specific key +func (k *Key) ID() int32 { + return int32(k.id) +} + +// Get the key's value as a byte slice +func (k *Key) Get() ([]byte, error) { + var ( + b []byte + err error + sizeRead int + ) + + if k.size == 0 { + k.size = 512 + } + + size := k.size + + b = make([]byte, int(size)) + sizeRead = size + 1 + for sizeRead > size { + r1, err := unix.KeyctlBuffer(unix.KEYCTL_READ, int(k.id), b, size) + if err != nil { + return nil, err + } + + if sizeRead = int(r1); sizeRead > size { + b = make([]byte, sizeRead) + size = sizeRead + sizeRead = size + 1 + } else { + k.size = sizeRead + } + } + return b[:k.size], err +} + +// Unlink a key from the keyring it was loaded from (or added to). If the key +// is not linked to any other keyrings, it is destroyed. +func (k *Key) Unlink() error { + _, err := unix.KeyctlInt(unix.KEYCTL_UNLINK, int(k.id), int(k.ring), 0, 0) + return err +} diff --git a/vendor/github.com/containers/image/pkg/keyctl/keyring.go b/vendor/github.com/containers/image/pkg/keyctl/keyring.go new file mode 100644 index 000000000..6e029c923 --- /dev/null +++ b/vendor/github.com/containers/image/pkg/keyctl/keyring.go @@ -0,0 +1,79 @@ +// Copyright 2015 Jesse Sipprell. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build linux + +// Package keyctl is a Go interface to linux kernel keyrings (keyctl interface) +// +// Deprecated: Most callers should use either golang.org/x/sys/unix directly, +// or the original (and more extensive) github.com/jsipprell/keyctl . +package keyctl + +import ( + "golang.org/x/sys/unix" +) + +// Keyring is the basic interface to a linux keyctl keyring. +type Keyring interface { + ID + Add(string, []byte) (*Key, error) + Search(string) (*Key, error) +} + +type keyring struct { + id keyID +} + +// ID is unique 32-bit serial number identifiers for all Keys and Keyrings have. +type ID interface { + ID() int32 +} + +// Add a new key to a keyring. The key can be searched for later by name. +func (kr *keyring) Add(name string, key []byte) (*Key, error) { + r, err := unix.AddKey("user", name, key, int(kr.id)) + if err == nil { + key := &Key{Name: name, id: keyID(r), ring: kr.id} + return key, nil + } + return nil, err +} + +// Search for a key by name, this also searches child keyrings linked to this +// one. The key, if found, is linked to the top keyring that Search() was called +// from. +func (kr *keyring) Search(name string) (*Key, error) { + id, err := unix.KeyctlSearch(int(kr.id), "user", name, 0) + if err == nil { + return &Key{Name: name, id: keyID(id), ring: kr.id}, nil + } + return nil, err +} + +// ID returns the 32-bit kernel identifier of a keyring +func (kr *keyring) ID() int32 { + return int32(kr.id) +} + +// SessionKeyring returns the current login session keyring +func SessionKeyring() (Keyring, error) { + return newKeyring(unix.KEY_SPEC_SESSION_KEYRING) +} + +// UserKeyring returns the keyring specific to the current user. +func UserKeyring() (Keyring, error) { + return newKeyring(unix.KEY_SPEC_USER_KEYRING) +} + +// Unlink an object from a keyring +func Unlink(parent Keyring, child ID) error { + _, err := unix.KeyctlInt(unix.KEYCTL_UNLINK, int(child.ID()), int(parent.ID()), 0, 0) + return err +} + +// Link a key into a keyring +func Link(parent Keyring, child ID) error { + _, err := unix.KeyctlInt(unix.KEYCTL_LINK, int(child.ID()), int(parent.ID()), 0, 0) + return err +} diff --git a/vendor/github.com/containers/image/pkg/keyctl/perm.go b/vendor/github.com/containers/image/pkg/keyctl/perm.go new file mode 100644 index 000000000..ae9697149 --- /dev/null +++ b/vendor/github.com/containers/image/pkg/keyctl/perm.go @@ -0,0 +1,33 @@ +// Copyright 2015 Jesse Sipprell. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build linux + +package keyctl + +import ( + "golang.org/x/sys/unix" +) + +// KeyPerm represents in-kernel access control permission to keys and keyrings +// as a 32-bit integer broken up into four permission sets, one per byte. +// In MSB order, the perms are: Processor, User, Group, Other. +type KeyPerm uint32 + +const ( + // PermOtherAll sets all permission for Other + PermOtherAll KeyPerm = 0x3f << (8 * iota) + // PermGroupAll sets all permission for Group + PermGroupAll + // PermUserAll sets all permission for User + PermUserAll + // PermProcessAll sets all permission for Processor + PermProcessAll +) + +// SetPerm sets the permissions on a key or keyring. +func SetPerm(k ID, p KeyPerm) error { + err := unix.KeyctlSetperm(int(k.ID()), uint32(p)) + return err +} diff --git a/vendor/github.com/containers/image/pkg/keyctl/sys_linux.go b/vendor/github.com/containers/image/pkg/keyctl/sys_linux.go new file mode 100644 index 000000000..196c82760 --- /dev/null +++ b/vendor/github.com/containers/image/pkg/keyctl/sys_linux.go @@ -0,0 +1,25 @@ +// Copyright 2015 Jesse Sipprell. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build linux + +package keyctl + +import ( + "golang.org/x/sys/unix" +) + +type keyID int32 + +func newKeyring(id keyID) (*keyring, error) { + r1, err := unix.KeyctlGetKeyringID(int(id), true) + if err != nil { + return nil, err + } + + if id < 0 { + r1 = int(id) + } + return &keyring{id: keyID(r1)}, nil +} diff --git a/vendor/github.com/containers/image/pkg/sysregistries/system_registries.go b/vendor/github.com/containers/image/pkg/sysregistries/system_registries.go deleted file mode 100644 index 09b419cc8..000000000 --- a/vendor/github.com/containers/image/pkg/sysregistries/system_registries.go +++ /dev/null @@ -1,103 +0,0 @@ -package sysregistries - -import ( - "strings" - - "github.com/BurntSushi/toml" - "github.com/containers/image/types" - "io/ioutil" - "path/filepath" -) - -// systemRegistriesConfPath is the path to the system-wide registry configuration file -// and is used to add/subtract potential registries for obtaining images. -// You can override this at build time with -// -ldflags '-X github.com/containers/image/sysregistries.systemRegistriesConfPath=$your_path' -var systemRegistriesConfPath = builtinRegistriesConfPath - -// builtinRegistriesConfPath is the path to registry configuration file -// DO NOT change this, instead see systemRegistriesConfPath above. -const builtinRegistriesConfPath = "/etc/containers/registries.conf" - -type registries struct { - Registries []string `toml:"registries"` -} - -type tomlConfig struct { - Registries struct { - Search registries `toml:"search"` - Insecure registries `toml:"insecure"` - Block registries `toml:"block"` - } `toml:"registries"` -} - -// normalizeRegistries removes trailing slashes from registries, which is a -// common pitfall when configuring registries (e.g., "docker.io/library/). -func normalizeRegistries(regs *registries) { - for i := range regs.Registries { - regs.Registries[i] = strings.TrimRight(regs.Registries[i], "/") - } -} - -// Reads the global registry file from the filesystem. Returns -// a byte array -func readRegistryConf(sys *types.SystemContext) ([]byte, error) { - return ioutil.ReadFile(RegistriesConfPath(sys)) -} - -// For mocking in unittests -var readConf = readRegistryConf - -// Loads the registry configuration file from the filesystem and -// then unmarshals it. Returns the unmarshalled object. -func loadRegistryConf(sys *types.SystemContext) (*tomlConfig, error) { - config := &tomlConfig{} - - configBytes, err := readConf(sys) - if err != nil { - return nil, err - } - - err = toml.Unmarshal(configBytes, &config) - normalizeRegistries(&config.Registries.Search) - normalizeRegistries(&config.Registries.Insecure) - normalizeRegistries(&config.Registries.Block) - return config, err -} - -// GetRegistries returns an array of strings that contain the names -// of the registries as defined in the system-wide -// registries file. it returns an empty array if none are -// defined -func GetRegistries(sys *types.SystemContext) ([]string, error) { - config, err := loadRegistryConf(sys) - if err != nil { - return nil, err - } - return config.Registries.Search.Registries, nil -} - -// GetInsecureRegistries returns an array of strings that contain the names -// of the insecure registries as defined in the system-wide -// registries file. it returns an empty array if none are -// defined -func GetInsecureRegistries(sys *types.SystemContext) ([]string, error) { - config, err := loadRegistryConf(sys) - if err != nil { - return nil, err - } - return config.Registries.Insecure.Registries, nil -} - -// RegistriesConfPath is the path to the system-wide registry configuration file -func RegistriesConfPath(ctx *types.SystemContext) string { - path := systemRegistriesConfPath - if ctx != nil { - if ctx.SystemRegistriesConfPath != "" { - path = ctx.SystemRegistriesConfPath - } else if ctx.RootForImplicitAbsolutePaths != "" { - path = filepath.Join(ctx.RootForImplicitAbsolutePaths, systemRegistriesConfPath) - } - } - return path -} diff --git a/vendor/github.com/containers/image/pkg/sysregistriesv2/system_registries_v2.go b/vendor/github.com/containers/image/pkg/sysregistriesv2/system_registries_v2.go index 0c13913ed..bed92cb90 100644 --- a/vendor/github.com/containers/image/pkg/sysregistriesv2/system_registries_v2.go +++ b/vendor/github.com/containers/image/pkg/sysregistriesv2/system_registries_v2.go @@ -303,9 +303,8 @@ func (config *V2RegistriesConf) postProcess() error { return nil } -// getConfigPath returns the system-registries config path if specified. -// Otherwise, systemRegistriesConfPath is returned. -func getConfigPath(ctx *types.SystemContext) string { +// ConfigPath returns the path to the system-wide registry configuration file. +func ConfigPath(ctx *types.SystemContext) string { confPath := systemRegistriesConfPath if ctx != nil { if ctx.SystemRegistriesConfPath != "" { @@ -336,14 +335,27 @@ func InvalidateCache() { // getConfig returns the config object corresponding to ctx, loading it if it is not yet cached. func getConfig(ctx *types.SystemContext) (*V2RegistriesConf, error) { - configPath := getConfigPath(ctx) + configPath := ConfigPath(ctx) configMutex.Lock() - defer configMutex.Unlock() // if the config has already been loaded, return the cached registries if config, inCache := configCache[configPath]; inCache { + configMutex.Unlock() return config, nil } + configMutex.Unlock() + + return TryUpdatingCache(ctx) +} + +// TryUpdatingCache loads the configuration from the provided `SystemContext` +// without using the internal cache. On success, the loaded configuration will +// be added into the internal registry cache. +func TryUpdatingCache(ctx *types.SystemContext) (*V2RegistriesConf, error) { + configPath := ConfigPath(ctx) + + configMutex.Lock() + defer configMutex.Unlock() // load the config config, err := loadRegistryConf(configPath) diff --git a/vendor/github.com/containers/image/signature/policy_types.go b/vendor/github.com/containers/image/signature/policy_types.go index 4cd770f11..d3b33bb7a 100644 --- a/vendor/github.com/containers/image/signature/policy_types.go +++ b/vendor/github.com/containers/image/signature/policy_types.go @@ -6,7 +6,7 @@ package signature -// NOTE: Keep this in sync with docs/policy.json.md! +// NOTE: Keep this in sync with docs/containers-policy.json.5.md! // Policy defines requirements for considering a signature, or an image, valid. type Policy struct { diff --git a/vendor/github.com/containers/image/transports/alltransports/alltransports.go b/vendor/github.com/containers/image/transports/alltransports/alltransports.go index 2335c567f..3a988f3f8 100644 --- a/vendor/github.com/containers/image/transports/alltransports/alltransports.go +++ b/vendor/github.com/containers/image/transports/alltransports/alltransports.go @@ -4,7 +4,7 @@ import ( "strings" // register all known transports - // NOTE: Make sure docs/policy.json.md is updated when adding or updating + // NOTE: Make sure docs/containers-policy.json.5.md is updated when adding or updating // a transport. _ "github.com/containers/image/directory" _ "github.com/containers/image/docker" diff --git a/vendor/github.com/containers/image/version/version.go b/vendor/github.com/containers/image/version/version.go index 807daf7a2..f1e795d9b 100644 --- a/vendor/github.com/containers/image/version/version.go +++ b/vendor/github.com/containers/image/version/version.go @@ -4,11 +4,11 @@ import "fmt" const ( // VersionMajor is for an API incompatible changes - VersionMajor = 2 + VersionMajor = 3 // VersionMinor is for functionality in a backwards-compatible manner VersionMinor = 0 // VersionPatch is for backwards-compatible bug fixes - VersionPatch = 1 + VersionPatch = 2 // VersionDev indicates development branch. Releases will be empty string. VersionDev = "" diff --git a/vendor/github.com/containers/storage/Makefile b/vendor/github.com/containers/storage/Makefile index de326aeb1..bb1de007b 100644 --- a/vendor/github.com/containers/storage/Makefile +++ b/vendor/github.com/containers/storage/Makefile @@ -27,10 +27,16 @@ PACKAGE := github.com/containers/storage GIT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD 2>/dev/null) GIT_BRANCH_CLEAN := $(shell echo $(GIT_BRANCH) | sed -e "s/[^[:alnum:]]/-/g") EPOCH_TEST_COMMIT := 0418ebf59f9e1f564831c0ba9378b7f8e40a1c73 -NATIVETAGS := exclude_graphdriver_devicemapper exclude_graphdriver_btrfs exclude_graphdriver_overlay +NATIVETAGS := AUTOTAGS := $(shell ./hack/btrfs_tag.sh) $(shell ./hack/libdm_tag.sh) $(shell ./hack/ostree_tag.sh) BUILDFLAGS := -tags "$(AUTOTAGS) $(TAGS)" $(FLAGS) -GO := go +GO ?= go + +GO_BUILD=$(GO) build +# Go module support: set `-mod=vendor` to use the vendored sources +ifeq ($(shell $(GO) help mod >/dev/null 2>&1 && echo true), true) + GO_BUILD=GO111MODULE=on $(GO) build -mod=vendor +endif RUNINVM := vagrant/runinvm.sh FFJSON := tests/tools/build/ffjson @@ -43,7 +49,7 @@ clean: ## remove all built files sources := $(wildcard *.go cmd/containers-storage/*.go drivers/*.go drivers/*/*.go pkg/*/*.go pkg/*/*/*.go) layers_ffjson.go images_ffjson.go containers_ffjson.go pkg/archive/archive_ffjson.go containers-storage: $(sources) ## build using gc on the host - $(GO) build -compiler gc $(BUILDFLAGS) ./cmd/containers-storage + $(GO_BUILD) -compiler gc $(BUILDFLAGS) ./cmd/containers-storage layers_ffjson.go: layers.go $(RM) $@ @@ -64,15 +70,15 @@ pkg/archive/archive_ffjson.go: pkg/archive/archive.go binary local-binary: containers-storage local-gccgo: ## build using gccgo on the host - GCCGO=$(PWD)/hack/gccgo-wrapper.sh $(GO) build -compiler gccgo $(BUILDFLAGS) -o containers-storage.gccgo ./cmd/containers-storage + GCCGO=$(PWD)/hack/gccgo-wrapper.sh $(GO_BUILD) -compiler gccgo $(BUILDFLAGS) -o containers-storage.gccgo ./cmd/containers-storage local-cross: ## cross build the binaries for arm, darwin, and\nfreebsd @for target in linux/amd64 linux/386 linux/arm linux/arm64 linux/ppc64 linux/ppc64le darwin/amd64 windows/amd64 ; do \ os=`echo $${target} | cut -f1 -d/` ; \ arch=`echo $${target} | cut -f2 -d/` ; \ suffix=$${os}.$${arch} ; \ - echo env CGO_ENABLED=0 GOOS=$${os} GOARCH=$${arch} $(GO) build -compiler gc -tags \"$(NATIVETAGS) $(TAGS)\" $(FLAGS) -o containers-storage.$${suffix} ./cmd/containers-storage ; \ - env CGO_ENABLED=0 GOOS=$${os} GOARCH=$${arch} $(GO) build -compiler gc -tags "$(NATIVETAGS) $(TAGS)" $(FLAGS) -o containers-storage.$${suffix} ./cmd/containers-storage || exit 1 ; \ + echo env CGO_ENABLED=0 GOOS=$${os} GOARCH=$${arch} $(GO_BUILD) -compiler gc -tags \"$(NATIVETAGS) $(TAGS)\" $(FLAGS) -o containers-storage.$${suffix} ./cmd/containers-storage ; \ + env CGO_ENABLED=0 GOOS=$${os} GOARCH=$${arch} $(GO_BUILD) -compiler gc -tags "$(NATIVETAGS) $(TAGS)" $(FLAGS) -o containers-storage.$${suffix} ./cmd/containers-storage || exit 1 ; \ done cross: ## cross build the binaries for arm, darwin, and\nfreebsd using VMs diff --git a/vendor/github.com/containers/storage/VERSION b/vendor/github.com/containers/storage/VERSION index c22268b43..065f9ec4c 100644 --- a/vendor/github.com/containers/storage/VERSION +++ b/vendor/github.com/containers/storage/VERSION @@ -1 +1 @@ -1.12.16 +1.13.3-dev diff --git a/vendor/github.com/containers/storage/drivers/btrfs/btrfs.go b/vendor/github.com/containers/storage/drivers/btrfs/btrfs.go index 6f632a98d..1f719fa85 100644 --- a/vendor/github.com/containers/storage/drivers/btrfs/btrfs.go +++ b/vendor/github.com/containers/storage/drivers/btrfs/btrfs.go @@ -1,4 +1,4 @@ -// +build linux +// +build linux,cgo package btrfs @@ -645,7 +645,15 @@ func (d *Driver) Get(id string, options graphdriver.MountOpts) (string, error) { if err != nil { return "", err } - if len(options.Options) > 0 { + switch len(options.Options) { + case 0: + case 1: + if options.Options[0] == "ro" { + // ignore "ro" option + break + } + fallthrough + default: return "", fmt.Errorf("btrfs driver does not support mount options") } diff --git a/vendor/github.com/containers/storage/drivers/btrfs/version.go b/vendor/github.com/containers/storage/drivers/btrfs/version.go index 73d90cdd7..edd8bdab8 100644 --- a/vendor/github.com/containers/storage/drivers/btrfs/version.go +++ b/vendor/github.com/containers/storage/drivers/btrfs/version.go @@ -1,4 +1,4 @@ -// +build linux,!btrfs_noversion +// +build linux,!btrfs_noversion,cgo package btrfs diff --git a/vendor/github.com/containers/storage/drivers/btrfs/version_none.go b/vendor/github.com/containers/storage/drivers/btrfs/version_none.go index f802fbc62..905e834e3 100644 --- a/vendor/github.com/containers/storage/drivers/btrfs/version_none.go +++ b/vendor/github.com/containers/storage/drivers/btrfs/version_none.go @@ -1,4 +1,4 @@ -// +build linux,btrfs_noversion +// +build !linux btrfs_noversion !cgo package btrfs diff --git a/vendor/github.com/containers/storage/drivers/devmapper/device_setup.go b/vendor/github.com/containers/storage/drivers/devmapper/device_setup.go index f63845252..7553a93eb 100644 --- a/vendor/github.com/containers/storage/drivers/devmapper/device_setup.go +++ b/vendor/github.com/containers/storage/drivers/devmapper/device_setup.go @@ -1,3 +1,5 @@ +// +build linux,cgo + package devmapper import ( diff --git a/vendor/github.com/containers/storage/drivers/devmapper/deviceset.go b/vendor/github.com/containers/storage/drivers/devmapper/deviceset.go index b6f22e90a..1ea6cfc36 100644 --- a/vendor/github.com/containers/storage/drivers/devmapper/deviceset.go +++ b/vendor/github.com/containers/storage/drivers/devmapper/deviceset.go @@ -1,4 +1,4 @@ -// +build linux +// +build linux,cgo package devmapper diff --git a/vendor/github.com/containers/storage/drivers/devmapper/devmapper_doc.go b/vendor/github.com/containers/storage/drivers/devmapper/devmapper_doc.go index 9ab3e4f86..418b9e610 100644 --- a/vendor/github.com/containers/storage/drivers/devmapper/devmapper_doc.go +++ b/vendor/github.com/containers/storage/drivers/devmapper/devmapper_doc.go @@ -1,3 +1,5 @@ +// +build linux,cgo + package devmapper // Definition of struct dm_task and sub structures (from lvm2) diff --git a/vendor/github.com/containers/storage/drivers/devmapper/driver.go b/vendor/github.com/containers/storage/drivers/devmapper/driver.go index f384a6242..3c044c12e 100644 --- a/vendor/github.com/containers/storage/drivers/devmapper/driver.go +++ b/vendor/github.com/containers/storage/drivers/devmapper/driver.go @@ -1,4 +1,4 @@ -// +build linux +// +build linux,cgo package devmapper diff --git a/vendor/github.com/containers/storage/drivers/devmapper/mount.go b/vendor/github.com/containers/storage/drivers/devmapper/mount.go index 1dc3262d2..41e73faf5 100644 --- a/vendor/github.com/containers/storage/drivers/devmapper/mount.go +++ b/vendor/github.com/containers/storage/drivers/devmapper/mount.go @@ -1,4 +1,4 @@ -// +build linux +// +build linux,cgo package devmapper diff --git a/vendor/github.com/containers/storage/drivers/fsdiff.go b/vendor/github.com/containers/storage/drivers/fsdiff.go index d34fdee3b..93743d177 100644 --- a/vendor/github.com/containers/storage/drivers/fsdiff.go +++ b/vendor/github.com/containers/storage/drivers/fsdiff.go @@ -82,6 +82,7 @@ func (gdw *NaiveDiffDriver) Diff(id string, idMappings *idtools.IDMappings, pare }), nil } + options.Options = append(options.Options, "ro") parentFs, err := driver.Get(parent, options) if err != nil { return nil, err diff --git a/vendor/github.com/containers/storage/drivers/overlay/overlay.go b/vendor/github.com/containers/storage/drivers/overlay/overlay.go index d70487ea7..032e5b28a 100644 --- a/vendor/github.com/containers/storage/drivers/overlay/overlay.go +++ b/vendor/github.com/containers/storage/drivers/overlay/overlay.go @@ -322,7 +322,7 @@ func parseOptions(options []string) (*overlayOptions, error) { return nil, fmt.Errorf("overlay: ostree_repo specified but support for ostree is missing") } o.ostreeRepo = val - case "overlay2.ignore_chown_errors", "overlay.ignore_chown_errors": + case ".ignore_chown_errors", "overlay2.ignore_chown_errors", "overlay.ignore_chown_errors": logrus.Debugf("overlay: ignore_chown_errors=%s", val) o.ignoreChownErrors, err = strconv.ParseBool(val) if err != nil { @@ -839,8 +839,17 @@ func (d *Driver) get(id string, disableShifting bool, options graphdriver.MountO if _, err := os.Stat(dir); err != nil { return "", err } + readWrite := true + // fuse-overlayfs doesn't support working without an upperdir. + if d.options.mountProgram == "" { + for _, o := range options.Options { + if o == "ro" { + readWrite = false + break + } + } + } - diffDir := path.Join(dir, "diff") lowers, err := ioutil.ReadFile(path.Join(dir, lowerFile)) if err != nil && !os.IsNotExist(err) { return "", err @@ -911,16 +920,32 @@ func (d *Driver) get(id string, disableShifting bool, options graphdriver.MountO // If the lowers list is still empty, use an empty lower so that we can still force an // SELinux context for the mount. + + // if we are doing a readOnly mount, and there is only one lower + // We should just return the lower directory, no reason to mount. + if !readWrite { + if len(absLowers) == 0 { + return path.Join(dir, "empty"), nil + } + if len(absLowers) == 1 { + return absLowers[0], nil + } + } if len(absLowers) == 0 { absLowers = append(absLowers, path.Join(dir, "empty")) relLowers = append(relLowers, path.Join(id, "empty")) } - // user namespace requires this to move a directory from lower to upper. rootUID, rootGID, err := idtools.GetRootUIDGID(d.uidMaps, d.gidMaps) if err != nil { return "", err } + diffDir := path.Join(dir, "diff") + if readWrite { + if err := idtools.MkdirAllAs(diffDir, 0755, rootUID, rootGID); err != nil && !os.IsExist(err) { + return "", err + } + } mergedDir := path.Join(dir, "merged") // Create the driver merged dir @@ -940,8 +965,12 @@ func (d *Driver) get(id string, disableShifting bool, options graphdriver.MountO } }() - workDir := path.Join(dir, "work") - opts := fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", strings.Join(absLowers, ":"), diffDir, workDir) + var opts string + if readWrite { + opts = fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", strings.Join(absLowers, ":"), diffDir, path.Join(dir, "work")) + } else { + opts = fmt.Sprintf("lowerdir=%s", strings.Join(absLowers, ":")) + } if len(options.Options) > 0 { opts = fmt.Sprintf("%s,%s", strings.Join(options.Options, ","), opts) } else if d.options.mountOptions != "" { @@ -979,7 +1008,12 @@ func (d *Driver) get(id string, disableShifting bool, options graphdriver.MountO } } else if len(mountData) > pageSize { //FIXME: We need to figure out to get this to work with additional stores - opts = fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", strings.Join(relLowers, ":"), path.Join(id, "diff"), path.Join(id, "work")) + if readWrite { + diffDir := path.Join(id, "diff") + opts = fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", strings.Join(relLowers, ":"), diffDir, path.Join(id, "work")) + } else { + opts = fmt.Sprintf("lowerdir=%s", strings.Join(absLowers, ":")) + } mountData = label.FormatMountLabel(opts, options.MountLabel) if len(mountData) > pageSize { return "", fmt.Errorf("cannot mount layer, mount label too large %d", len(mountData)) @@ -995,11 +1029,6 @@ func (d *Driver) get(id string, disableShifting bool, options graphdriver.MountO return "", fmt.Errorf("error creating overlay mount to %s: %v", mountTarget, err) } - // chown "workdir/work" to the remapped root UID/GID. Overlay fs inside a - if err := os.Chown(path.Join(workDir, "work"), rootUID, rootGID); err != nil { - return "", err - } - return mergedDir, nil } @@ -1018,7 +1047,7 @@ func (d *Driver) Put(id string) error { if _, err := ioutil.ReadFile(path.Join(dir, lowerFile)); err != nil && !os.IsNotExist(err) { return err } - if err := unix.Unmount(mountpoint, unix.MNT_DETACH); err != nil { + if err := unix.Unmount(mountpoint, unix.MNT_DETACH); err != nil && !os.IsNotExist(err) { logrus.Debugf("Failed to unmount %s overlay: %s - %v", id, mountpoint, err) } diff --git a/vendor/github.com/containers/storage/drivers/quota/projectquota.go b/vendor/github.com/containers/storage/drivers/quota/projectquota.go index 6ef35d8ad..d5aa0f891 100644 --- a/vendor/github.com/containers/storage/drivers/quota/projectquota.go +++ b/vendor/github.com/containers/storage/drivers/quota/projectquota.go @@ -1,4 +1,4 @@ -// +build linux,!exclude_disk_quota +// +build linux,!exclude_disk_quota,cgo // // projectquota.go - implements XFS project quota controls diff --git a/vendor/github.com/containers/storage/drivers/quota/projectquota_unsupported.go b/vendor/github.com/containers/storage/drivers/quota/projectquota_unsupported.go index b6db1e1d8..be6c75a58 100644 --- a/vendor/github.com/containers/storage/drivers/quota/projectquota_unsupported.go +++ b/vendor/github.com/containers/storage/drivers/quota/projectquota_unsupported.go @@ -1,4 +1,4 @@ -// +build linux,exclude_disk_quota +// +build !linux exclude_disk_quota !cgo package quota diff --git a/vendor/github.com/containers/storage/drivers/register/register_devicemapper.go b/vendor/github.com/containers/storage/drivers/register/register_devicemapper.go index 08ac984b0..cefe2e8c7 100644 --- a/vendor/github.com/containers/storage/drivers/register/register_devicemapper.go +++ b/vendor/github.com/containers/storage/drivers/register/register_devicemapper.go @@ -1,4 +1,4 @@ -// +build !exclude_graphdriver_devicemapper,linux +// +build !exclude_graphdriver_devicemapper,linux,cgo package register diff --git a/vendor/github.com/containers/storage/drivers/register/register_overlay.go b/vendor/github.com/containers/storage/drivers/register/register_overlay.go index 2d61219bb..30e3b4d74 100644 --- a/vendor/github.com/containers/storage/drivers/register/register_overlay.go +++ b/vendor/github.com/containers/storage/drivers/register/register_overlay.go @@ -1,4 +1,4 @@ -// +build !exclude_graphdriver_overlay,linux +// +build !exclude_graphdriver_overlay,linux,cgo package register diff --git a/vendor/github.com/containers/storage/drivers/vfs/driver.go b/vendor/github.com/containers/storage/drivers/vfs/driver.go index fe02a711e..6c02a45dc 100644 --- a/vendor/github.com/containers/storage/drivers/vfs/driver.go +++ b/vendor/github.com/containers/storage/drivers/vfs/driver.go @@ -58,7 +58,7 @@ func Init(home string, options graphdriver.Options) (graphdriver.Driver, error) d.ostreeRepo = val case "vfs.mountopt": return nil, fmt.Errorf("vfs driver does not support mount options") - case "vfs.ignore_chown_errors": + case ".ignore_chown_errors", "vfs.ignore_chown_errors": logrus.Debugf("vfs: ignore_chown_errors=%s", val) var err error d.ignoreChownErrors, err = strconv.ParseBool(val) @@ -226,7 +226,15 @@ func (d *Driver) Remove(id string) error { // Get returns the directory for the given id. func (d *Driver) Get(id string, options graphdriver.MountOpts) (_ string, retErr error) { dir := d.dir(id) - if len(options.Options) > 0 { + switch len(options.Options) { + case 0: + case 1: + if options.Options[0] == "ro" { + // ignore "ro" option + break + } + fallthrough + default: return "", fmt.Errorf("vfs driver does not support mount options") } if st, err := os.Stat(dir); err != nil { diff --git a/vendor/github.com/containers/storage/drivers/windows/windows.go b/vendor/github.com/containers/storage/drivers/windows/windows.go index 11f1c98b1..c1ab93e1d 100644 --- a/vendor/github.com/containers/storage/drivers/windows/windows.go +++ b/vendor/github.com/containers/storage/drivers/windows/windows.go @@ -372,7 +372,15 @@ func (d *Driver) Get(id string, options graphdriver.MountOpts) (string, error) { logrus.Debugf("WindowsGraphDriver Get() id %s mountLabel %s", id, options.MountLabel) var dir string - if len(options.Options) > 0 { + switch len(options.Options) { + case 0: + case 1: + if options.Options[0] == "ro" { + // ignore "ro" option + break + } + fallthrough + default: return "", fmt.Errorf("windows driver does not support mount options") } rID, err := d.resolveID(id) diff --git a/vendor/github.com/containers/storage/ffjson_deps.go b/vendor/github.com/containers/storage/ffjson_deps.go new file mode 100644 index 000000000..33e5340a5 --- /dev/null +++ b/vendor/github.com/containers/storage/ffjson_deps.go @@ -0,0 +1,10 @@ +package storage + +// NOTE: this is a hack to trick go modules into vendoring the below +// dependencies. Those are required during ffjson generation +// but do NOT end up in the final file. + +import ( + _ "github.com/pquerna/ffjson/inception" // nolint:typecheck + _ "github.com/pquerna/ffjson/shared" // nolint:typecheck +) diff --git a/vendor/github.com/containers/storage/go.mod b/vendor/github.com/containers/storage/go.mod index fed53569d..d8f943d30 100644 --- a/vendor/github.com/containers/storage/go.mod +++ b/vendor/github.com/containers/storage/go.mod @@ -1,7 +1,5 @@ module github.com/containers/storage -go 1.12 - require ( github.com/BurntSushi/toml v0.3.1 github.com/DataDog/zstd v1.4.0 // indirect diff --git a/vendor/github.com/containers/storage/layers.go b/vendor/github.com/containers/storage/layers.go index 4ef567210..d746ba061 100644 --- a/vendor/github.com/containers/storage/layers.go +++ b/vendor/github.com/containers/storage/layers.go @@ -1,7 +1,6 @@ package storage import ( - "archive/tar" "bytes" "encoding/json" "fmt" @@ -27,6 +26,7 @@ import ( digest "github.com/opencontainers/go-digest" "github.com/opencontainers/selinux/go-selinux/label" "github.com/pkg/errors" + "github.com/vbatts/tar-split/archive/tar" "github.com/vbatts/tar-split/tar/asm" "github.com/vbatts/tar-split/tar/storage" ) diff --git a/vendor/github.com/containers/storage/pkg/idtools/idtools.go b/vendor/github.com/containers/storage/pkg/idtools/idtools.go index a5c73d311..5105720ba 100644 --- a/vendor/github.com/containers/storage/pkg/idtools/idtools.go +++ b/vendor/github.com/containers/storage/pkg/idtools/idtools.go @@ -10,6 +10,7 @@ import ( "strings" "syscall" + "github.com/containers/storage/pkg/system" "github.com/pkg/errors" ) @@ -296,9 +297,19 @@ func checkChownErr(err error, name string, uid, gid int) error { } func SafeChown(name string, uid, gid int) error { + if stat, statErr := system.Stat(name); statErr == nil { + if stat.UID() == uint32(uid) && stat.GID() == uint32(gid) { + return nil + } + } return checkChownErr(os.Chown(name, uid, gid), name, uid, gid) } func SafeLchown(name string, uid, gid int) error { + if stat, statErr := system.Lstat(name); statErr == nil { + if stat.UID() == uint32(uid) && stat.GID() == uint32(gid) { + return nil + } + } return checkChownErr(os.Lchown(name, uid, gid), name, uid, gid) } diff --git a/vendor/github.com/containers/storage/pkg/loopback/attach_loopback.go b/vendor/github.com/containers/storage/pkg/loopback/attach_loopback.go index 34c80548d..be8680fbc 100644 --- a/vendor/github.com/containers/storage/pkg/loopback/attach_loopback.go +++ b/vendor/github.com/containers/storage/pkg/loopback/attach_loopback.go @@ -1,4 +1,4 @@ -// +build linux +// +build linux,cgo package loopback diff --git a/vendor/github.com/containers/storage/pkg/loopback/ioctl.go b/vendor/github.com/containers/storage/pkg/loopback/ioctl.go index 0714eb5f8..ea6841958 100644 --- a/vendor/github.com/containers/storage/pkg/loopback/ioctl.go +++ b/vendor/github.com/containers/storage/pkg/loopback/ioctl.go @@ -1,4 +1,4 @@ -// +build linux +// +build linux,cgo package loopback diff --git a/vendor/github.com/containers/storage/pkg/loopback/loop_wrapper.go b/vendor/github.com/containers/storage/pkg/loopback/loop_wrapper.go index e1100ce15..a50de7f07 100644 --- a/vendor/github.com/containers/storage/pkg/loopback/loop_wrapper.go +++ b/vendor/github.com/containers/storage/pkg/loopback/loop_wrapper.go @@ -1,4 +1,4 @@ -// +build linux +// +build linux,cgo package loopback diff --git a/vendor/github.com/containers/storage/pkg/loopback/loopback.go b/vendor/github.com/containers/storage/pkg/loopback/loopback.go index a8ec3c616..05d537dc8 100644 --- a/vendor/github.com/containers/storage/pkg/loopback/loopback.go +++ b/vendor/github.com/containers/storage/pkg/loopback/loopback.go @@ -1,4 +1,4 @@ -// +build linux +// +build linux,cgo package loopback diff --git a/vendor/github.com/containers/storage/pkg/loopback/loopback_unsupported.go b/vendor/github.com/containers/storage/pkg/loopback/loopback_unsupported.go new file mode 100644 index 000000000..460b30709 --- /dev/null +++ b/vendor/github.com/containers/storage/pkg/loopback/loopback_unsupported.go @@ -0,0 +1 @@ +package loopback diff --git a/vendor/github.com/containers/storage/pkg/ostree/no_ostree.go b/vendor/github.com/containers/storage/pkg/ostree/no_ostree.go index 0a8e7d679..bf83ccf25 100644 --- a/vendor/github.com/containers/storage/pkg/ostree/no_ostree.go +++ b/vendor/github.com/containers/storage/pkg/ostree/no_ostree.go @@ -1,4 +1,4 @@ -// +build !ostree +// +build !ostree !cgo package ostree diff --git a/vendor/github.com/containers/storage/pkg/ostree/ostree.go b/vendor/github.com/containers/storage/pkg/ostree/ostree.go index e9b57a0fb..7d324f2b2 100644 --- a/vendor/github.com/containers/storage/pkg/ostree/ostree.go +++ b/vendor/github.com/containers/storage/pkg/ostree/ostree.go @@ -1,4 +1,4 @@ -// +build ostree +// +build ostree,cgo package ostree diff --git a/vendor/github.com/containers/storage/pkg/tarlog/tarlogger.go b/vendor/github.com/containers/storage/pkg/tarlog/tarlogger.go index 8451de01e..98d3ee96a 100644 --- a/vendor/github.com/containers/storage/pkg/tarlog/tarlogger.go +++ b/vendor/github.com/containers/storage/pkg/tarlog/tarlogger.go @@ -1,47 +1,69 @@ package tarlog import ( - "archive/tar" "io" - "os" "sync" - "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "github.com/vbatts/tar-split/archive/tar" ) type tarLogger struct { - writer *os.File - wg sync.WaitGroup + writer *io.PipeWriter + closeMutex *sync.Mutex + stateMutex *sync.Mutex + closed bool } // NewLogger returns a writer that, when a tar archive is written to it, calls // `logger` for each file header it encounters in the archive. func NewLogger(logger func(*tar.Header)) (io.WriteCloser, error) { - reader, writer, err := os.Pipe() - if err != nil { - return nil, errors.Wrapf(err, "error creating pipe for tar logger") + reader, writer := io.Pipe() + t := &tarLogger{ + writer: writer, + closeMutex: new(sync.Mutex), + stateMutex: new(sync.Mutex), + closed: false, } - t := &tarLogger{writer: writer} tr := tar.NewReader(reader) - t.wg.Add(1) + tr.RawAccounting = true + t.closeMutex.Lock() go func() { hdr, err := tr.Next() for err == nil { logger(hdr) hdr, err = tr.Next() + + } + // Make sure to avoid writes after the reader has been closed. + t.stateMutex.Lock() + t.closed = true + if err := reader.Close(); err != nil { + logrus.Errorf("error closing tarlogger reader: %v", err) } - reader.Close() - t.wg.Done() + t.stateMutex.Unlock() + // Unblock the Close(). + t.closeMutex.Unlock() }() return t, nil } func (t *tarLogger) Write(b []byte) (int, error) { + t.stateMutex.Lock() + if t.closed { + // We cannot use os.Pipe() as this alters the tar's digest. Using + // io.Pipe() requires this workaround as it does not allow for writes + // after close. + t.stateMutex.Unlock() + return len(b), nil + } + t.stateMutex.Unlock() return t.writer.Write(b) } func (t *tarLogger) Close() error { err := t.writer.Close() - t.wg.Wait() + // Wait for the reader to finish. + t.closeMutex.Lock() return err } diff --git a/vendor/github.com/pquerna/ffjson/inception/decoder.go b/vendor/github.com/pquerna/ffjson/inception/decoder.go new file mode 100644 index 000000000..908347a32 --- /dev/null +++ b/vendor/github.com/pquerna/ffjson/inception/decoder.go @@ -0,0 +1,323 @@ +/** + * Copyright 2014 Paul Querna + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package ffjsoninception + +import ( + "fmt" + "reflect" + "strings" + + "github.com/pquerna/ffjson/shared" +) + +var validValues []string = []string{ + "FFTok_left_brace", + "FFTok_left_bracket", + "FFTok_integer", + "FFTok_double", + "FFTok_string", + "FFTok_bool", + "FFTok_null", +} + +func CreateUnmarshalJSON(ic *Inception, si *StructInfo) error { + out := "" + ic.OutputImports[`fflib "github.com/pquerna/ffjson/fflib/v1"`] = true + if len(si.Fields) > 0 { + ic.OutputImports[`"bytes"`] = true + } + ic.OutputImports[`"fmt"`] = true + + out += tplStr(decodeTpl["header"], header{ + IC: ic, + SI: si, + }) + + out += tplStr(decodeTpl["ujFunc"], ujFunc{ + SI: si, + IC: ic, + ValidValues: validValues, + ResetFields: ic.ResetFields, + }) + + ic.OutputFuncs = append(ic.OutputFuncs, out) + + return nil +} + +func handleField(ic *Inception, name string, typ reflect.Type, ptr bool, quoted bool) string { + return handleFieldAddr(ic, name, false, typ, ptr, quoted) +} + +func handleFieldAddr(ic *Inception, name string, takeAddr bool, typ reflect.Type, ptr bool, quoted bool) string { + out := fmt.Sprintf("/* handler: %s type=%v kind=%v quoted=%t*/\n", name, typ, typ.Kind(), quoted) + + umlx := typ.Implements(unmarshalFasterType) || typeInInception(ic, typ, shared.MustDecoder) + umlx = umlx || reflect.PtrTo(typ).Implements(unmarshalFasterType) + + umlstd := typ.Implements(unmarshalerType) || reflect.PtrTo(typ).Implements(unmarshalerType) + + out += tplStr(decodeTpl["handleUnmarshaler"], handleUnmarshaler{ + IC: ic, + Name: name, + Typ: typ, + Ptr: reflect.Ptr, + TakeAddr: takeAddr || ptr, + UnmarshalJSONFFLexer: umlx, + Unmarshaler: umlstd, + }) + + if umlx || umlstd { + return out + } + + // TODO(pquerna): generic handling of token type mismatching struct type + switch typ.Kind() { + case reflect.Int, + reflect.Int8, + reflect.Int16, + reflect.Int32, + reflect.Int64: + + allowed := buildTokens(quoted, "FFTok_string", "FFTok_integer", "FFTok_null") + out += getAllowTokens(typ.Name(), allowed...) + + out += getNumberHandler(ic, name, takeAddr || ptr, typ, "ParseInt") + + case reflect.Uint, + reflect.Uint8, + reflect.Uint16, + reflect.Uint32, + reflect.Uint64: + + allowed := buildTokens(quoted, "FFTok_string", "FFTok_integer", "FFTok_null") + out += getAllowTokens(typ.Name(), allowed...) + + out += getNumberHandler(ic, name, takeAddr || ptr, typ, "ParseUint") + + case reflect.Float32, + reflect.Float64: + + allowed := buildTokens(quoted, "FFTok_string", "FFTok_double", "FFTok_integer", "FFTok_null") + out += getAllowTokens(typ.Name(), allowed...) + + out += getNumberHandler(ic, name, takeAddr || ptr, typ, "ParseFloat") + + case reflect.Bool: + ic.OutputImports[`"bytes"`] = true + ic.OutputImports[`"errors"`] = true + + allowed := buildTokens(quoted, "FFTok_string", "FFTok_bool", "FFTok_null") + out += getAllowTokens(typ.Name(), allowed...) + + out += tplStr(decodeTpl["handleBool"], handleBool{ + Name: name, + Typ: typ, + TakeAddr: takeAddr || ptr, + }) + + case reflect.Ptr: + out += tplStr(decodeTpl["handlePtr"], handlePtr{ + IC: ic, + Name: name, + Typ: typ, + Quoted: quoted, + }) + + case reflect.Array, + reflect.Slice: + out += getArrayHandler(ic, name, typ, ptr) + + case reflect.String: + // Is it a json.Number? + if typ.PkgPath() == "encoding/json" && typ.Name() == "Number" { + // Fall back to json package to rely on the valid number check. + // See: https://github.com/golang/go/blob/f05c3aa24d815cd3869153750c9875e35fc48a6e/src/encoding/json/decode.go#L897 + ic.OutputImports[`"encoding/json"`] = true + out += tplStr(decodeTpl["handleFallback"], handleFallback{ + Name: name, + Typ: typ, + Kind: typ.Kind(), + }) + } else { + out += tplStr(decodeTpl["handleString"], handleString{ + IC: ic, + Name: name, + Typ: typ, + TakeAddr: takeAddr || ptr, + Quoted: quoted, + }) + } + case reflect.Interface: + ic.OutputImports[`"encoding/json"`] = true + out += tplStr(decodeTpl["handleFallback"], handleFallback{ + Name: name, + Typ: typ, + Kind: typ.Kind(), + }) + case reflect.Map: + out += tplStr(decodeTpl["handleObject"], handleObject{ + IC: ic, + Name: name, + Typ: typ, + Ptr: reflect.Ptr, + TakeAddr: takeAddr || ptr, + }) + default: + ic.OutputImports[`"encoding/json"`] = true + out += tplStr(decodeTpl["handleFallback"], handleFallback{ + Name: name, + Typ: typ, + Kind: typ.Kind(), + }) + } + + return out +} + +func getArrayHandler(ic *Inception, name string, typ reflect.Type, ptr bool) string { + if typ.Kind() == reflect.Slice && typ.Elem().Kind() == reflect.Uint8 { + ic.OutputImports[`"encoding/base64"`] = true + useReflectToSet := false + if typ.Elem().Name() != "byte" { + ic.OutputImports[`"reflect"`] = true + useReflectToSet = true + } + + return tplStr(decodeTpl["handleByteSlice"], handleArray{ + IC: ic, + Name: name, + Typ: typ, + Ptr: reflect.Ptr, + UseReflectToSet: useReflectToSet, + }) + } + + if typ.Elem().Kind() == reflect.Struct && typ.Elem().Name() != "" { + goto sliceOrArray + } + + if (typ.Elem().Kind() == reflect.Struct || typ.Elem().Kind() == reflect.Map) || + typ.Elem().Kind() == reflect.Array || typ.Elem().Kind() == reflect.Slice && + typ.Elem().Name() == "" { + ic.OutputImports[`"encoding/json"`] = true + + return tplStr(decodeTpl["handleFallback"], handleFallback{ + Name: name, + Typ: typ, + Kind: typ.Kind(), + }) + } + +sliceOrArray: + + if typ.Kind() == reflect.Array { + return tplStr(decodeTpl["handleArray"], handleArray{ + IC: ic, + Name: name, + Typ: typ, + IsPtr: ptr, + Ptr: reflect.Ptr, + }) + } + + return tplStr(decodeTpl["handleSlice"], handleArray{ + IC: ic, + Name: name, + Typ: typ, + IsPtr: ptr, + Ptr: reflect.Ptr, + }) +} + +func getAllowTokens(name string, tokens ...string) string { + return tplStr(decodeTpl["allowTokens"], allowTokens{ + Name: name, + Tokens: tokens, + }) +} + +func getNumberHandler(ic *Inception, name string, takeAddr bool, typ reflect.Type, parsefunc string) string { + return tplStr(decodeTpl["handlerNumeric"], handlerNumeric{ + IC: ic, + Name: name, + ParseFunc: parsefunc, + TakeAddr: takeAddr, + Typ: typ, + }) +} + +func getNumberSize(typ reflect.Type) string { + return fmt.Sprintf("%d", typ.Bits()) +} + +func getType(ic *Inception, name string, typ reflect.Type) string { + s := typ.Name() + + if typ.PkgPath() != "" && typ.PkgPath() != ic.PackagePath { + path := removeVendor(typ.PkgPath()) + ic.OutputImports[`"`+path+`"`] = true + s = typ.String() + } + + if s == "" { + return typ.String() + } + + return s +} + +// removeVendor removes everything before and including a '/vendor/' +// substring in the package path. +// This is needed becuase that full path can't be used in the +// import statement. +func removeVendor(path string) string { + i := strings.Index(path, "/vendor/") + if i == -1 { + return path + } + return path[i+8:] +} + +func buildTokens(containsOptional bool, optional string, required ...string) []string { + if containsOptional { + return append(required, optional) + } + + return required +} + +func unquoteField(quoted bool) string { + // The outer quote of a string is already stripped out by + // the lexer. We need to check if the inner string is also + // quoted. If so, we will decode it as json string. If decoding + // fails, we will use the original string + if quoted { + return ` + unquoted, ok := fflib.UnquoteBytes(outBuf) + if ok { + outBuf = unquoted + } + ` + } + return "" +} + +func getTmpVarFor(name string) string { + return "tmp" + strings.Replace(strings.Title(name), ".", "", -1) +} diff --git a/vendor/github.com/pquerna/ffjson/inception/decoder_tpl.go b/vendor/github.com/pquerna/ffjson/inception/decoder_tpl.go new file mode 100644 index 000000000..098506122 --- /dev/null +++ b/vendor/github.com/pquerna/ffjson/inception/decoder_tpl.go @@ -0,0 +1,773 @@ +/** + * Copyright 2014 Paul Querna + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package ffjsoninception + +import ( + "reflect" + "strconv" + "text/template" +) + +var decodeTpl map[string]*template.Template + +func init() { + decodeTpl = make(map[string]*template.Template) + + funcs := map[string]string{ + "handlerNumeric": handlerNumericTxt, + "allowTokens": allowTokensTxt, + "handleFallback": handleFallbackTxt, + "handleString": handleStringTxt, + "handleObject": handleObjectTxt, + "handleArray": handleArrayTxt, + "handleSlice": handleSliceTxt, + "handleByteSlice": handleByteSliceTxt, + "handleBool": handleBoolTxt, + "handlePtr": handlePtrTxt, + "header": headerTxt, + "ujFunc": ujFuncTxt, + "handleUnmarshaler": handleUnmarshalerTxt, + } + + tplFuncs := template.FuncMap{ + "getAllowTokens": getAllowTokens, + "getNumberSize": getNumberSize, + "getType": getType, + "handleField": handleField, + "handleFieldAddr": handleFieldAddr, + "unquoteField": unquoteField, + "getTmpVarFor": getTmpVarFor, + } + + for k, v := range funcs { + decodeTpl[k] = template.Must(template.New(k).Funcs(tplFuncs).Parse(v)) + } +} + +type handlerNumeric struct { + IC *Inception + Name string + ParseFunc string + Typ reflect.Type + TakeAddr bool +} + +var handlerNumericTxt = ` +{ + {{$ic := .IC}} + + if tok == fflib.FFTok_null { + {{if eq .TakeAddr true}} + {{.Name}} = nil + {{end}} + } else { + {{if eq .ParseFunc "ParseFloat" }} + tval, err := fflib.{{ .ParseFunc}}(fs.Output.Bytes(), {{getNumberSize .Typ}}) + {{else}} + tval, err := fflib.{{ .ParseFunc}}(fs.Output.Bytes(), 10, {{getNumberSize .Typ}}) + {{end}} + + if err != nil { + return fs.WrapErr(err) + } + {{if eq .TakeAddr true}} + ttypval := {{getType $ic .Name .Typ}}(tval) + {{.Name}} = &ttypval + {{else}} + {{.Name}} = {{getType $ic .Name .Typ}}(tval) + {{end}} + } +} +` + +type allowTokens struct { + Name string + Tokens []string +} + +var allowTokensTxt = ` +{ + if {{range $index, $element := .Tokens}}{{if ne $index 0 }}&&{{end}} tok != fflib.{{$element}}{{end}} { + return fs.WrapErr(fmt.Errorf("cannot unmarshal %s into Go value for {{.Name}}", tok)) + } +} +` + +type handleFallback struct { + Name string + Typ reflect.Type + Kind reflect.Kind +} + +var handleFallbackTxt = ` +{ + /* Falling back. type={{printf "%v" .Typ}} kind={{printf "%v" .Kind}} */ + tbuf, err := fs.CaptureField(tok) + if err != nil { + return fs.WrapErr(err) + } + + err = json.Unmarshal(tbuf, &{{.Name}}) + if err != nil { + return fs.WrapErr(err) + } +} +` + +type handleString struct { + IC *Inception + Name string + Typ reflect.Type + TakeAddr bool + Quoted bool +} + +var handleStringTxt = ` +{ + {{$ic := .IC}} + + {{getAllowTokens .Typ.Name "FFTok_string" "FFTok_null"}} + if tok == fflib.FFTok_null { + {{if eq .TakeAddr true}} + {{.Name}} = nil + {{end}} + } else { + {{if eq .TakeAddr true}} + var tval {{getType $ic .Name .Typ}} + outBuf := fs.Output.Bytes() + {{unquoteField .Quoted}} + tval = {{getType $ic .Name .Typ}}(string(outBuf)) + {{.Name}} = &tval + {{else}} + outBuf := fs.Output.Bytes() + {{unquoteField .Quoted}} + {{.Name}} = {{getType $ic .Name .Typ}}(string(outBuf)) + {{end}} + } +} +` + +type handleObject struct { + IC *Inception + Name string + Typ reflect.Type + Ptr reflect.Kind + TakeAddr bool +} + +var handleObjectTxt = ` +{ + {{$ic := .IC}} + {{getAllowTokens .Typ.Name "FFTok_left_bracket" "FFTok_null"}} + if tok == fflib.FFTok_null { + {{.Name}} = nil + } else { + + {{if eq .TakeAddr true}} + {{if eq .Typ.Elem.Kind .Ptr }} + {{if eq .Typ.Key.Kind .Ptr }} + var tval = make(map[*{{getType $ic .Name .Typ.Key.Elem}}]*{{getType $ic .Name .Typ.Elem.Elem}}, 0) + {{else}} + var tval = make(map[{{getType $ic .Name .Typ.Key}}]*{{getType $ic .Name .Typ.Elem.Elem}}, 0) + {{end}} + {{else}} + {{if eq .Typ.Key.Kind .Ptr }} + var tval = make(map[*{{getType $ic .Name .Typ.Key.Elem}}]{{getType $ic .Name .Typ.Elem}}, 0) + {{else}} + var tval = make(map[{{getType $ic .Name .Typ.Key}}]{{getType $ic .Name .Typ.Elem}}, 0) + {{end}} + {{end}} + {{else}} + {{if eq .Typ.Elem.Kind .Ptr }} + {{if eq .Typ.Key.Kind .Ptr }} + {{.Name}} = make(map[*{{getType $ic .Name .Typ.Key.Elem}}]*{{getType $ic .Name .Typ.Elem.Elem}}, 0) + {{else}} + {{.Name}} = make(map[{{getType $ic .Name .Typ.Key}}]*{{getType $ic .Name .Typ.Elem.Elem}}, 0) + {{end}} + {{else}} + {{if eq .Typ.Key.Kind .Ptr }} + {{.Name}} = make(map[*{{getType $ic .Name .Typ.Key.Elem}}]{{getType $ic .Name .Typ.Elem}}, 0) + {{else}} + {{.Name}} = make(map[{{getType $ic .Name .Typ.Key}}]{{getType $ic .Name .Typ.Elem}}, 0) + {{end}} + {{end}} + {{end}} + + wantVal := true + + for { + {{$keyPtr := false}} + {{if eq .Typ.Key.Kind .Ptr }} + {{$keyPtr := true}} + var k *{{getType $ic .Name .Typ.Key.Elem}} + {{else}} + var k {{getType $ic .Name .Typ.Key}} + {{end}} + + {{$valPtr := false}} + {{$tmpVar := getTmpVarFor .Name}} + {{if eq .Typ.Elem.Kind .Ptr }} + {{$valPtr := true}} + var {{$tmpVar}} *{{getType $ic .Name .Typ.Elem.Elem}} + {{else}} + var {{$tmpVar}} {{getType $ic .Name .Typ.Elem}} + {{end}} + + tok = fs.Scan() + if tok == fflib.FFTok_error { + goto tokerror + } + if tok == fflib.FFTok_right_bracket { + break + } + + if tok == fflib.FFTok_comma { + if wantVal == true { + // TODO(pquerna): this isn't an ideal error message, this handles + // things like [,,,] as an array value. + return fs.WrapErr(fmt.Errorf("wanted value token, but got token: %v", tok)) + } + continue + } else { + wantVal = true + } + + {{handleField .IC "k" .Typ.Key $keyPtr false}} + + // Expect ':' after key + tok = fs.Scan() + if tok != fflib.FFTok_colon { + return fs.WrapErr(fmt.Errorf("wanted colon token, but got token: %v", tok)) + } + + tok = fs.Scan() + {{handleField .IC $tmpVar .Typ.Elem $valPtr false}} + + {{if eq .TakeAddr true}} + tval[k] = {{$tmpVar}} + {{else}} + {{.Name}}[k] = {{$tmpVar}} + {{end}} + wantVal = false + } + + {{if eq .TakeAddr true}} + {{.Name}} = &tval + {{end}} + } +} +` + +type handleArray struct { + IC *Inception + Name string + Typ reflect.Type + Ptr reflect.Kind + UseReflectToSet bool + IsPtr bool +} + +var handleArrayTxt = ` +{ + {{$ic := .IC}} + {{getAllowTokens .Typ.Name "FFTok_left_brace" "FFTok_null"}} + {{if eq .Typ.Elem.Kind .Ptr}} + {{.Name}} = [{{.Typ.Len}}]*{{getType $ic .Name .Typ.Elem.Elem}}{} + {{else}} + {{.Name}} = [{{.Typ.Len}}]{{getType $ic .Name .Typ.Elem}}{} + {{end}} + if tok != fflib.FFTok_null { + wantVal := true + + idx := 0 + for { + {{$ptr := false}} + {{$tmpVar := getTmpVarFor .Name}} + {{if eq .Typ.Elem.Kind .Ptr }} + {{$ptr := true}} + var {{$tmpVar}} *{{getType $ic .Name .Typ.Elem.Elem}} + {{else}} + var {{$tmpVar}} {{getType $ic .Name .Typ.Elem}} + {{end}} + + tok = fs.Scan() + if tok == fflib.FFTok_error { + goto tokerror + } + if tok == fflib.FFTok_right_brace { + break + } + + if tok == fflib.FFTok_comma { + if wantVal == true { + // TODO(pquerna): this isn't an ideal error message, this handles + // things like [,,,] as an array value. + return fs.WrapErr(fmt.Errorf("wanted value token, but got token: %v", tok)) + } + continue + } else { + wantVal = true + } + + {{handleField .IC $tmpVar .Typ.Elem $ptr false}} + + // Standard json.Unmarshal ignores elements out of array bounds, + // that what we do as well. + if idx < {{.Typ.Len}} { + {{.Name}}[idx] = {{$tmpVar}} + idx++ + } + + wantVal = false + } + } +} +` + +var handleSliceTxt = ` +{ + {{$ic := .IC}} + {{getAllowTokens .Typ.Name "FFTok_left_brace" "FFTok_null"}} + if tok == fflib.FFTok_null { + {{.Name}} = nil + } else { + {{if eq .Typ.Elem.Kind .Ptr }} + {{if eq .IsPtr true}} + {{.Name}} = &[]*{{getType $ic .Name .Typ.Elem.Elem}}{} + {{else}} + {{.Name}} = []*{{getType $ic .Name .Typ.Elem.Elem}}{} + {{end}} + {{else}} + {{if eq .IsPtr true}} + {{.Name}} = &[]{{getType $ic .Name .Typ.Elem}}{} + {{else}} + {{.Name}} = []{{getType $ic .Name .Typ.Elem}}{} + {{end}} + {{end}} + + wantVal := true + + for { + {{$ptr := false}} + {{$tmpVar := getTmpVarFor .Name}} + {{if eq .Typ.Elem.Kind .Ptr }} + {{$ptr := true}} + var {{$tmpVar}} *{{getType $ic .Name .Typ.Elem.Elem}} + {{else}} + var {{$tmpVar}} {{getType $ic .Name .Typ.Elem}} + {{end}} + + tok = fs.Scan() + if tok == fflib.FFTok_error { + goto tokerror + } + if tok == fflib.FFTok_right_brace { + break + } + + if tok == fflib.FFTok_comma { + if wantVal == true { + // TODO(pquerna): this isn't an ideal error message, this handles + // things like [,,,] as an array value. + return fs.WrapErr(fmt.Errorf("wanted value token, but got token: %v", tok)) + } + continue + } else { + wantVal = true + } + + {{handleField .IC $tmpVar .Typ.Elem $ptr false}} + {{if eq .IsPtr true}} + *{{.Name}} = append(*{{.Name}}, {{$tmpVar}}) + {{else}} + {{.Name}} = append({{.Name}}, {{$tmpVar}}) + {{end}} + wantVal = false + } + } +} +` + +var handleByteSliceTxt = ` +{ + {{getAllowTokens .Typ.Name "FFTok_string" "FFTok_null"}} + if tok == fflib.FFTok_null { + {{.Name}} = nil + } else { + b := make([]byte, base64.StdEncoding.DecodedLen(fs.Output.Len())) + n, err := base64.StdEncoding.Decode(b, fs.Output.Bytes()) + if err != nil { + return fs.WrapErr(err) + } + {{if eq .UseReflectToSet true}} + v := reflect.ValueOf(&{{.Name}}).Elem() + v.SetBytes(b[0:n]) + {{else}} + {{.Name}} = append([]byte(), b[0:n]...) + {{end}} + } +} +` + +type handleBool struct { + Name string + Typ reflect.Type + TakeAddr bool +} + +var handleBoolTxt = ` +{ + if tok == fflib.FFTok_null { + {{if eq .TakeAddr true}} + {{.Name}} = nil + {{end}} + } else { + tmpb := fs.Output.Bytes() + + {{if eq .TakeAddr true}} + var tval bool + {{end}} + + if bytes.Compare([]byte{'t', 'r', 'u', 'e'}, tmpb) == 0 { + {{if eq .TakeAddr true}} + tval = true + {{else}} + {{.Name}} = true + {{end}} + } else if bytes.Compare([]byte{'f', 'a', 'l', 's', 'e'}, tmpb) == 0 { + {{if eq .TakeAddr true}} + tval = false + {{else}} + {{.Name}} = false + {{end}} + } else { + err = errors.New("unexpected bytes for true/false value") + return fs.WrapErr(err) + } + + {{if eq .TakeAddr true}} + {{.Name}} = &tval + {{end}} + } +} +` + +type handlePtr struct { + IC *Inception + Name string + Typ reflect.Type + Quoted bool +} + +var handlePtrTxt = ` +{ + {{$ic := .IC}} + + if tok == fflib.FFTok_null { + {{.Name}} = nil + } else { + if {{.Name}} == nil { + {{.Name}} = new({{getType $ic .Typ.Elem.Name .Typ.Elem}}) + } + + {{handleFieldAddr .IC .Name true .Typ.Elem false .Quoted}} + } +} +` + +type header struct { + IC *Inception + SI *StructInfo +} + +var headerTxt = ` +const ( + ffjt{{.SI.Name}}base = iota + ffjt{{.SI.Name}}nosuchkey + {{with $si := .SI}} + {{range $index, $field := $si.Fields}} + {{if ne $field.JsonName "-"}} + ffjt{{$si.Name}}{{$field.Name}} + {{end}} + {{end}} + {{end}} +) + +{{with $si := .SI}} + {{range $index, $field := $si.Fields}} + {{if ne $field.JsonName "-"}} +var ffjKey{{$si.Name}}{{$field.Name}} = []byte({{$field.JsonName}}) + {{end}} + {{end}} +{{end}} + +` + +type ujFunc struct { + IC *Inception + SI *StructInfo + ValidValues []string + ResetFields bool +} + +var ujFuncTxt = ` +{{$si := .SI}} +{{$ic := .IC}} + +// UnmarshalJSON umarshall json - template of ffjson +func (j *{{.SI.Name}}) UnmarshalJSON(input []byte) error { + fs := fflib.NewFFLexer(input) + return j.UnmarshalJSONFFLexer(fs, fflib.FFParse_map_start) +} + +// UnmarshalJSONFFLexer fast json unmarshall - template ffjson +func (j *{{.SI.Name}}) UnmarshalJSONFFLexer(fs *fflib.FFLexer, state fflib.FFParseState) error { + var err error + currentKey := ffjt{{.SI.Name}}base + _ = currentKey + tok := fflib.FFTok_init + wantedTok := fflib.FFTok_init + + {{if eq .ResetFields true}} + {{range $index, $field := $si.Fields}} + var ffjSet{{$si.Name}}{{$field.Name}} = false + {{end}} + {{end}} + +mainparse: + for { + tok = fs.Scan() + // println(fmt.Sprintf("debug: tok: %v state: %v", tok, state)) + if tok == fflib.FFTok_error { + goto tokerror + } + + switch state { + + case fflib.FFParse_map_start: + if tok != fflib.FFTok_left_bracket { + wantedTok = fflib.FFTok_left_bracket + goto wrongtokenerror + } + state = fflib.FFParse_want_key + continue + + case fflib.FFParse_after_value: + if tok == fflib.FFTok_comma { + state = fflib.FFParse_want_key + } else if tok == fflib.FFTok_right_bracket { + goto done + } else { + wantedTok = fflib.FFTok_comma + goto wrongtokenerror + } + + case fflib.FFParse_want_key: + // json {} ended. goto exit. woo. + if tok == fflib.FFTok_right_bracket { + goto done + } + if tok != fflib.FFTok_string { + wantedTok = fflib.FFTok_string + goto wrongtokenerror + } + + kn := fs.Output.Bytes() + if len(kn) <= 0 { + // "" case. hrm. + currentKey = ffjt{{.SI.Name}}nosuchkey + state = fflib.FFParse_want_colon + goto mainparse + } else { + switch kn[0] { + {{range $byte, $fields := $si.FieldsByFirstByte}} + case '{{$byte}}': + {{range $index, $field := $fields}} + {{if ne $index 0 }}} else if {{else}}if {{end}} bytes.Equal(ffjKey{{$si.Name}}{{$field.Name}}, kn) { + currentKey = ffjt{{$si.Name}}{{$field.Name}} + state = fflib.FFParse_want_colon + goto mainparse + {{end}} } + {{end}} + } + {{range $index, $field := $si.ReverseFields}} + if {{$field.FoldFuncName}}(ffjKey{{$si.Name}}{{$field.Name}}, kn) { + currentKey = ffjt{{$si.Name}}{{$field.Name}} + state = fflib.FFParse_want_colon + goto mainparse + } + {{end}} + currentKey = ffjt{{.SI.Name}}nosuchkey + state = fflib.FFParse_want_colon + goto mainparse + } + + case fflib.FFParse_want_colon: + if tok != fflib.FFTok_colon { + wantedTok = fflib.FFTok_colon + goto wrongtokenerror + } + state = fflib.FFParse_want_value + continue + case fflib.FFParse_want_value: + + if {{range $index, $v := .ValidValues}}{{if ne $index 0 }}||{{end}}tok == fflib.{{$v}}{{end}} { + switch currentKey { + {{range $index, $field := $si.Fields}} + case ffjt{{$si.Name}}{{$field.Name}}: + goto handle_{{$field.Name}} + {{end}} + case ffjt{{$si.Name}}nosuchkey: + err = fs.SkipField(tok) + if err != nil { + return fs.WrapErr(err) + } + state = fflib.FFParse_after_value + goto mainparse + } + } else { + goto wantedvalue + } + } + } +{{range $index, $field := $si.Fields}} +handle_{{$field.Name}}: + {{with $fieldName := $field.Name | printf "j.%s"}} + {{handleField $ic $fieldName $field.Typ $field.Pointer $field.ForceString}} + {{if eq $.ResetFields true}} + ffjSet{{$si.Name}}{{$field.Name}} = true + {{end}} + state = fflib.FFParse_after_value + goto mainparse + {{end}} +{{end}} + +wantedvalue: + return fs.WrapErr(fmt.Errorf("wanted value token, but got token: %v", tok)) +wrongtokenerror: + return fs.WrapErr(fmt.Errorf("ffjson: wanted token: %v, but got token: %v output=%s", wantedTok, tok, fs.Output.String())) +tokerror: + if fs.BigError != nil { + return fs.WrapErr(fs.BigError) + } + err = fs.Error.ToError() + if err != nil { + return fs.WrapErr(err) + } + panic("ffjson-generated: unreachable, please report bug.") +done: +{{if eq .ResetFields true}} +{{range $index, $field := $si.Fields}} + if !ffjSet{{$si.Name}}{{$field.Name}} { + {{with $fieldName := $field.Name | printf "j.%s"}} + {{if eq $field.Pointer true}} + {{$fieldName}} = nil + {{else if eq $field.Typ.Kind ` + strconv.FormatUint(uint64(reflect.Interface), 10) + `}} + {{$fieldName}} = nil + {{else if eq $field.Typ.Kind ` + strconv.FormatUint(uint64(reflect.Slice), 10) + `}} + {{$fieldName}} = nil + {{else if eq $field.Typ.Kind ` + strconv.FormatUint(uint64(reflect.Array), 10) + `}} + {{$fieldName}} = [{{$field.Typ.Len}}]{{getType $ic $fieldName $field.Typ.Elem}}{} + {{else if eq $field.Typ.Kind ` + strconv.FormatUint(uint64(reflect.Map), 10) + `}} + {{$fieldName}} = nil + {{else if eq $field.Typ.Kind ` + strconv.FormatUint(uint64(reflect.Bool), 10) + `}} + {{$fieldName}} = false + {{else if eq $field.Typ.Kind ` + strconv.FormatUint(uint64(reflect.String), 10) + `}} + {{$fieldName}} = "" + {{else if eq $field.Typ.Kind ` + strconv.FormatUint(uint64(reflect.Struct), 10) + `}} + {{$fieldName}} = {{getType $ic $fieldName $field.Typ}}{} + {{else}} + {{$fieldName}} = {{getType $ic $fieldName $field.Typ}}(0) + {{end}} + {{end}} + } +{{end}} +{{end}} + return nil +} +` + +type handleUnmarshaler struct { + IC *Inception + Name string + Typ reflect.Type + Ptr reflect.Kind + TakeAddr bool + UnmarshalJSONFFLexer bool + Unmarshaler bool +} + +var handleUnmarshalerTxt = ` + {{$ic := .IC}} + + {{if eq .UnmarshalJSONFFLexer true}} + { + if tok == fflib.FFTok_null { + {{if eq .Typ.Kind .Ptr }} + {{.Name}} = nil + {{end}} + {{if eq .TakeAddr true }} + {{.Name}} = nil + {{end}} + } else { + {{if eq .Typ.Kind .Ptr }} + if {{.Name}} == nil { + {{.Name}} = new({{getType $ic .Typ.Elem.Name .Typ.Elem}}) + } + {{end}} + {{if eq .TakeAddr true }} + if {{.Name}} == nil { + {{.Name}} = new({{getType $ic .Typ.Name .Typ}}) + } + {{end}} + err = {{.Name}}.UnmarshalJSONFFLexer(fs, fflib.FFParse_want_key) + if err != nil { + return err + } + } + state = fflib.FFParse_after_value + } + {{else}} + {{if eq .Unmarshaler true}} + { + if tok == fflib.FFTok_null { + {{if eq .TakeAddr true }} + {{.Name}} = nil + {{end}} + } else { + + tbuf, err := fs.CaptureField(tok) + if err != nil { + return fs.WrapErr(err) + } + + {{if eq .TakeAddr true }} + if {{.Name}} == nil { + {{.Name}} = new({{getType $ic .Typ.Name .Typ}}) + } + {{end}} + err = {{.Name}}.UnmarshalJSON(tbuf) + if err != nil { + return fs.WrapErr(err) + } + } + state = fflib.FFParse_after_value + } + {{end}} + {{end}} +` diff --git a/vendor/github.com/pquerna/ffjson/inception/encoder.go b/vendor/github.com/pquerna/ffjson/inception/encoder.go new file mode 100644 index 000000000..3e37a2814 --- /dev/null +++ b/vendor/github.com/pquerna/ffjson/inception/encoder.go @@ -0,0 +1,544 @@ +/** + * Copyright 2014 Paul Querna + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package ffjsoninception + +import ( + "fmt" + "reflect" + + "github.com/pquerna/ffjson/shared" +) + +func typeInInception(ic *Inception, typ reflect.Type, f shared.Feature) bool { + for _, v := range ic.objs { + if v.Typ == typ { + return v.Options.HasFeature(f) + } + if typ.Kind() == reflect.Ptr { + if v.Typ == typ.Elem() { + return v.Options.HasFeature(f) + } + } + } + + return false +} + +func getOmitEmpty(ic *Inception, sf *StructField) string { + ptname := "j." + sf.Name + if sf.Pointer { + ptname = "*" + ptname + return "if true {\n" + } + switch sf.Typ.Kind() { + + case reflect.Array, reflect.Map, reflect.Slice, reflect.String: + return "if len(" + ptname + ") != 0 {" + "\n" + + case reflect.Int, + reflect.Int8, + reflect.Int16, + reflect.Int32, + reflect.Int64, + reflect.Uint, + reflect.Uint8, + reflect.Uint16, + reflect.Uint32, + reflect.Uint64, + reflect.Uintptr, + reflect.Float32, + reflect.Float64: + return "if " + ptname + " != 0 {" + "\n" + + case reflect.Bool: + return "if " + ptname + " != false {" + "\n" + + case reflect.Interface, reflect.Ptr: + return "if " + ptname + " != nil {" + "\n" + + default: + // TODO(pquerna): fix types + return "if true {" + "\n" + } +} + +func getMapValue(ic *Inception, name string, typ reflect.Type, ptr bool, forceString bool) string { + var out = "" + + if typ.Key().Kind() != reflect.String { + out += fmt.Sprintf("/* Falling back. type=%v kind=%v */\n", typ, typ.Kind()) + out += ic.q.Flush() + out += "err = buf.Encode(" + name + ")" + "\n" + out += "if err != nil {" + "\n" + out += " return err" + "\n" + out += "}" + "\n" + return out + } + + var elemKind reflect.Kind + elemKind = typ.Elem().Kind() + + switch elemKind { + case reflect.String, + reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr, + reflect.Float32, + reflect.Float64, + reflect.Bool: + + ic.OutputImports[`fflib "github.com/pquerna/ffjson/fflib/v1"`] = true + + out += "if " + name + " == nil {" + "\n" + ic.q.Write("null") + out += ic.q.GetQueued() + ic.q.DeleteLast() + out += "} else {" + "\n" + out += ic.q.WriteFlush("{ ") + out += " for key, value := range " + name + " {" + "\n" + out += " fflib.WriteJsonString(buf, key)" + "\n" + out += " buf.WriteString(`:`)" + "\n" + out += getGetInnerValue(ic, "value", typ.Elem(), false, forceString) + out += " buf.WriteByte(',')" + "\n" + out += " }" + "\n" + out += "buf.Rewind(1)" + "\n" + out += ic.q.WriteFlush("}") + out += "}" + "\n" + + default: + out += ic.q.Flush() + out += fmt.Sprintf("/* Falling back. type=%v kind=%v */\n", typ, typ.Kind()) + out += "err = buf.Encode(" + name + ")" + "\n" + out += "if err != nil {" + "\n" + out += " return err" + "\n" + out += "}" + "\n" + } + return out +} + +func getGetInnerValue(ic *Inception, name string, typ reflect.Type, ptr bool, forceString bool) string { + var out = "" + + // Flush if not bool or maps + if typ.Kind() != reflect.Bool && typ.Kind() != reflect.Map && typ.Kind() != reflect.Struct { + out += ic.q.Flush() + } + + if typ.Implements(marshalerFasterType) || + reflect.PtrTo(typ).Implements(marshalerFasterType) || + typeInInception(ic, typ, shared.MustEncoder) || + typ.Implements(marshalerType) || + reflect.PtrTo(typ).Implements(marshalerType) { + + out += ic.q.Flush() + out += tplStr(encodeTpl["handleMarshaler"], handleMarshaler{ + IC: ic, + Name: name, + Typ: typ, + Ptr: reflect.Ptr, + MarshalJSONBuf: typ.Implements(marshalerFasterType) || reflect.PtrTo(typ).Implements(marshalerFasterType) || typeInInception(ic, typ, shared.MustEncoder), + Marshaler: typ.Implements(marshalerType) || reflect.PtrTo(typ).Implements(marshalerType), + }) + return out + } + + ptname := name + if ptr { + ptname = "*" + name + } + + switch typ.Kind() { + case reflect.Int, + reflect.Int8, + reflect.Int16, + reflect.Int32, + reflect.Int64: + ic.OutputImports[`fflib "github.com/pquerna/ffjson/fflib/v1"`] = true + out += "fflib.FormatBits2(buf, uint64(" + ptname + "), 10, " + ptname + " < 0)" + "\n" + case reflect.Uint, + reflect.Uint8, + reflect.Uint16, + reflect.Uint32, + reflect.Uint64, + reflect.Uintptr: + ic.OutputImports[`fflib "github.com/pquerna/ffjson/fflib/v1"`] = true + out += "fflib.FormatBits2(buf, uint64(" + ptname + "), 10, false)" + "\n" + case reflect.Float32: + ic.OutputImports[`fflib "github.com/pquerna/ffjson/fflib/v1"`] = true + out += "fflib.AppendFloat(buf, float64(" + ptname + "), 'g', -1, 32)" + "\n" + case reflect.Float64: + ic.OutputImports[`fflib "github.com/pquerna/ffjson/fflib/v1"`] = true + out += "fflib.AppendFloat(buf, float64(" + ptname + "), 'g', -1, 64)" + "\n" + case reflect.Array, + reflect.Slice: + + // Arrays cannot be nil + if typ.Kind() != reflect.Array { + out += "if " + name + "!= nil {" + "\n" + } + // Array and slice values encode as JSON arrays, except that + // []byte encodes as a base64-encoded string, and a nil slice + // encodes as the null JSON object. + if typ.Kind() == reflect.Slice && typ.Elem().Kind() == reflect.Uint8 { + ic.OutputImports[`"encoding/base64"`] = true + + out += "buf.WriteString(`\"`)" + "\n" + out += `{` + "\n" + out += `enc := base64.NewEncoder(base64.StdEncoding, buf)` + "\n" + if typ.Elem().Name() != "byte" { + ic.OutputImports[`"reflect"`] = true + out += `enc.Write(reflect.Indirect(reflect.ValueOf(` + ptname + `)).Bytes())` + "\n" + + } else { + out += `enc.Write(` + ptname + `)` + "\n" + } + out += `enc.Close()` + "\n" + out += `}` + "\n" + out += "buf.WriteString(`\"`)" + "\n" + } else { + out += "buf.WriteString(`[`)" + "\n" + out += "for i, v := range " + ptname + "{" + "\n" + out += "if i != 0 {" + "\n" + out += "buf.WriteString(`,`)" + "\n" + out += "}" + "\n" + out += getGetInnerValue(ic, "v", typ.Elem(), false, false) + out += "}" + "\n" + out += "buf.WriteString(`]`)" + "\n" + } + if typ.Kind() != reflect.Array { + out += "} else {" + "\n" + out += "buf.WriteString(`null`)" + "\n" + out += "}" + "\n" + } + case reflect.String: + // Is it a json.Number? + if typ.PkgPath() == "encoding/json" && typ.Name() == "Number" { + // Fall back to json package to rely on the valid number check. + // See: https://github.com/golang/go/blob/92cd6e3af9f423ab4d8ac78f24e7fd81c31a8ce6/src/encoding/json/encode.go#L550 + out += fmt.Sprintf("/* json.Number */\n") + out += "err = buf.Encode(" + name + ")" + "\n" + out += "if err != nil {" + "\n" + out += " return err" + "\n" + out += "}" + "\n" + } else { + ic.OutputImports[`fflib "github.com/pquerna/ffjson/fflib/v1"`] = true + if forceString { + // Forcestring on strings does double-escaping of the entire value. + // We create a temporary buffer, encode to that an re-encode it. + out += "{" + "\n" + out += "tmpbuf := fflib.Buffer{}" + "\n" + out += "tmpbuf.Grow(len(" + ptname + ") + 16)" + "\n" + out += "fflib.WriteJsonString(&tmpbuf, string(" + ptname + "))" + "\n" + out += "fflib.WriteJsonString(buf, string( tmpbuf.Bytes() " + `))` + "\n" + out += "}" + "\n" + } else { + out += "fflib.WriteJsonString(buf, string(" + ptname + "))" + "\n" + } + } + case reflect.Ptr: + out += "if " + name + "!= nil {" + "\n" + switch typ.Elem().Kind() { + case reflect.Struct: + out += getGetInnerValue(ic, name, typ.Elem(), false, false) + default: + out += getGetInnerValue(ic, "*"+name, typ.Elem(), false, false) + } + out += "} else {" + "\n" + out += "buf.WriteString(`null`)" + "\n" + out += "}" + "\n" + case reflect.Bool: + out += "if " + ptname + " {" + "\n" + ic.q.Write("true") + out += ic.q.GetQueued() + out += "} else {" + "\n" + // Delete 'true' + ic.q.DeleteLast() + out += ic.q.WriteFlush("false") + out += "}" + "\n" + case reflect.Interface: + out += fmt.Sprintf("/* Interface types must use runtime reflection. type=%v kind=%v */\n", typ, typ.Kind()) + out += "err = buf.Encode(" + name + ")" + "\n" + out += "if err != nil {" + "\n" + out += " return err" + "\n" + out += "}" + "\n" + case reflect.Map: + out += getMapValue(ic, ptname, typ, ptr, forceString) + case reflect.Struct: + if typ.Name() == "" { + ic.q.Write("{") + ic.q.Write(" ") + out += fmt.Sprintf("/* Inline struct. type=%v kind=%v */\n", typ, typ.Kind()) + newV := reflect.Indirect(reflect.New(typ)).Interface() + fields := extractFields(newV) + + // Output all fields + for _, field := range fields { + // Adjust field name + field.Name = name + "." + field.Name + out += getField(ic, field, "") + } + + if lastConditional(fields) { + out += ic.q.Flush() + out += `buf.Rewind(1)` + "\n" + } else { + ic.q.DeleteLast() + } + out += ic.q.WriteFlush("}") + } else { + out += fmt.Sprintf("/* Struct fall back. type=%v kind=%v */\n", typ, typ.Kind()) + out += ic.q.Flush() + if ptr { + out += "err = buf.Encode(" + name + ")" + "\n" + } else { + // We send pointer to avoid copying entire struct + out += "err = buf.Encode(&" + name + ")" + "\n" + } + out += "if err != nil {" + "\n" + out += " return err" + "\n" + out += "}" + "\n" + } + default: + out += fmt.Sprintf("/* Falling back. type=%v kind=%v */\n", typ, typ.Kind()) + out += "err = buf.Encode(" + name + ")" + "\n" + out += "if err != nil {" + "\n" + out += " return err" + "\n" + out += "}" + "\n" + } + + return out +} + +func getValue(ic *Inception, sf *StructField, prefix string) string { + closequote := false + if sf.ForceString { + switch sf.Typ.Kind() { + case reflect.Int, + reflect.Int8, + reflect.Int16, + reflect.Int32, + reflect.Int64, + reflect.Uint, + reflect.Uint8, + reflect.Uint16, + reflect.Uint32, + reflect.Uint64, + reflect.Uintptr, + reflect.Float32, + reflect.Float64, + reflect.Bool: + ic.q.Write(`"`) + closequote = true + } + } + out := getGetInnerValue(ic, prefix+sf.Name, sf.Typ, sf.Pointer, sf.ForceString) + if closequote { + if sf.Pointer { + out += ic.q.WriteFlush(`"`) + } else { + ic.q.Write(`"`) + } + } + + return out +} + +func p2(v uint32) uint32 { + v-- + v |= v >> 1 + v |= v >> 2 + v |= v >> 4 + v |= v >> 8 + v |= v >> 16 + v++ + return v +} + +func getTypeSize(t reflect.Type) uint32 { + switch t.Kind() { + case reflect.String: + // TODO: consider runtime analysis. + return 32 + case reflect.Array, reflect.Map, reflect.Slice: + // TODO: consider runtime analysis. + return 4 * getTypeSize(t.Elem()) + case reflect.Int, + reflect.Int8, + reflect.Int16, + reflect.Int32, + reflect.Uint, + reflect.Uint8, + reflect.Uint16, + reflect.Uint32: + return 8 + case reflect.Int64, + reflect.Uint64, + reflect.Uintptr: + return 16 + case reflect.Float32, + reflect.Float64: + return 16 + case reflect.Bool: + return 4 + case reflect.Ptr: + return getTypeSize(t.Elem()) + default: + return 16 + } +} + +func getTotalSize(si *StructInfo) uint32 { + rv := uint32(si.Typ.Size()) + for _, f := range si.Fields { + rv += getTypeSize(f.Typ) + } + return rv +} + +func getBufGrowSize(si *StructInfo) uint32 { + + // TOOD(pquerna): automatically calc a better grow size based on history + // of a struct. + return p2(getTotalSize(si)) +} + +func isIntish(t reflect.Type) bool { + if t.Kind() >= reflect.Int && t.Kind() <= reflect.Uintptr { + return true + } + if t.Kind() == reflect.Array || t.Kind() == reflect.Slice || t.Kind() == reflect.Ptr { + if t.Kind() == reflect.Slice && t.Elem().Kind() == reflect.Uint8 { + // base64 special case. + return false + } else { + return isIntish(t.Elem()) + } + } + return false +} + +func getField(ic *Inception, f *StructField, prefix string) string { + out := "" + if f.OmitEmpty { + out += ic.q.Flush() + if f.Pointer { + out += "if " + prefix + f.Name + " != nil {" + "\n" + } + out += getOmitEmpty(ic, f) + } + + if f.Pointer && !f.OmitEmpty { + // Pointer values encode as the value pointed to. A nil pointer encodes as the null JSON object. + out += "if " + prefix + f.Name + " != nil {" + "\n" + } + + // JsonName is already escaped and quoted. + // getInnervalue should flush + ic.q.Write(f.JsonName + ":") + // We save a copy in case we need it + t := ic.q + + out += getValue(ic, f, prefix) + ic.q.Write(",") + + if f.Pointer && !f.OmitEmpty { + out += "} else {" + "\n" + out += t.WriteFlush("null") + out += "}" + "\n" + } + + if f.OmitEmpty { + out += ic.q.Flush() + if f.Pointer { + out += "}" + "\n" + } + out += "}" + "\n" + } + return out +} + +// We check if the last field is conditional. +func lastConditional(fields []*StructField) bool { + if len(fields) > 0 { + f := fields[len(fields)-1] + return f.OmitEmpty + } + return false +} + +func CreateMarshalJSON(ic *Inception, si *StructInfo) error { + conditionalWrites := lastConditional(si.Fields) + out := "" + + out += "// MarshalJSON marshal bytes to json - template\n" + out += `func (j *` + si.Name + `) MarshalJSON() ([]byte, error) {` + "\n" + out += `var buf fflib.Buffer` + "\n" + + out += `if j == nil {` + "\n" + out += ` buf.WriteString("null")` + "\n" + out += " return buf.Bytes(), nil" + "\n" + out += `}` + "\n" + + out += `err := j.MarshalJSONBuf(&buf)` + "\n" + out += `if err != nil {` + "\n" + out += " return nil, err" + "\n" + out += `}` + "\n" + out += `return buf.Bytes(), nil` + "\n" + out += `}` + "\n" + + out += "// MarshalJSONBuf marshal buff to json - template\n" + out += `func (j *` + si.Name + `) MarshalJSONBuf(buf fflib.EncodingBuffer) (error) {` + "\n" + out += ` if j == nil {` + "\n" + out += ` buf.WriteString("null")` + "\n" + out += " return nil" + "\n" + out += ` }` + "\n" + + out += `var err error` + "\n" + out += `var obj []byte` + "\n" + out += `_ = obj` + "\n" + out += `_ = err` + "\n" + + ic.q.Write("{") + + // The extra space is inserted here. + // If nothing is written to the field this will be deleted + // instead of the last comma. + if conditionalWrites || len(si.Fields) == 0 { + ic.q.Write(" ") + } + + for _, f := range si.Fields { + out += getField(ic, f, "j.") + } + + // Handling the last comma is tricky. + // If the last field has omitempty, conditionalWrites is set. + // If something has been written, we delete the last comma, + // by backing up the buffer, otherwise it will delete a space. + if conditionalWrites { + out += ic.q.Flush() + out += `buf.Rewind(1)` + "\n" + } else { + ic.q.DeleteLast() + } + + out += ic.q.WriteFlush("}") + out += `return nil` + "\n" + out += `}` + "\n" + ic.OutputFuncs = append(ic.OutputFuncs, out) + return nil +} diff --git a/vendor/github.com/pquerna/ffjson/inception/encoder_tpl.go b/vendor/github.com/pquerna/ffjson/inception/encoder_tpl.go new file mode 100644 index 000000000..22ab5292e --- /dev/null +++ b/vendor/github.com/pquerna/ffjson/inception/encoder_tpl.go @@ -0,0 +1,73 @@ +/** + * Copyright 2014 Paul Querna + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package ffjsoninception + +import ( + "reflect" + "text/template" +) + +var encodeTpl map[string]*template.Template + +func init() { + encodeTpl = make(map[string]*template.Template) + + funcs := map[string]string{ + "handleMarshaler": handleMarshalerTxt, + } + tplFuncs := template.FuncMap{} + + for k, v := range funcs { + encodeTpl[k] = template.Must(template.New(k).Funcs(tplFuncs).Parse(v)) + } +} + +type handleMarshaler struct { + IC *Inception + Name string + Typ reflect.Type + Ptr reflect.Kind + MarshalJSONBuf bool + Marshaler bool +} + +var handleMarshalerTxt = ` + { + {{if eq .Typ.Kind .Ptr}} + if {{.Name}} == nil { + buf.WriteString("null") + } else { + {{end}} + + {{if eq .MarshalJSONBuf true}} + err = {{.Name}}.MarshalJSONBuf(buf) + if err != nil { + return err + } + {{else if eq .Marshaler true}} + obj, err = {{.Name}}.MarshalJSON() + if err != nil { + return err + } + buf.Write(obj) + {{end}} + {{if eq .Typ.Kind .Ptr}} + } + {{end}} + } +` diff --git a/vendor/github.com/pquerna/ffjson/inception/inception.go b/vendor/github.com/pquerna/ffjson/inception/inception.go new file mode 100644 index 000000000..10cb2712c --- /dev/null +++ b/vendor/github.com/pquerna/ffjson/inception/inception.go @@ -0,0 +1,160 @@ +/** + * Copyright 2014 Paul Querna + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package ffjsoninception + +import ( + "errors" + "fmt" + "github.com/pquerna/ffjson/shared" + "io/ioutil" + "os" + "reflect" + "sort" +) + +type Inception struct { + objs []*StructInfo + InputPath string + OutputPath string + PackageName string + PackagePath string + OutputImports map[string]bool + OutputFuncs []string + q ConditionalWrite + ResetFields bool +} + +func NewInception(inputPath string, packageName string, outputPath string, resetFields bool) *Inception { + return &Inception{ + objs: make([]*StructInfo, 0), + InputPath: inputPath, + OutputPath: outputPath, + PackageName: packageName, + OutputFuncs: make([]string, 0), + OutputImports: make(map[string]bool), + ResetFields: resetFields, + } +} + +func (i *Inception) AddMany(objs []shared.InceptionType) { + for _, obj := range objs { + i.Add(obj) + } +} + +func (i *Inception) Add(obj shared.InceptionType) { + i.objs = append(i.objs, NewStructInfo(obj)) + i.PackagePath = i.objs[0].Typ.PkgPath() +} + +func (i *Inception) wantUnmarshal(si *StructInfo) bool { + if si.Options.SkipDecoder { + return false + } + typ := si.Typ + umlx := typ.Implements(unmarshalFasterType) || reflect.PtrTo(typ).Implements(unmarshalFasterType) + umlstd := typ.Implements(unmarshalerType) || reflect.PtrTo(typ).Implements(unmarshalerType) + if umlstd && !umlx { + // structure has UnmarshalJSON, but not our faster version -- skip it. + return false + } + return true +} + +func (i *Inception) wantMarshal(si *StructInfo) bool { + if si.Options.SkipEncoder { + return false + } + typ := si.Typ + mlx := typ.Implements(marshalerFasterType) || reflect.PtrTo(typ).Implements(marshalerFasterType) + mlstd := typ.Implements(marshalerType) || reflect.PtrTo(typ).Implements(marshalerType) + if mlstd && !mlx { + // structure has MarshalJSON, but not our faster version -- skip it. + return false + } + return true +} + +type sortedStructs []*StructInfo + +func (p sortedStructs) Len() int { return len(p) } +func (p sortedStructs) Less(i, j int) bool { return p[i].Name < p[j].Name } +func (p sortedStructs) Swap(i, j int) { p[i], p[j] = p[j], p[i] } +func (p sortedStructs) Sort() { sort.Sort(p) } + +func (i *Inception) generateCode() error { + // We sort the structs by name, so output if predictable. + sorted := sortedStructs(i.objs) + sorted.Sort() + + for _, si := range sorted { + if i.wantMarshal(si) { + err := CreateMarshalJSON(i, si) + if err != nil { + return err + } + } + + if i.wantUnmarshal(si) { + err := CreateUnmarshalJSON(i, si) + if err != nil { + return err + } + } + } + return nil +} + +func (i *Inception) handleError(err error) { + fmt.Fprintf(os.Stderr, "Error: %s:\n\n", err) + os.Exit(1) +} + +func (i *Inception) Execute() { + if len(os.Args) != 1 { + i.handleError(errors.New(fmt.Sprintf("Internal ffjson error: inception executable takes no args: %v", os.Args))) + return + } + + err := i.generateCode() + if err != nil { + i.handleError(err) + return + } + + data, err := RenderTemplate(i) + if err != nil { + i.handleError(err) + return + } + + stat, err := os.Stat(i.InputPath) + + if err != nil { + i.handleError(err) + return + } + + err = ioutil.WriteFile(i.OutputPath, data, stat.Mode()) + + if err != nil { + i.handleError(err) + return + } + +} diff --git a/vendor/github.com/pquerna/ffjson/inception/reflect.go b/vendor/github.com/pquerna/ffjson/inception/reflect.go new file mode 100644 index 000000000..8fb0bd5cb --- /dev/null +++ b/vendor/github.com/pquerna/ffjson/inception/reflect.go @@ -0,0 +1,290 @@ +/** + * Copyright 2014 Paul Querna + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package ffjsoninception + +import ( + fflib "github.com/pquerna/ffjson/fflib/v1" + "github.com/pquerna/ffjson/shared" + + "bytes" + "encoding/json" + "reflect" + "unicode/utf8" +) + +type StructField struct { + Name string + JsonName string + FoldFuncName string + Typ reflect.Type + OmitEmpty bool + ForceString bool + HasMarshalJSON bool + HasUnmarshalJSON bool + Pointer bool + Tagged bool +} + +type FieldByJsonName []*StructField + +func (a FieldByJsonName) Len() int { return len(a) } +func (a FieldByJsonName) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a FieldByJsonName) Less(i, j int) bool { return a[i].JsonName < a[j].JsonName } + +type StructInfo struct { + Name string + Obj interface{} + Typ reflect.Type + Fields []*StructField + Options shared.StructOptions +} + +func NewStructInfo(obj shared.InceptionType) *StructInfo { + t := reflect.TypeOf(obj.Obj) + return &StructInfo{ + Obj: obj.Obj, + Name: t.Name(), + Typ: t, + Fields: extractFields(obj.Obj), + Options: obj.Options, + } +} + +func (si *StructInfo) FieldsByFirstByte() map[string][]*StructField { + rv := make(map[string][]*StructField) + for _, f := range si.Fields { + b := string(f.JsonName[1]) + rv[b] = append(rv[b], f) + } + return rv +} + +func (si *StructInfo) ReverseFields() []*StructField { + var i int + rv := make([]*StructField, 0) + for i = len(si.Fields) - 1; i >= 0; i-- { + rv = append(rv, si.Fields[i]) + } + return rv +} + +const ( + caseMask = ^byte(0x20) // Mask to ignore case in ASCII. +) + +func foldFunc(key []byte) string { + nonLetter := false + special := false // special letter + for _, b := range key { + if b >= utf8.RuneSelf { + return "bytes.EqualFold" + } + upper := b & caseMask + if upper < 'A' || upper > 'Z' { + nonLetter = true + } else if upper == 'K' || upper == 'S' { + // See above for why these letters are special. + special = true + } + } + if special { + return "fflib.EqualFoldRight" + } + if nonLetter { + return "fflib.AsciiEqualFold" + } + return "fflib.SimpleLetterEqualFold" +} + +type MarshalerFaster interface { + MarshalJSONBuf(buf fflib.EncodingBuffer) error +} + +type UnmarshalFaster interface { + UnmarshalJSONFFLexer(l *fflib.FFLexer, state fflib.FFParseState) error +} + +var marshalerType = reflect.TypeOf(new(json.Marshaler)).Elem() +var marshalerFasterType = reflect.TypeOf(new(MarshalerFaster)).Elem() +var unmarshalerType = reflect.TypeOf(new(json.Unmarshaler)).Elem() +var unmarshalFasterType = reflect.TypeOf(new(UnmarshalFaster)).Elem() + +// extractFields returns a list of fields that JSON should recognize for the given type. +// The algorithm is breadth-first search over the set of structs to include - the top struct +// and then any reachable anonymous structs. +func extractFields(obj interface{}) []*StructField { + t := reflect.TypeOf(obj) + // Anonymous fields to explore at the current level and the next. + current := []StructField{} + next := []StructField{{Typ: t}} + + // Count of queued names for current level and the next. + count := map[reflect.Type]int{} + nextCount := map[reflect.Type]int{} + + // Types already visited at an earlier level. + visited := map[reflect.Type]bool{} + + // Fields found. + var fields []*StructField + + for len(next) > 0 { + current, next = next, current[:0] + count, nextCount = nextCount, map[reflect.Type]int{} + + for _, f := range current { + if visited[f.Typ] { + continue + } + visited[f.Typ] = true + + // Scan f.typ for fields to include. + for i := 0; i < f.Typ.NumField(); i++ { + sf := f.Typ.Field(i) + if sf.PkgPath != "" { // unexported + continue + } + tag := sf.Tag.Get("json") + if tag == "-" { + continue + } + name, opts := parseTag(tag) + if !isValidTag(name) { + name = "" + } + + ft := sf.Type + ptr := false + if ft.Kind() == reflect.Ptr { + ptr = true + } + + if ft.Name() == "" && ft.Kind() == reflect.Ptr { + // Follow pointer. + ft = ft.Elem() + } + + // Record found field and index sequence. + if name != "" || !sf.Anonymous || ft.Kind() != reflect.Struct { + tagged := name != "" + if name == "" { + name = sf.Name + } + + var buf bytes.Buffer + fflib.WriteJsonString(&buf, name) + + field := &StructField{ + Name: sf.Name, + JsonName: string(buf.Bytes()), + FoldFuncName: foldFunc([]byte(name)), + Typ: ft, + HasMarshalJSON: ft.Implements(marshalerType), + HasUnmarshalJSON: ft.Implements(unmarshalerType), + OmitEmpty: opts.Contains("omitempty"), + ForceString: opts.Contains("string"), + Pointer: ptr, + Tagged: tagged, + } + + fields = append(fields, field) + + if count[f.Typ] > 1 { + // If there were multiple instances, add a second, + // so that the annihilation code will see a duplicate. + // It only cares about the distinction between 1 or 2, + // so don't bother generating any more copies. + fields = append(fields, fields[len(fields)-1]) + } + continue + } + + // Record new anonymous struct to explore in next round. + nextCount[ft]++ + if nextCount[ft] == 1 { + next = append(next, StructField{ + Name: ft.Name(), + Typ: ft, + }) + } + } + } + } + + // Delete all fields that are hidden by the Go rules for embedded fields, + // except that fields with JSON tags are promoted. + + // The fields are sorted in primary order of name, secondary order + // of field index length. Loop over names; for each name, delete + // hidden fields by choosing the one dominant field that survives. + out := fields[:0] + for advance, i := 0, 0; i < len(fields); i += advance { + // One iteration per name. + // Find the sequence of fields with the name of this first field. + fi := fields[i] + name := fi.JsonName + for advance = 1; i+advance < len(fields); advance++ { + fj := fields[i+advance] + if fj.JsonName != name { + break + } + } + if advance == 1 { // Only one field with this name + out = append(out, fi) + continue + } + dominant, ok := dominantField(fields[i : i+advance]) + if ok { + out = append(out, dominant) + } + } + + fields = out + + return fields +} + +// dominantField looks through the fields, all of which are known to +// have the same name, to find the single field that dominates the +// others using Go's embedding rules, modified by the presence of +// JSON tags. If there are multiple top-level fields, the boolean +// will be false: This condition is an error in Go and we skip all +// the fields. +func dominantField(fields []*StructField) (*StructField, bool) { + tagged := -1 // Index of first tagged field. + for i, f := range fields { + if f.Tagged { + if tagged >= 0 { + // Multiple tagged fields at the same level: conflict. + // Return no field. + return nil, false + } + tagged = i + } + } + if tagged >= 0 { + return fields[tagged], true + } + // All remaining fields have the same length. If there's more than one, + // we have a conflict (two fields named "X" at the same level) and we + // return no field. + if len(fields) > 1 { + return nil, false + } + return fields[0], true +} diff --git a/vendor/github.com/pquerna/ffjson/inception/tags.go b/vendor/github.com/pquerna/ffjson/inception/tags.go new file mode 100644 index 000000000..ccce101b8 --- /dev/null +++ b/vendor/github.com/pquerna/ffjson/inception/tags.go @@ -0,0 +1,79 @@ +/** + * Copyright 2014 Paul Querna + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package ffjsoninception + +import ( + "strings" + "unicode" +) + +// from: http://golang.org/src/pkg/encoding/json/tags.go + +// tagOptions is the string following a comma in a struct field's "json" +// tag, or the empty string. It does not include the leading comma. +type tagOptions string + +// parseTag splits a struct field's json tag into its name and +// comma-separated options. +func parseTag(tag string) (string, tagOptions) { + if idx := strings.Index(tag, ","); idx != -1 { + return tag[:idx], tagOptions(tag[idx+1:]) + } + return tag, tagOptions("") +} + +// Contains reports whether a comma-separated list of options +// contains a particular substr flag. substr must be surrounded by a +// string boundary or commas. +func (o tagOptions) Contains(optionName string) bool { + if len(o) == 0 { + return false + } + s := string(o) + for s != "" { + var next string + i := strings.Index(s, ",") + if i >= 0 { + s, next = s[:i], s[i+1:] + } + if s == optionName { + return true + } + s = next + } + return false +} + +func isValidTag(s string) bool { + if s == "" { + return false + } + for _, c := range s { + switch { + case strings.ContainsRune("!#$%&()*+-./:<=>?@[]^_{|}~ ", c): + // Backslash and quote chars are reserved, but + // otherwise any punctuation chars are allowed + // in a tag name. + default: + if !unicode.IsLetter(c) && !unicode.IsDigit(c) { + return false + } + } + } + return true +} diff --git a/vendor/github.com/pquerna/ffjson/inception/template.go b/vendor/github.com/pquerna/ffjson/inception/template.go new file mode 100644 index 000000000..121a23dd8 --- /dev/null +++ b/vendor/github.com/pquerna/ffjson/inception/template.go @@ -0,0 +1,60 @@ +/** + * Copyright 2014 Paul Querna + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package ffjsoninception + +import ( + "bytes" + "go/format" + "text/template" +) + +const ffjsonTemplate = ` +// Code generated by ffjson <https://github.com/pquerna/ffjson>. DO NOT EDIT. +// source: {{.InputPath}} + +package {{.PackageName}} + +import ( +{{range $k, $v := .OutputImports}}{{$k}} +{{end}} +) + +{{range .OutputFuncs}} +{{.}} +{{end}} + +` + +func RenderTemplate(ic *Inception) ([]byte, error) { + t := template.Must(template.New("ffjson.go").Parse(ffjsonTemplate)) + buf := new(bytes.Buffer) + err := t.Execute(buf, ic) + if err != nil { + return nil, err + } + return format.Source(buf.Bytes()) +} + +func tplStr(t *template.Template, data interface{}) string { + buf := bytes.Buffer{} + err := t.Execute(&buf, data) + if err != nil { + panic(err) + } + return buf.String() +} diff --git a/vendor/github.com/pquerna/ffjson/inception/writerstack.go b/vendor/github.com/pquerna/ffjson/inception/writerstack.go new file mode 100644 index 000000000..1521961c9 --- /dev/null +++ b/vendor/github.com/pquerna/ffjson/inception/writerstack.go @@ -0,0 +1,65 @@ +package ffjsoninception + +import "strings" + +// ConditionalWrite is a stack containing a number of pending writes +type ConditionalWrite struct { + Queued []string +} + +// Write will add a string to be written +func (w *ConditionalWrite) Write(s string) { + w.Queued = append(w.Queued, s) +} + +// DeleteLast will delete the last added write +func (w *ConditionalWrite) DeleteLast() { + if len(w.Queued) == 0 { + return + } + w.Queued = w.Queued[:len(w.Queued)-1] +} + +// Last will return the last added write +func (w *ConditionalWrite) Last() string { + if len(w.Queued) == 0 { + return "" + } + return w.Queued[len(w.Queued)-1] +} + +// Flush will return all queued writes, and return +// "" (empty string) in nothing has been queued +// "buf.WriteByte('" + byte + "')" + '\n' if one bute has been queued. +// "buf.WriteString(`" + string + "`)" + "\n" if more than one byte has been queued. +func (w *ConditionalWrite) Flush() string { + combined := strings.Join(w.Queued, "") + if len(combined) == 0 { + return "" + } + + w.Queued = nil + if len(combined) == 1 { + return "buf.WriteByte('" + combined + "')" + "\n" + } + return "buf.WriteString(`" + combined + "`)" + "\n" +} + +func (w *ConditionalWrite) FlushTo(out string) string { + out += w.Flush() + return out +} + +// WriteFlush will add a string and return the Flush result for the queue +func (w *ConditionalWrite) WriteFlush(s string) string { + w.Write(s) + return w.Flush() +} + +// GetQueued will return the current queued content without flushing. +func (w *ConditionalWrite) GetQueued() string { + t := w.Queued + s := w.Flush() + w.Queued = t + return s +} diff --git a/vendor/github.com/pquerna/ffjson/shared/options.go b/vendor/github.com/pquerna/ffjson/shared/options.go new file mode 100644 index 000000000..d74edc135 --- /dev/null +++ b/vendor/github.com/pquerna/ffjson/shared/options.go @@ -0,0 +1,51 @@ +/** + * Copyright 2014 Paul Querna, Klaus Post + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package shared + +type StructOptions struct { + SkipDecoder bool + SkipEncoder bool +} + +type InceptionType struct { + Obj interface{} + Options StructOptions +} +type Feature int + +const ( + Nothing Feature = 0 + MustDecoder = 1 << 1 + MustEncoder = 1 << 2 + MustEncDec = MustDecoder | MustEncoder +) + +func (i InceptionType) HasFeature(f Feature) bool { + return i.HasFeature(f) +} + +func (s StructOptions) HasFeature(f Feature) bool { + hasNeeded := true + if f&MustDecoder != 0 && s.SkipDecoder { + hasNeeded = false + } + if f&MustEncoder != 0 && s.SkipEncoder { + hasNeeded = false + } + return hasNeeded +} diff --git a/vendor/modules.txt b/vendor/modules.txt index b2202a6ac..efb7d99da 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -45,7 +45,7 @@ github.com/containernetworking/cni/pkg/version github.com/containernetworking/cni/pkg/types/020 # github.com/containernetworking/plugins v0.8.1 github.com/containernetworking/plugins/pkg/ns -# github.com/containers/buildah v1.9.2 +# github.com/containers/buildah v1.10.1 github.com/containers/buildah github.com/containers/buildah/imagebuildah github.com/containers/buildah/pkg/chrootuser @@ -60,7 +60,7 @@ github.com/containers/buildah/docker github.com/containers/buildah/pkg/blobcache github.com/containers/buildah/pkg/overlay github.com/containers/buildah/pkg/unshare -# github.com/containers/image v2.0.1+incompatible +# github.com/containers/image v3.0.2+incompatible github.com/containers/image/directory github.com/containers/image/docker github.com/containers/image/docker/archive @@ -79,12 +79,12 @@ github.com/containers/image/tarball github.com/containers/image/pkg/sysregistriesv2 github.com/containers/image/image github.com/containers/image/oci/layout -github.com/containers/image/pkg/sysregistries github.com/containers/image/directory/explicitfilepath github.com/containers/image/docker/policyconfiguration github.com/containers/image/pkg/blobinfocache/none github.com/containers/image/pkg/tlsclientconfig github.com/containers/image/pkg/strslice +github.com/containers/image/pkg/keyctl github.com/containers/image/version github.com/containers/image/docker/daemon github.com/containers/image/openshift @@ -103,7 +103,7 @@ github.com/containers/psgo/internal/dev github.com/containers/psgo/internal/proc github.com/containers/psgo/internal/process github.com/containers/psgo/internal/host -# github.com/containers/storage v1.12.16 +# github.com/containers/storage v1.13.2 github.com/containers/storage github.com/containers/storage/pkg/archive github.com/containers/storage/pkg/chrootarchive @@ -389,6 +389,8 @@ github.com/pkg/profile github.com/pmezard/go-difflib/difflib # github.com/pquerna/ffjson v0.0.0-20181028064349-e517b90714f7 github.com/pquerna/ffjson/fflib/v1 +github.com/pquerna/ffjson/inception +github.com/pquerna/ffjson/shared github.com/pquerna/ffjson/fflib/v1/internal # github.com/prometheus/client_golang v1.0.0 github.com/prometheus/client_golang/prometheus @@ -451,9 +453,9 @@ github.com/varlink/go/varlink github.com/varlink/go/cmd/varlink-go-interface-generator github.com/varlink/go/varlink/idl # github.com/vbatts/tar-split v0.11.1 +github.com/vbatts/tar-split/archive/tar github.com/vbatts/tar-split/tar/asm github.com/vbatts/tar-split/tar/storage -github.com/vbatts/tar-split/archive/tar # github.com/vbauerster/mpb v3.4.0+incompatible github.com/vbauerster/mpb github.com/vbauerster/mpb/decor diff --git a/version/version.go b/version/version.go index 286f66093..d5f91210e 100644 --- a/version/version.go +++ b/version/version.go @@ -4,7 +4,7 @@ package version // NOTE: remember to bump the version at the top // of the top-level README.md file when this is // bumped. -const Version = "1.4.5-dev" +const Version = "1.5.1-dev" // RemoteAPIVersion is the version for the remote // client API. It is used to determine compatibility |