diff options
238 files changed, 4830 insertions, 1905 deletions
diff --git a/.cirrus.yml b/.cirrus.yml index e36b4f484..cc645e601 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -223,6 +223,9 @@ meta_task: depends_on: - "gating" + - "vendor" + - "varlink_api" + - "build_each_commit" container: image: "quay.io/libpod/imgts:latest" # see contrib/imgts @@ -253,8 +256,8 @@ testing_task: depends_on: - "gating" - - "varlink_api" - "vendor" + - "varlink_api" - "build_each_commit" # Only test build cache-images, if that's what's requested @@ -264,7 +267,7 @@ testing_task: matrix: # Images are generated separately, from build_images_task (below) image_name: "${FEDORA_CACHE_IMAGE_NAME}" - image_name: "${PRIOR_FEDORA_CACHE_IMAGE_NAME}" + #image_name: "${PRIOR_FEDORA_CACHE_IMAGE_NAME}" image_name: "${UBUNTU_CACHE_IMAGE_NAME}" timeout_in: 120m @@ -292,7 +295,7 @@ testing_task: # This task executes tests under unique environments/conditions -special_testing_task: +special_testing_rootless_task: depends_on: - "gating" @@ -303,9 +306,11 @@ special_testing_task: only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\*\*\*\s*CIRRUS:\s*TEST\s*IMAGES\s*\*\*\*.*' env: + SPECIALMODE: 'rootless' # See docs + matrix: - SPECIALMODE: 'rootless' # See docs - SPECIALMODE: 'in_podman' # See docs + TEST_REMOTE_CLIENT: true + TEST_REMOTE_CLIENT: false timeout_in: 60m @@ -322,32 +327,33 @@ special_testing_task: failed_audit_log_script: 'cat /var/log/audit/audit.log || cat /var/log/kern.log || echo "Uh oh, cat audit.log failed"' failed_journalctl_b_script: 'journalctl -b || echo "Uh oh, journalctl -b failed"' - -# Because system tests are stored within the repository, it is sometimes -# necessary to execute them within a PR to validate changes. -optional_testing_task: +special_testing_in_podman_task: depends_on: - "gating" + - "varlink_api" + - "vendor" + - "build_each_commit" - # Only run system tests in PRs (not on merge) if magic string is present - # in the PR description. Post-merge system testing is assumed to happen - # later from OS distribution's build systems. - only_if: >- - $CIRRUS_BRANCH != 'master' && - $CIRRUS_CHANGE_MESSAGE !=~ '.*\*\*\*\s*CIRRUS:\s*TEST\s*IMAGES\s*\*\*\*.*' && - $CIRRUS_CHANGE_MESSAGE =~ '.*\*\*\*\s*CIRRUS:\s*SYSTEM\s*TEST\s*\*\*\*.*' + only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\*\*\*\s*CIRRUS:\s*TEST\s*IMAGES\s*\*\*\*.*' - gce_instance: - matrix: - image_name: "${FEDORA_CACHE_IMAGE_NAME}" - image_name: "${PRIOR_FEDORA_CACHE_IMAGE_NAME}" - image_name: "${UBUNTU_CACHE_IMAGE_NAME}" + env: + SPECIALMODE: 'in_podman' # See docs timeout_in: 60m setup_environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}' - system_test_script: '$SCRIPT_BASE/system_test.sh |& ${TIMESTAMP}' + integration_test_script: '$SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP}' + df_script: '${DFCMD}' + audit_log_script: 'cat /var/log/audit/audit.log || cat /var/log/kern.log' + journalctl_b_script: 'journalctl -b' + + on_failure: + failed_master_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_master_failure.sh' + # Job has already failed, don't fail again and miss collecting data + failed_df_script: '${DFCMD}' + failed_audit_log_script: 'cat /var/log/audit/audit.log || cat /var/log/kern.log || echo "Uh oh, cat audit.log failed"' + failed_journalctl_b_script: 'journalctl -b || echo "Uh oh, journalctl -b failed"' # Test building of new cache-images for future PR testing, in this PR. @@ -395,7 +401,7 @@ verify_test_built_images_task: gce_instance: matrix: # Images are generated separately, from build_images_task (below) - image_name: "fedora-28${BUILT_IMAGE_SUFFIX}" + #image_name: "fedora-28${BUILT_IMAGE_SUFFIX}" image_name: "fedora-29${BUILT_IMAGE_SUFFIX}" image_name: "ubuntu-18${BUILT_IMAGE_SUFFIX}" @@ -479,12 +485,13 @@ success_task: depends_on: # ignores any dependent task conditions - "gating" - - "build_each_commit" - "vendor" - "varlink_api" + - "build_each_commit" + - "meta" - "testing" - - "special_testing" - - "optional_testing" + - "special_testing_rootless" + - "special_testing_in_podman" - "test_build_cache_images" - "verify_test_built_images" - "build_cache_images" @@ -11,7 +11,7 @@ in the [API.md](https://github.com/containers/libpod/blob/master/API.md) file in [func BuildImageHierarchyMap(name: string) string](#BuildImageHierarchyMap) -[func Commit(name: string, image_name: string, changes: []string, author: string, message: string, pause: bool, manifestType: string) string](#Commit) +[func Commit(name: string, image_name: string, changes: []string, author: string, message: string, pause: bool, manifestType: string) MoreResponse](#Commit) [func ContainerArtifacts(name: string, artifactName: string) string](#ContainerArtifacts) @@ -308,14 +308,14 @@ BuildImageHierarchyMap is for the development of Podman and should not be used. ### <a name="Commit"></a>func Commit <div style="background-color: #E8E8E8; padding: 15px; margin: 10px; border-radius: 10px;"> -method Commit(name: [string](https://godoc.org/builtin#string), image_name: [string](https://godoc.org/builtin#string), changes: [[]string](#[]string), author: [string](https://godoc.org/builtin#string), message: [string](https://godoc.org/builtin#string), pause: [bool](https://godoc.org/builtin#bool), manifestType: [string](https://godoc.org/builtin#string)) [string](https://godoc.org/builtin#string)</div> +method Commit(name: [string](https://godoc.org/builtin#string), image_name: [string](https://godoc.org/builtin#string), changes: [[]string](#[]string), author: [string](https://godoc.org/builtin#string), message: [string](https://godoc.org/builtin#string), pause: [bool](https://godoc.org/builtin#bool), manifestType: [string](https://godoc.org/builtin#string)) [MoreResponse](#MoreResponse)</div> Commit, creates an image from an existing container. It requires the name or ID of the container as well as the resulting image name. Optionally, you can define an author and message to be added to the resulting image. You can also define changes to the resulting image for the following attributes: _CMD, ENTRYPOINT, ENV, EXPOSE, LABEL, ONBUILD, STOPSIGNAL, USER, VOLUME, and WORKDIR_. To pause the container while it is being committed, pass a _true_ bool for the pause argument. If the container cannot be found by the ID or name provided, a (ContainerNotFound)[#ContainerNotFound] error will be returned; otherwise, -the resulting image's ID will be returned as a string. +the resulting image's ID will be returned as a string inside a MoreResponse. ### <a name="ContainerArtifacts"></a>func ContainerArtifacts <div style="background-color: #E8E8E8; padding: 15px; margin: 10px; border-radius: 10px;"> @@ -1,6 +1,6 @@ GO ?= go DESTDIR ?= / -EPOCH_TEST_COMMIT ?= 8161802f7df857e0850f842261079c83290f9891 +EPOCH_TEST_COMMIT ?= 1f31892a9fd8573d4b25274b208e6b9f860cdf81 HEAD ?= HEAD CHANGELOG_BASE ?= HEAD~ CHANGELOG_TARGET ?= HEAD @@ -18,6 +18,7 @@ At a high level, the scope of libpod and Podman is the following: * Full management of container lifecycle * Support for pods to manage groups of containers together * Resource isolation of containers and pods. +* Support for a Docker-compatible CLI interface through Podman. * Integration with CRI-O to share containers and backend code. This project tests all builds against each supported version of Fedora, the latest released version of Red Hat Enterprise Linux, and the latest Ubuntu Long Term Support release. The community has also reported success with other Linux flavors. diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 5eb85d0bc..e36757edc 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,5 +1,36 @@ # Release Notes +## 1.4.0 +### Features +- The `podman checkpoint` and `podman restore` commands can now be used to migrate containers between Podman installations on different systems ([#1618](https://github.com/containers/libpod/issues/1618)) +- The `podman cp` command now supports a `pause` flag to pause containers while copying into them +- The remote client now supports a configuration file for pre-configuring connections to remote Podman installations + +### Bugfixes +- Fixed CVE-2019-10152 - The `podman cp` command improperly dereferenced symlinks in host context +- Fixed a bug where `podman commit` could improperly set environment variables that contained `=` characters ([#3132](https://github.com/containers/libpod/issues/3132)) +- Fixed a bug where rootless Podman would sometimes fail to start containers with forwarded ports ([#2942](https://github.com/containers/libpod/issues/2942)) +- Fixed a bug where `podman version` on the remote client could segfault ([#3145](https://github.com/containers/libpod/issues/3145)) +- Fixed a bug where `podman container runlabel` would use `/proc/self/exe` instead of the path of the Podman command when printing the command being executed +- Fixed a bug where filtering images by label did not work ([#3163](https://github.com/containers/libpod/issues/3163)) +- Fixed a bug where specifying a bing mount or tmpfs mount over an image volume would cause a container to be unable to start ([#3174](https://github.com/containers/libpod/issues/3174)) +- Fixed a bug where `podman generate kube` did not work with containers with named volumes +- Fixed a bug where rootless Podman would receive `permission denied` errors accessing `conmon.pid` ([#3187](https://github.com/containers/libpod/issues/3187)) +- Fixed a bug where `podman cp` with a folder specified as target would replace the folder, as opposed to copying into it ([#3184](https://github.com/containers/libpod/issues/3184)) +- Fixed a bug where rootless Podman commands could double-unlock a lock, causing a crash ([#3207](https://github.com/containers/libpod/issues/3207)) +- Fixed a bug where Podman incorrectly set `tmpcopyup` on `/dev/` mounts, causing errors when using the Kata containers runtime ([#3229](https://github.com/containers/libpod/issues/3229)) +- Fixed a bug where `podman exec` would fail on older kernels ([#2968](https://github.com/containers/libpod/issues/2968)) + +### Misc +- The `podman commit` command is now usable with the Podman remote client +- The `--signature-policy` flag (used with several image-related commands) has been deprecated +- The `podman unshare` command now defines two environment variables in the spawned shell: `CONTAINERS_RUNROOT` and `CONTAINERS_GRAPHROOT`, pointing to temporary and permanent storage for rootless containers +- Updated vendored containers/storage and containers/image libraries with numerous bugfixes +- Updated vendored Buildah to v1.8.3 +- Podman now requires [Conmon v0.2.0](https://github.com/containers/conmon/releases/tag/v0.2.0) +- The `podman cp` command is now aliased as `podman container cp` +- Rootless Podman will now default `init_path` using root Podman's configuration files (`/etc/containers/libpod.conf` and `/usr/share/containers/libpod.conf`) if not overridden in the rootless configuration + ## 1.3.1 ### Features - The `podman cp` command can now read input redirected to `STDIN`, and output to `STDOUT` instead of a file, using `-` instead of an argument. diff --git a/changelog.txt b/changelog.txt index ec0a62f26..ece82e15b 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,3 +1,137 @@ +- Changelog for v1.4.0 (2019-06-07) + * Update release notes for v1.4.0 + * Update release notes for v1.4.0 + * Disable a very badly flaking healthcheck test + * rootless: skip NS_GET_PARENT on old kernels + * Cirrus: Track VM Image calling GCE project + * remove -c for podman remote global options + * Vendor Buildah v1.8.3 + * Cirrus: Disable testing on F28 (EOL) + * migration: add possibility to restore a container with a new name + * Inherit rootless init_path from system libpod.conf + * Also download container images during restore + * Include container migration into tutorial + * Add man-pages for container migration + * Added bash completion for container migration + * Add test case for container migration + * Added support to migrate containers + * Added helper functions for container migration + * Fix restore options help text and comments + * fix timing issues with some tests + * pkg/varlinkapi/virtwriter/virtwriter.go: simplify func Reader + * rootless: block signals on re-exec + * cirrus: minor cleanup and refactoring + * manpage: podman-tool table: un-confuse version and varlink + * Create Dockerfiles for podmanimage + * rootless: use TEMP_FAILURE_RETRY macro + * rootless: fix return type + * rootless: make sure the buffer is NUL terminated + * split rootless local and remote testing + * Fix podman cp test by reordering operations + * Small fix to readme to force tests to run + * Do not set tmpcopyup on /dev + * do not run remote tests inside container + * podman remote-client commit + * Fix podman cp tests + * podman-remote.conf enablement + * Error when trying to copy into a running rootless ctr + * rootless: skip check fo /etc/containers/registries.conf + * We can't pause rootless containers during cp + * Fix bug in e2e tests for podman cp + * Tolerate non-running containers in paused cp + * Add test to ensure symlinks are resolved in ctr scope + * Add --pause to podman cp manpage and bash completions + * Pause containers while copying into them + * Use securejoin to merge paths in `podman cp` + * use imagecaches for local tests + * add dns flags to docs + * add missing container cp command + * Podman logs man page shouldn't include timestamps + * Fix the varlink upgraded calls + * hack: support setting local region/zone + * document missing container update command + * Add --follow to journald ctr logging + * Address comments + * Implement podman logs with log-driver journald + * bump go-systemd version + * Added --log-driver and journald logging + * Update completions and docs to use k8s file as log driver + * bump conmon to v0.2.0 + * runtime: unlock the alive lock only once + * rootless: make JoinUserAndMountNS private + * Revert "rootless: change default path for conmon.pid" + * rootless: enable loginctl linger + * rootless: new function to join existing conmon processes + * rootless: block signals for pause + * Update install.md ostree Debian dependencies. + * fix bug dest path of copying tar + * podman: honor env variable PODMAN_USERNS + * userns: add new option --userns=keep-id + * warn when --security-opt and --privileged + * baseline tests: apparmor with --privileged + * rootless: store also the original GID in the host + * Fix a potential flake in the tests for podman cp + * cirrus: update images w/ zip pkg + * Cirrus: Add zip package to images + * rootless: fix top huser and hgroup + * vendor: update psgo to v1.3.0 + * apparmor: don't load/set profile in privileged mode + * hack: ignore from all VCS files when tarballing + * hack: shrink xfer tarball size + * hack: Display IP address of VM from script + * document nullable types + * Add test cases for login and logout + * Remove unused return statement in kube volume code + * Fix play kube when a pod is specified + * Fix a 'generate kube' bug on ctrs with named volumes + * Add test for image volume conflict with user volume + * Cirrus: Fix missing CRIO_COMMIT -> CONMON_COMMIT + * When superceding mounts, check for opposite types + * make remote resize channel buffered + * Cirrus: workaround root expand failure + * Cirrus: Stub in F30 support + * Cirrus: fixups based on review feedback + * Cirrus: Overhaul/Simplify env. var setup + * Cirrus: Run tests on test-built cache-images + * Cirrus: Support testing of VM cache-image changes + * Cirrus: Remove "too new" runc hack + * libpod: prefer WaitForFile to polling + * Remove conmon from fedora install instructions + * rootless: force resources to be nil on cgroup v1 + * Fixup Flags + * Minor fix filtering images by label + * container: move channel close to its writer + * util: fix race condition in WaitForFile + * Update vendor of buildah and containers/images + * Add Jhon Honce (@jwhonce on github) to OWNERS + * Don't set apparmor if --priviliged + * docs/libpod.conf.5: Add "have" to "higher precedence" typo + * Output name of process on runlabel command + * Minor fix splitting env vars in podman-commit + * Fixup conmon documentation + * troubleshooting.md: add note about updating subuid/subgid + * system: migrate stops the pause process + * rootless: join namespace immediately when possible + * rootless: use a pause process + * migrate: not create a new namespace + * install.remote should be separate for install.bin + * Cirrus: Confirm networking is working + * Use containers/conmon + * Fix a typo in release notes, and bump README version + * s|kubernetes-sigs/cri-o|cri-o/cri-o|g + * Bump github.com/containers/storage to v1.12.7 + * remote: version: fix nil dereference + * Bump gitvalidation epoch + * Bump to v1.3.2-dev + * Add connection information to podman-remote info + * unshare: define CONTAINERS_GRAPHROOT and CONTAINERS_RUNROOT + * Touchup run man page + * unshare: use rootless from libpod + * Replace root-based rootless tests + * rootless: default --cgroup-manager=systemd in unified mode + * create: skip resources validation with cgroup v2 + * rootless, spec: allow resources with cgroup v2 + - Changelog for v1.3.1 (2019-05-16) * More release notes * Add unshare to podman diff --git a/cmd/podman/checkpoint.go b/cmd/podman/checkpoint.go index 234d683bb..86bc8b973 100644 --- a/cmd/podman/checkpoint.go +++ b/cmd/podman/checkpoint.go @@ -46,6 +46,7 @@ func init() { flags.BoolVar(&checkpointCommand.TcpEstablished, "tcp-established", false, "Checkpoint a container with established TCP connections") flags.BoolVarP(&checkpointCommand.All, "all", "a", false, "Checkpoint all running containers") flags.BoolVarP(&checkpointCommand.Latest, "latest", "l", false, "Act on the latest container podman is aware of") + flags.StringVarP(&checkpointCommand.Export, "export", "e", "", "Export the checkpoint image to a tar.gz") markFlagHiddenForRemoteClient("latest", flags) } @@ -64,6 +65,7 @@ func checkpointCmd(c *cliconfig.CheckpointValues) error { Keep: c.Keep, KeepRunning: c.LeaveRunning, TCPEstablished: c.TcpEstablished, + TargetFile: c.Export, } return runtime.Checkpoint(c, options) } diff --git a/cmd/podman/cliconfig/config.go b/cmd/podman/cliconfig/config.go index aaa4513d8..b8b1648b8 100644 --- a/cmd/podman/cliconfig/config.go +++ b/cmd/podman/cliconfig/config.go @@ -33,9 +33,11 @@ type MainFlags struct { LogLevel string TmpDir string - RemoteUserName string - RemoteHost string - VarlinkAddress string + RemoteUserName string + RemoteHost string + VarlinkAddress string + ConnectionName string + RemoteConfigFilePath string } type AttachValues struct { @@ -89,6 +91,7 @@ type CheckpointValues struct { TcpEstablished bool All bool Latest bool + Export string } type CommitValues struct { @@ -426,6 +429,8 @@ type RestoreValues struct { Keep bool Latest bool TcpEstablished bool + Import string + Name string } type RmValues struct { diff --git a/cmd/podman/cliconfig/create.go b/cmd/podman/cliconfig/create.go index 49ab3d827..5fb2eed10 100644 --- a/cmd/podman/cliconfig/create.go +++ b/cmd/podman/cliconfig/create.go @@ -24,4 +24,5 @@ type BuildValues struct { type CpValues struct { PodmanCommand Extract bool + Pause bool } diff --git a/cmd/podman/commands.go b/cmd/podman/commands.go index 2ac465b9d..18b0b7857 100644 --- a/cmd/podman/commands.go +++ b/cmd/podman/commands.go @@ -11,7 +11,6 @@ const remoteclient = false // Commands that the local client implements func getMainCommands() []*cobra.Command { rootCommands := []*cobra.Command{ - _commitCommand, _execCommand, _playCommand, _loginCommand, @@ -41,7 +40,6 @@ func getContainerSubCommands() []*cobra.Command { return []*cobra.Command{ _cleanupCommand, - _commitCommand, _execCommand, _mountCommand, _refreshCommand, diff --git a/cmd/podman/commit.go b/cmd/podman/commit.go index 2b38bab35..01e2ec701 100644 --- a/cmd/podman/commit.go +++ b/cmd/podman/commit.go @@ -2,16 +2,11 @@ package main import ( "fmt" - "io" - "os" "strings" - "github.com/containers/buildah" - "github.com/containers/image/manifest" "github.com/containers/libpod/cmd/podman/cliconfig" - "github.com/containers/libpod/cmd/podman/libpodruntime" "github.com/containers/libpod/libpod" - "github.com/containers/libpod/libpod/image" + "github.com/containers/libpod/pkg/adapter" "github.com/containers/libpod/pkg/util" "github.com/pkg/errors" "github.com/spf13/cobra" @@ -52,32 +47,17 @@ func init() { } func commitCmd(c *cliconfig.CommitValues) error { - runtime, err := libpodruntime.GetRuntime(getContext(), &c.PodmanCommand) + runtime, err := adapter.GetRuntime(getContext(), &c.PodmanCommand) if err != nil { return errors.Wrapf(err, "could not get runtime") } defer runtime.Shutdown(false) - var ( - writer io.Writer - mimeType string - ) args := c.InputArgs if len(args) != 2 { return errors.Errorf("you must provide a container name or ID and a target image name") } - switch c.Format { - case "oci": - mimeType = buildah.OCIv1ImageManifest - if c.Flag("message").Changed { - return errors.Errorf("messages are only compatible with the docker image format (-f docker)") - } - case "docker": - mimeType = manifest.DockerV2Schema2MediaType - default: - return errors.Errorf("unrecognized image format %q", c.Format) - } container := args[0] reference := args[1] if c.Flag("change").Changed { @@ -92,38 +72,10 @@ func commitCmd(c *cliconfig.CommitValues) error { } } - if !c.Quiet { - writer = os.Stderr - } - ctr, err := runtime.LookupContainer(container) - if err != nil { - return errors.Wrapf(err, "error looking up container %q", container) - } - - rtc, err := runtime.GetConfig() - if err != nil { - return err - } - - sc := image.GetSystemContext(rtc.SignaturePolicyPath, "", false) - coptions := buildah.CommitOptions{ - SignaturePolicyPath: rtc.SignaturePolicyPath, - ReportWriter: writer, - SystemContext: sc, - PreferredManifestType: mimeType, - } - options := libpod.ContainerCommitOptions{ - CommitOptions: coptions, - Pause: c.Pause, - IncludeVolumes: c.IncludeVolumes, - Message: c.Message, - Changes: c.Change, - Author: c.Author, - } - newImage, err := ctr.Commit(getContext(), reference, options) + iid, err := runtime.Commit(getContext(), c, container, reference) if err != nil { return err } - fmt.Println(newImage.ID()) + fmt.Println(iid) return nil } diff --git a/cmd/podman/container.go b/cmd/podman/container.go index 530175a55..cb54317c0 100644 --- a/cmd/podman/container.go +++ b/cmd/podman/container.go @@ -52,8 +52,10 @@ var ( containerCommands = []*cobra.Command{ _attachCommand, _checkpointCommand, + _commitCommand, _containerExistsCommand, _contInspectSubCommand, + _cpCommand, _diffCommand, _exportCommand, _createCommand, diff --git a/cmd/podman/cp.go b/cmd/podman/cp.go index 8240cc193..7679ebcf1 100644 --- a/cmd/podman/cp.go +++ b/cmd/podman/cp.go @@ -13,10 +13,12 @@ import ( "github.com/containers/libpod/cmd/podman/cliconfig" "github.com/containers/libpod/cmd/podman/libpodruntime" "github.com/containers/libpod/libpod" + "github.com/containers/libpod/pkg/rootless" "github.com/containers/storage" "github.com/containers/storage/pkg/archive" "github.com/containers/storage/pkg/chrootarchive" "github.com/containers/storage/pkg/idtools" + securejoin "github.com/cyphar/filepath-securejoin" digest "github.com/opencontainers/go-digest" specs "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" @@ -49,6 +51,7 @@ func init() { cpCommand.Command = _cpCommand flags := cpCommand.Flags() flags.BoolVar(&cpCommand.Extract, "extract", false, "Extract the tar file into the destination directory.") + flags.BoolVar(&cpCommand.Pause, "pause", false, "Pause the container while copying") cpCommand.SetHelpTemplate(HelpTemplate()) cpCommand.SetUsageTemplate(UsageTemplate()) rootCmd.AddCommand(cpCommand.Command) @@ -66,11 +69,10 @@ func cpCmd(c *cliconfig.CpValues) error { } defer runtime.Shutdown(false) - extract := c.Flag("extract").Changed - return copyBetweenHostAndContainer(runtime, args[0], args[1], extract) + return copyBetweenHostAndContainer(runtime, args[0], args[1], c.Extract, c.Pause) } -func copyBetweenHostAndContainer(runtime *libpod.Runtime, src string, dest string, extract bool) error { +func copyBetweenHostAndContainer(runtime *libpod.Runtime, src string, dest string, extract bool, pause bool) error { srcCtr, srcPath := parsePath(runtime, src) destCtr, destPath := parsePath(runtime, dest) @@ -93,6 +95,38 @@ func copyBetweenHostAndContainer(runtime *libpod.Runtime, src string, dest strin return err } defer ctr.Unmount(false) + + // We can't pause rootless containers. + if pause && rootless.IsRootless() { + state, err := ctr.State() + if err != nil { + return err + } + if state == libpod.ContainerStateRunning { + return errors.Errorf("cannot copy into running rootless container with pause set - pass --pause=false to force copying") + } + } + + if pause && !rootless.IsRootless() { + if err := ctr.Pause(); err != nil { + // An invalid state error is fine. + // The container isn't running or is already paused. + // TODO: We can potentially start the container while + // the copy is running, which still allows a race where + // malicious code could mess with the symlink. + if errors.Cause(err) != libpod.ErrCtrStateInvalid { + return err + } + } else if err == nil { + // Only add the defer if we actually paused + defer func() { + if err := ctr.Unpause(); err != nil { + logrus.Errorf("Error unpausing container after copying: %v", err) + } + }() + } + } + user, err := getUser(mountPoint, ctr.User()) if err != nil { return err @@ -112,19 +146,38 @@ func copyBetweenHostAndContainer(runtime *libpod.Runtime, src string, dest strin var glob []string if isFromHostToCtr { if filepath.IsAbs(destPath) { - destPath = filepath.Join(mountPoint, destPath) - + cleanedPath, err := securejoin.SecureJoin(mountPoint, destPath) + if err != nil { + return err + } + destPath = cleanedPath } else { - if err = idtools.MkdirAllAndChownNew(filepath.Join(mountPoint, ctr.WorkingDir()), 0755, hostOwner); err != nil { + ctrWorkDir, err := securejoin.SecureJoin(mountPoint, ctr.WorkingDir()) + if err != nil { + return err + } + if err = idtools.MkdirAllAndChownNew(ctrWorkDir, 0755, hostOwner); err != nil { return errors.Wrapf(err, "error creating directory %q", destPath) } - destPath = filepath.Join(mountPoint, ctr.WorkingDir(), destPath) + cleanedPath, err := securejoin.SecureJoin(mountPoint, filepath.Join(ctr.WorkingDir(), destPath)) + if err != nil { + return err + } + destPath = cleanedPath } } else { if filepath.IsAbs(srcPath) { - srcPath = filepath.Join(mountPoint, srcPath) + cleanedPath, err := securejoin.SecureJoin(mountPoint, srcPath) + if err != nil { + return err + } + srcPath = cleanedPath } else { - srcPath = filepath.Join(mountPoint, ctr.WorkingDir(), srcPath) + cleanedPath, err := securejoin.SecureJoin(mountPoint, filepath.Join(ctr.WorkingDir(), srcPath)) + if err != nil { + return err + } + srcPath = cleanedPath } } glob, err = filepath.Glob(srcPath) @@ -158,7 +211,7 @@ func copyBetweenHostAndContainer(runtime *libpod.Runtime, src string, dest strin } func getUser(mountPoint string, userspec string) (specs.User, error) { - uid, gid, err := chrootuser.GetUser(mountPoint, userspec) + uid, gid, _, err := chrootuser.GetUser(mountPoint, userspec) u := specs.User{ UID: uid, GID: gid, diff --git a/cmd/podman/main.go b/cmd/podman/main.go index 787dd55c0..a149a47f9 100644 --- a/cmd/podman/main.go +++ b/cmd/podman/main.go @@ -30,6 +30,7 @@ var ( var mainCommands = []*cobra.Command{ _attachCommand, _buildCommand, + _commitCommand, _diffCommand, _createCommand, _eventsCommand, diff --git a/cmd/podman/main_local.go b/cmd/podman/main_local.go index 5af05a11e..b4f21bd0c 100644 --- a/cmd/podman/main_local.go +++ b/cmd/podman/main_local.go @@ -4,11 +4,9 @@ package main import ( "context" - "io/ioutil" "log/syslog" "os" "runtime/pprof" - "strconv" "strings" "syscall" @@ -120,18 +118,10 @@ func setupRootless(cmd *cobra.Command, args []string) error { return errors.Wrapf(err, "could not get pause process pid file path") } - data, err := ioutil.ReadFile(pausePidPath) - if err != nil && !os.IsNotExist(err) { - return errors.Wrapf(err, "cannot read pause process pid file %s", pausePidPath) - } - if err == nil { - pausePid, err := strconv.Atoi(string(data)) - if err != nil { - return errors.Wrapf(err, "cannot parse pause pid file %s", pausePidPath) - } - became, ret, err := rootless.JoinUserAndMountNS(uint(pausePid), "") + if _, err := os.Stat(pausePidPath); err == nil { + became, ret, err := rootless.TryJoinFromFilePaths("", false, []string{pausePidPath}) if err != nil { - logrus.Errorf("cannot join pause process pid %d. You may need to remove %s and stop all containers", pausePid, pausePidPath) + logrus.Errorf("cannot join pause process. You may need to remove %s and stop all containers", pausePidPath) logrus.Errorf("you can use `system migrate` to recreate the pause process") logrus.Errorf(err.Error()) os.Exit(1) @@ -154,28 +144,13 @@ func setupRootless(cmd *cobra.Command, args []string) error { logrus.Errorf(err.Error()) os.Exit(1) } - var became bool - var ret int - if len(ctrs) == 0 { - became, ret, err = rootless.BecomeRootInUserNS(pausePidPath) - } else { - for _, ctr := range ctrs { - data, err := ioutil.ReadFile(ctr.Config().ConmonPidFile) - if err != nil { - logrus.Errorf(err.Error()) - continue - } - conmonPid, err := strconv.Atoi(string(data)) - if err != nil { - logrus.Errorf(err.Error()) - continue - } - became, ret, err = rootless.JoinUserAndMountNS(uint(conmonPid), pausePidPath) - if err == nil { - break - } - } + + paths := []string{} + for _, ctr := range ctrs { + paths = append(paths, ctr.Config().ConmonPidFile) } + + became, ret, err := rootless.TryJoinFromFilePaths(pausePidPath, true, paths) if err != nil { logrus.Errorf(err.Error()) os.Exit(1) @@ -185,6 +160,7 @@ func setupRootless(cmd *cobra.Command, args []string) error { } return nil } + func setRLimits() error { rlimits := new(syscall.Rlimit) rlimits.Cur = 1048576 diff --git a/cmd/podman/main_remote.go b/cmd/podman/main_remote.go index c8bb3ad3e..de6c2c760 100644 --- a/cmd/podman/main_remote.go +++ b/cmd/podman/main_remote.go @@ -9,6 +9,8 @@ import ( const remote = true func init() { + rootCmd.PersistentFlags().StringVar(&MainGlobalOpts.ConnectionName, "connection", "", "remote connection name") + rootCmd.PersistentFlags().StringVar(&MainGlobalOpts.RemoteConfigFilePath, "remote-config-path", "", "alternate path for configuration file") rootCmd.PersistentFlags().StringVar(&MainGlobalOpts.RemoteUserName, "username", "", "username on the remote host") rootCmd.PersistentFlags().StringVar(&MainGlobalOpts.RemoteHost, "remote-host", "", "remote host") // TODO maybe we allow the altering of this for bridge connections? diff --git a/cmd/podman/platform_linux.go b/cmd/podman/platform_linux.go index 2127923ae..eb11867cc 100644 --- a/cmd/podman/platform_linux.go +++ b/cmd/podman/platform_linux.go @@ -4,13 +4,25 @@ package main import ( "os" + "path/filepath" + "github.com/containers/libpod/pkg/rootless" "github.com/sirupsen/logrus" ) +// userRegistriesFile is the path to the per user registry configuration file. +var userRegistriesFile = filepath.Join(os.Getenv("HOME"), ".config/containers/registries.conf") + func CheckForRegistries() { if _, err := os.Stat("/etc/containers/registries.conf"); err != nil { if os.IsNotExist(err) { + // If it is running in rootless mode, also check the user configuration file + if rootless.IsRootless() { + if _, err := os.Stat(userRegistriesFile); err != nil { + logrus.Warnf("unable to find %s. some podman (image shortnames) commands may be limited", userRegistriesFile) + } + return + } logrus.Warn("unable to find /etc/containers/registries.conf. some podman (image shortnames) commands may be limited") } } diff --git a/cmd/podman/remoteclientconfig/config.go b/cmd/podman/remoteclientconfig/config.go new file mode 100644 index 000000000..01f293ec3 --- /dev/null +++ b/cmd/podman/remoteclientconfig/config.go @@ -0,0 +1,21 @@ +package remoteclientconfig + +const remoteConfigFileName string = "podman-remote.conf" + +// RemoteConfig describes the podman remote configuration file +type RemoteConfig struct { + Connections map[string]RemoteConnection +} + +// RemoteConnection describes the attributes of a podman-remote endpoint +type RemoteConnection struct { + Destination string `toml:"destination"` + Username string `toml:"username"` + IsDefault bool `toml:"default"` +} + +// GetConfigFilePath is a simple helper to export the configuration file's +// path based on arch, etc +func GetConfigFilePath() string { + return getConfigFilePath() +} diff --git a/cmd/podman/remoteclientconfig/config_darwin.go b/cmd/podman/remoteclientconfig/config_darwin.go new file mode 100644 index 000000000..b94941381 --- /dev/null +++ b/cmd/podman/remoteclientconfig/config_darwin.go @@ -0,0 +1,12 @@ +package remoteclientconfig + +import ( + "path/filepath" + + "github.com/docker/docker/pkg/homedir" +) + +func getConfigFilePath() string { + homeDir := homedir.Get() + return filepath.Join(homeDir, ".config", "containers", remoteConfigFileName) +} diff --git a/cmd/podman/remoteclientconfig/config_linux.go b/cmd/podman/remoteclientconfig/config_linux.go new file mode 100644 index 000000000..b94941381 --- /dev/null +++ b/cmd/podman/remoteclientconfig/config_linux.go @@ -0,0 +1,12 @@ +package remoteclientconfig + +import ( + "path/filepath" + + "github.com/docker/docker/pkg/homedir" +) + +func getConfigFilePath() string { + homeDir := homedir.Get() + return filepath.Join(homeDir, ".config", "containers", remoteConfigFileName) +} diff --git a/cmd/podman/remoteclientconfig/config_windows.go b/cmd/podman/remoteclientconfig/config_windows.go new file mode 100644 index 000000000..fa6ffca63 --- /dev/null +++ b/cmd/podman/remoteclientconfig/config_windows.go @@ -0,0 +1,12 @@ +package remoteclientconfig + +import ( + "path/filepath" + + "github.com/docker/docker/pkg/homedir" +) + +func getConfigFilePath() string { + homeDir := homedir.Get() + return filepath.Join(homeDir, "AppData", "podman", remoteConfigFileName) +} diff --git a/cmd/podman/remoteclientconfig/configfile.go b/cmd/podman/remoteclientconfig/configfile.go new file mode 100644 index 000000000..aa3e82a31 --- /dev/null +++ b/cmd/podman/remoteclientconfig/configfile.go @@ -0,0 +1,62 @@ +package remoteclientconfig + +import ( + "io" + + "github.com/BurntSushi/toml" + "github.com/pkg/errors" +) + +// ReadRemoteConfig takes an io.Reader representing the remote configuration +// file and returns a remoteconfig +func ReadRemoteConfig(reader io.Reader) (*RemoteConfig, error) { + var remoteConfig RemoteConfig + // the configuration file does not exist + if reader == nil { + return &remoteConfig, ErrNoConfigationFile + } + _, err := toml.DecodeReader(reader, &remoteConfig) + if err != nil { + return nil, err + } + // We need to validate each remote connection has fields filled out + for name, conn := range remoteConfig.Connections { + if len(conn.Destination) < 1 { + return nil, errors.Errorf("connection %s has no destination defined", name) + } + } + return &remoteConfig, err +} + +// GetDefault returns the default RemoteConnection. If there is only one +// connection, we assume it is the default as well +func (r *RemoteConfig) GetDefault() (*RemoteConnection, error) { + if len(r.Connections) == 0 { + return nil, ErrNoDefinedConnections + } + for _, v := range r.Connections { + if len(r.Connections) == 1 { + // if there is only one defined connection, we assume it is + // the default whether tagged as such or not + return &v, nil + } + if v.IsDefault { + return &v, nil + } + } + return nil, ErrNoDefaultConnection +} + +// GetRemoteConnection "looks up" a remote connection by name and returns it in the +// form of a RemoteConnection +func (r *RemoteConfig) GetRemoteConnection(name string) (*RemoteConnection, error) { + if len(r.Connections) == 0 { + return nil, ErrNoDefinedConnections + } + for k, v := range r.Connections { + if k == name { + return &v, nil + } + } + return nil, errors.Wrap(ErrConnectionNotFound, name) +} diff --git a/cmd/podman/remoteclientconfig/configfile_test.go b/cmd/podman/remoteclientconfig/configfile_test.go new file mode 100644 index 000000000..66e0a4693 --- /dev/null +++ b/cmd/podman/remoteclientconfig/configfile_test.go @@ -0,0 +1,201 @@ +package remoteclientconfig + +import ( + "io" + "reflect" + "strings" + "testing" +) + +var goodConfig = ` +[connections] + +[connections.homer] +destination = "192.168.1.1" +username = "myuser" +default = true + +[connections.bart] +destination = "foobar.com" +username = "root" +` +var noDest = ` +[connections] + +[connections.homer] +destination = "192.168.1.1" +username = "myuser" +default = true + +[connections.bart] +username = "root" +` + +var noUser = ` +[connections] + +[connections.homer] +destination = "192.168.1.1" +` + +func makeGoodResult() *RemoteConfig { + var goodConnections = make(map[string]RemoteConnection) + goodConnections["homer"] = RemoteConnection{ + Destination: "192.168.1.1", + Username: "myuser", + IsDefault: true, + } + goodConnections["bart"] = RemoteConnection{ + Destination: "foobar.com", + Username: "root", + } + var goodResult = RemoteConfig{ + Connections: goodConnections, + } + return &goodResult +} + +func makeNoUserResult() *RemoteConfig { + var goodConnections = make(map[string]RemoteConnection) + goodConnections["homer"] = RemoteConnection{ + Destination: "192.168.1.1", + } + var goodResult = RemoteConfig{ + Connections: goodConnections, + } + return &goodResult +} + +func TestReadRemoteConfig(t *testing.T) { + type args struct { + reader io.Reader + } + tests := []struct { + name string + args args + want *RemoteConfig + wantErr bool + }{ + // good test should pass + {"good", args{reader: strings.NewReader(goodConfig)}, makeGoodResult(), false}, + // a connection with no destination is an error + {"nodest", args{reader: strings.NewReader(noDest)}, nil, true}, + // a connnection with no user is OK + {"nouser", args{reader: strings.NewReader(noUser)}, makeNoUserResult(), false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ReadRemoteConfig(tt.args.reader) + if (err != nil) != tt.wantErr { + t.Errorf("ReadRemoteConfig() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("ReadRemoteConfig() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestRemoteConfig_GetDefault(t *testing.T) { + good := make(map[string]RemoteConnection) + good["homer"] = RemoteConnection{ + Username: "myuser", + Destination: "192.168.1.1", + IsDefault: true, + } + good["bart"] = RemoteConnection{ + Username: "root", + Destination: "foobar.com", + } + noDefault := make(map[string]RemoteConnection) + noDefault["homer"] = RemoteConnection{ + Username: "myuser", + Destination: "192.168.1.1", + } + noDefault["bart"] = RemoteConnection{ + Username: "root", + Destination: "foobar.com", + } + single := make(map[string]RemoteConnection) + single["homer"] = RemoteConnection{ + Username: "myuser", + Destination: "192.168.1.1", + } + + none := make(map[string]RemoteConnection) + + type fields struct { + Connections map[string]RemoteConnection + } + tests := []struct { + name string + fields fields + want *RemoteConnection + wantErr bool + }{ + // A good toml should return the connection that is marked isDefault + {"good", fields{Connections: makeGoodResult().Connections}, &RemoteConnection{"192.168.1.1", "myuser", true}, false}, + // If nothing is marked as isDefault and there is more than one connection, error should occur + {"nodefault", fields{Connections: noDefault}, nil, true}, + // if nothing is marked as isDefault but there is only one connection, the one connection is considered the default + {"single", fields{Connections: none}, nil, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := &RemoteConfig{ + Connections: tt.fields.Connections, + } + got, err := r.GetDefault() + if (err != nil) != tt.wantErr { + t.Errorf("RemoteConfig.GetDefault() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("RemoteConfig.GetDefault() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestRemoteConfig_GetRemoteConnection(t *testing.T) { + type fields struct { + Connections map[string]RemoteConnection + } + type args struct { + name string + } + + blank := make(map[string]RemoteConnection) + tests := []struct { + name string + fields fields + args args + want *RemoteConnection + wantErr bool + }{ + // Good connection + {"goodhomer", fields{Connections: makeGoodResult().Connections}, args{name: "homer"}, &RemoteConnection{"192.168.1.1", "myuser", true}, false}, + // Good connection + {"goodbart", fields{Connections: makeGoodResult().Connections}, args{name: "bart"}, &RemoteConnection{"foobar.com", "root", false}, false}, + // Getting an unknown connection should result in error + {"noexist", fields{Connections: makeGoodResult().Connections}, args{name: "foobar"}, nil, true}, + // Getting a connection when there are none should result in an error + {"none", fields{Connections: blank}, args{name: "foobar"}, nil, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := &RemoteConfig{ + Connections: tt.fields.Connections, + } + got, err := r.GetRemoteConnection(tt.args.name) + if (err != nil) != tt.wantErr { + t.Errorf("RemoteConfig.GetRemoteConnection() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("RemoteConfig.GetRemoteConnection() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/cmd/podman/remoteclientconfig/errors.go b/cmd/podman/remoteclientconfig/errors.go new file mode 100644 index 000000000..2689d3b49 --- /dev/null +++ b/cmd/podman/remoteclientconfig/errors.go @@ -0,0 +1,14 @@ +package remoteclientconfig + +import "errors" + +var ( + // ErrNoDefaultConnection no default connection is defined in the podman-remote.conf file + ErrNoDefaultConnection = errors.New("no default connection is defined") + // ErrNoDefinedConnections no connections are defined in the podman-remote.conf file + ErrNoDefinedConnections = errors.New("no remote connections have been defined") + // ErrConnectionNotFound unable to lookup connection by name + ErrConnectionNotFound = errors.New("remote connection not found by name") + // ErrNoConfigationFile no config file found + ErrNoConfigationFile = errors.New("no configuration file found") +) diff --git a/cmd/podman/restore.go b/cmd/podman/restore.go index 8cfd5ca0d..9c77d4a5e 100644 --- a/cmd/podman/restore.go +++ b/cmd/podman/restore.go @@ -24,10 +24,10 @@ var ( restoreCommand.InputArgs = args restoreCommand.GlobalFlags = MainGlobalOpts restoreCommand.Remote = remoteclient - return restoreCmd(&restoreCommand) + return restoreCmd(&restoreCommand, cmd) }, Args: func(cmd *cobra.Command, args []string) error { - return checkAllAndLatest(cmd, args, false) + return checkAllAndLatest(cmd, args, true) }, Example: `podman container restore ctrID podman container restore --latest @@ -43,13 +43,14 @@ func init() { flags.BoolVarP(&restoreCommand.All, "all", "a", false, "Restore all checkpointed containers") flags.BoolVarP(&restoreCommand.Keep, "keep", "k", false, "Keep all temporary checkpoint files") flags.BoolVarP(&restoreCommand.Latest, "latest", "l", false, "Act on the latest container podman is aware of") - // TODO: add ContainerStateCheckpointed - flags.BoolVar(&restoreCommand.TcpEstablished, "tcp-established", false, "Checkpoint a container with established TCP connections") + flags.BoolVar(&restoreCommand.TcpEstablished, "tcp-established", false, "Restore a container with established TCP connections") + flags.StringVarP(&restoreCommand.Import, "import", "i", "", "Restore from exported checkpoint archive (tar.gz)") + flags.StringVarP(&restoreCommand.Name, "name", "n", "", "Specify new name for container restored from exported checkpoint (only works with --import)") markFlagHiddenForRemoteClient("latest", flags) } -func restoreCmd(c *cliconfig.RestoreValues) error { +func restoreCmd(c *cliconfig.RestoreValues, cmd *cobra.Command) error { if rootless.IsRootless() { return errors.New("restoring a container requires root") } @@ -63,6 +64,20 @@ func restoreCmd(c *cliconfig.RestoreValues) error { options := libpod.ContainerCheckpointOptions{ Keep: c.Keep, TCPEstablished: c.TcpEstablished, + TargetFile: c.Import, + Name: c.Name, } - return runtime.Restore(c, options) + + if c.Import == "" && c.Name != "" { + return errors.Errorf("--name can only used with --import") + } + + if c.Name != "" && c.TcpEstablished { + return errors.Errorf("--tcp-established cannot be used with --name") + } + + if (c.Import != "") && (c.All || c.Latest) { + return errors.Errorf("Cannot use --import and --all or --latest at the same time") + } + return runtime.Restore(getContext(), c, options) } diff --git a/cmd/podman/shared/container.go b/cmd/podman/shared/container.go index fe447d10d..c97eaa290 100644 --- a/cmd/podman/shared/container.go +++ b/cmd/podman/shared/container.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "io" - v1 "k8s.io/api/core/v1" "os" "path/filepath" "regexp" @@ -17,15 +16,13 @@ import ( "github.com/containers/image/types" "github.com/containers/libpod/libpod" "github.com/containers/libpod/libpod/image" - "github.com/containers/libpod/pkg/inspect" - cc "github.com/containers/libpod/pkg/spec" "github.com/containers/libpod/pkg/util" "github.com/cri-o/ocicni/pkg/ocicni" "github.com/docker/go-units" "github.com/google/shlex" - "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" "github.com/sirupsen/logrus" + v1 "k8s.io/api/core/v1" ) const ( @@ -621,140 +618,6 @@ func getStrFromSquareBrackets(cmd string) string { return strings.Join(arr, ",") } -// GetCtrInspectInfo takes container inspect data and collects all its info into a ContainerData -// structure for inspection related methods -func GetCtrInspectInfo(config *libpod.ContainerConfig, ctrInspectData *inspect.ContainerInspectData, createArtifact *cc.CreateConfig) (*inspect.ContainerData, error) { - spec := config.Spec - - cpus, mems, period, quota, realtimePeriod, realtimeRuntime, shares := getCPUInfo(spec) - blkioWeight, blkioWeightDevice, blkioReadBps, blkioWriteBps, blkioReadIOPS, blkioeWriteIOPS := getBLKIOInfo(spec) - memKernel, memReservation, memSwap, memSwappiness, memDisableOOMKiller := getMemoryInfo(spec) - pidsLimit := getPidsInfo(spec) - cgroup := getCgroup(spec) - - data := &inspect.ContainerData{ - ctrInspectData, - &inspect.HostConfig{ - ConsoleSize: spec.Process.ConsoleSize, - OomScoreAdj: spec.Process.OOMScoreAdj, - CPUShares: shares, - BlkioWeight: blkioWeight, - BlkioWeightDevice: blkioWeightDevice, - BlkioDeviceReadBps: blkioReadBps, - BlkioDeviceWriteBps: blkioWriteBps, - BlkioDeviceReadIOps: blkioReadIOPS, - BlkioDeviceWriteIOps: blkioeWriteIOPS, - CPUPeriod: period, - CPUQuota: quota, - CPURealtimePeriod: realtimePeriod, - CPURealtimeRuntime: realtimeRuntime, - CPUSetCPUs: cpus, - CPUSetMems: mems, - Devices: spec.Linux.Devices, - KernelMemory: memKernel, - MemoryReservation: memReservation, - MemorySwap: memSwap, - MemorySwappiness: memSwappiness, - OomKillDisable: memDisableOOMKiller, - PidsLimit: pidsLimit, - Privileged: config.Privileged, - ReadOnlyRootfs: spec.Root.Readonly, - ReadOnlyTmpfs: createArtifact.ReadOnlyTmpfs, - Runtime: config.OCIRuntime, - NetworkMode: string(createArtifact.NetMode), - IpcMode: string(createArtifact.IpcMode), - Cgroup: cgroup, - UTSMode: string(createArtifact.UtsMode), - UsernsMode: string(createArtifact.UsernsMode), - GroupAdd: spec.Process.User.AdditionalGids, - ContainerIDFile: createArtifact.CidFile, - AutoRemove: createArtifact.Rm, - CapAdd: createArtifact.CapAdd, - CapDrop: createArtifact.CapDrop, - DNS: createArtifact.DNSServers, - DNSOptions: createArtifact.DNSOpt, - DNSSearch: createArtifact.DNSSearch, - PidMode: string(createArtifact.PidMode), - CgroupParent: createArtifact.CgroupParent, - ShmSize: createArtifact.Resources.ShmSize, - Memory: createArtifact.Resources.Memory, - Ulimits: createArtifact.Resources.Ulimit, - SecurityOpt: createArtifact.SecurityOpts, - Tmpfs: createArtifact.Tmpfs, - }, - &inspect.CtrConfig{ - Hostname: spec.Hostname, - User: spec.Process.User, - Env: spec.Process.Env, - Image: config.RootfsImageName, - WorkingDir: spec.Process.Cwd, - Labels: config.Labels, - Annotations: spec.Annotations, - Tty: spec.Process.Terminal, - OpenStdin: config.Stdin, - StopSignal: config.StopSignal, - Cmd: config.Spec.Process.Args, - Entrypoint: strings.Join(createArtifact.Entrypoint, " "), - Healthcheck: config.HealthCheckConfig, - }, - } - return data, nil -} - -func getCPUInfo(spec *specs.Spec) (string, string, *uint64, *int64, *uint64, *int64, *uint64) { - if spec.Linux.Resources == nil { - return "", "", nil, nil, nil, nil, nil - } - cpu := spec.Linux.Resources.CPU - if cpu == nil { - return "", "", nil, nil, nil, nil, nil - } - return cpu.Cpus, cpu.Mems, cpu.Period, cpu.Quota, cpu.RealtimePeriod, cpu.RealtimeRuntime, cpu.Shares -} - -func getBLKIOInfo(spec *specs.Spec) (*uint16, []specs.LinuxWeightDevice, []specs.LinuxThrottleDevice, []specs.LinuxThrottleDevice, []specs.LinuxThrottleDevice, []specs.LinuxThrottleDevice) { - if spec.Linux.Resources == nil { - return nil, nil, nil, nil, nil, nil - } - blkio := spec.Linux.Resources.BlockIO - if blkio == nil { - return nil, nil, nil, nil, nil, nil - } - return blkio.Weight, blkio.WeightDevice, blkio.ThrottleReadBpsDevice, blkio.ThrottleWriteBpsDevice, blkio.ThrottleReadIOPSDevice, blkio.ThrottleWriteIOPSDevice -} - -func getMemoryInfo(spec *specs.Spec) (*int64, *int64, *int64, *uint64, *bool) { - if spec.Linux.Resources == nil { - return nil, nil, nil, nil, nil - } - memory := spec.Linux.Resources.Memory - if memory == nil { - return nil, nil, nil, nil, nil - } - return memory.Kernel, memory.Reservation, memory.Swap, memory.Swappiness, memory.DisableOOMKiller -} - -func getPidsInfo(spec *specs.Spec) *int64 { - if spec.Linux.Resources == nil { - return nil - } - pids := spec.Linux.Resources.Pids - if pids == nil { - return nil - } - return &pids.Limit -} - -func getCgroup(spec *specs.Spec) string { - cgroup := "host" - for _, ns := range spec.Linux.Namespaces { - if ns.Type == specs.CgroupNamespace && ns.Path != "" { - cgroup = "container" - } - } - return cgroup -} - func comparePorts(i, j ocicni.PortMapping) bool { if i.ContainerPort != j.ContainerPort { return i.ContainerPort < j.ContainerPort diff --git a/cmd/podman/shared/container_inspect.go b/cmd/podman/shared/container_inspect.go new file mode 100644 index 000000000..97a1d0238 --- /dev/null +++ b/cmd/podman/shared/container_inspect.go @@ -0,0 +1,255 @@ +package shared + +import ( + "strings" + + "github.com/containers/image/manifest" + "github.com/containers/libpod/libpod" + cc "github.com/containers/libpod/pkg/spec" + "github.com/docker/go-connections/nat" + specs "github.com/opencontainers/runtime-spec/specs-go" +) + +// InspectContainer holds all inspect data for a container. +// The format of individual components is fixed so the overall structure, when +// JSON encoded, matches the output of `docker inspect`. +// It combines Libpod-source inspect data with Podman-specific inspect data. +type InspectContainer struct { + *libpod.InspectContainerData + HostConfig *InspectContainerHostConfig `json:"HostConfig"` + Config *InspectContainerConfig `json:"Config"` +} + +// InspectContainerHostConfig holds Container configuration that is not specific +// to Libpod. This information is (mostly) stored by Podman as an artifact. +// This struct is matched to the output of `docker inspect`. +type InspectContainerHostConfig struct { + ContainerIDFile string `json:"ContainerIDFile"` + LogConfig *InspectLogConfig `json:"LogConfig"` //TODO + NetworkMode string `json:"NetworkMode"` + PortBindings nat.PortMap `json:"PortBindings"` //TODO + AutoRemove bool `json:"AutoRemove"` + CapAdd []string `json:"CapAdd"` + CapDrop []string `json:"CapDrop"` + DNS []string `json:"DNS"` + DNSOptions []string `json:"DNSOptions"` + DNSSearch []string `json:"DNSSearch"` + ExtraHosts []string `json:"ExtraHosts"` + GroupAdd []uint32 `json:"GroupAdd"` + IpcMode string `json:"IpcMode"` + Cgroup string `json:"Cgroup"` + OomScoreAdj *int `json:"OomScoreAdj"` + PidMode string `json:"PidMode"` + Privileged bool `json:"Privileged"` + PublishAllPorts bool `json:"PublishAllPorts"` //TODO + ReadOnlyRootfs bool `json:"ReadonlyRootfs"` + ReadOnlyTmpfs bool `json:"ReadonlyTmpfs"` + SecurityOpt []string `json:"SecurityOpt"` + UTSMode string `json:"UTSMode"` + UsernsMode string `json:"UsernsMode"` + ShmSize int64 `json:"ShmSize"` + Runtime string `json:"Runtime"` + ConsoleSize *specs.Box `json:"ConsoleSize"` + CPUShares *uint64 `json:"CpuShares"` + Memory int64 `json:"Memory"` + NanoCPUs int `json:"NanoCpus"` + CgroupParent string `json:"CgroupParent"` + BlkioWeight *uint16 `json:"BlkioWeight"` + BlkioWeightDevice []specs.LinuxWeightDevice `json:"BlkioWeightDevice"` + BlkioDeviceReadBps []specs.LinuxThrottleDevice `json:"BlkioDeviceReadBps"` + BlkioDeviceWriteBps []specs.LinuxThrottleDevice `json:"BlkioDeviceWriteBps"` + BlkioDeviceReadIOps []specs.LinuxThrottleDevice `json:"BlkioDeviceReadIOps"` + BlkioDeviceWriteIOps []specs.LinuxThrottleDevice `json:"BlkioDeviceWriteIOps"` + CPUPeriod *uint64 `json:"CpuPeriod"` + CPUQuota *int64 `json:"CpuQuota"` + CPURealtimePeriod *uint64 `json:"CpuRealtimePeriod"` + CPURealtimeRuntime *int64 `json:"CpuRealtimeRuntime"` + CPUSetCPUs string `json:"CpuSetCpus"` + CPUSetMems string `json:"CpuSetMems"` + Devices []specs.LinuxDevice `json:"Devices"` + DiskQuota int `json:"DiskQuota"` //check type, TODO + KernelMemory *int64 `json:"KernelMemory"` + MemoryReservation *int64 `json:"MemoryReservation"` + MemorySwap *int64 `json:"MemorySwap"` + MemorySwappiness *uint64 `json:"MemorySwappiness"` + OomKillDisable *bool `json:"OomKillDisable"` + PidsLimit *int64 `json:"PidsLimit"` + Ulimits []string `json:"Ulimits"` + CPUCount int `json:"CpuCount"` + CPUPercent int `json:"CpuPercent"` + IOMaximumIOps int `json:"IOMaximumIOps"` //check type, TODO + IOMaximumBandwidth int `json:"IOMaximumBandwidth"` //check type, TODO + Tmpfs []string `json:"Tmpfs"` +} + +// InspectContainerConfig holds further data about a container, again mostly +// not directly stored in Libpod. This struct is matched to the output of +// `docker inspect`. +type InspectContainerConfig struct { + Hostname string `json:"Hostname"` + DomainName string `json:"Domainname"` //TODO + User specs.User `json:"User"` + AttachStdin bool `json:"AttachStdin"` //TODO + AttachStdout bool `json:"AttachStdout"` //TODO + AttachStderr bool `json:"AttachStderr"` //TODO + Tty bool `json:"Tty"` + OpenStdin bool `json:"OpenStdin"` + StdinOnce bool `json:"StdinOnce"` //TODO + Env []string `json:"Env"` + Cmd []string `json:"Cmd"` + Image string `json:"Image"` + Volumes map[string]struct{} `json:"Volumes"` + WorkingDir string `json:"WorkingDir"` + Entrypoint string `json:"Entrypoint"` + Labels map[string]string `json:"Labels"` + Annotations map[string]string `json:"Annotations"` + StopSignal uint `json:"StopSignal"` + Healthcheck *manifest.Schema2HealthConfig `json:"Healthcheck,omitempty"` +} + +// InspectLogConfig holds information about a container's configured log driver +// and is presently unused. It is retained for Docker compatability. +type InspectLogConfig struct { + Type string `json:"Type"` + Config map[string]string `json:"Config"` //idk type, TODO +} + +// GetCtrInspectInfo inspects a container, combining Libpod inspect information +// with other information not stored in Libpod and returning a struct that, when +// formatted for JSON output, is compatible with `docker inspect`. +func GetCtrInspectInfo(config *libpod.ContainerConfig, ctrInspectData *libpod.InspectContainerData, createArtifact *cc.CreateConfig) (*InspectContainer, error) { + spec := config.Spec + + cpus, mems, period, quota, realtimePeriod, realtimeRuntime, shares := getCPUInfo(spec) + blkioWeight, blkioWeightDevice, blkioReadBps, blkioWriteBps, blkioReadIOPS, blkioeWriteIOPS := getBLKIOInfo(spec) + memKernel, memReservation, memSwap, memSwappiness, memDisableOOMKiller := getMemoryInfo(spec) + pidsLimit := getPidsInfo(spec) + cgroup := getCgroup(spec) + logConfig := InspectLogConfig{ + config.LogDriver, + make(map[string]string), + } + + data := &InspectContainer{ + ctrInspectData, + &InspectContainerHostConfig{ + ConsoleSize: spec.Process.ConsoleSize, + OomScoreAdj: spec.Process.OOMScoreAdj, + CPUShares: shares, + BlkioWeight: blkioWeight, + BlkioWeightDevice: blkioWeightDevice, + BlkioDeviceReadBps: blkioReadBps, + BlkioDeviceWriteBps: blkioWriteBps, + BlkioDeviceReadIOps: blkioReadIOPS, + BlkioDeviceWriteIOps: blkioeWriteIOPS, + CPUPeriod: period, + CPUQuota: quota, + CPURealtimePeriod: realtimePeriod, + CPURealtimeRuntime: realtimeRuntime, + CPUSetCPUs: cpus, + CPUSetMems: mems, + Devices: spec.Linux.Devices, + KernelMemory: memKernel, + LogConfig: &logConfig, + MemoryReservation: memReservation, + MemorySwap: memSwap, + MemorySwappiness: memSwappiness, + OomKillDisable: memDisableOOMKiller, + PidsLimit: pidsLimit, + Privileged: config.Privileged, + ReadOnlyRootfs: spec.Root.Readonly, + ReadOnlyTmpfs: createArtifact.ReadOnlyTmpfs, + Runtime: config.OCIRuntime, + NetworkMode: string(createArtifact.NetMode), + IpcMode: string(createArtifact.IpcMode), + Cgroup: cgroup, + UTSMode: string(createArtifact.UtsMode), + UsernsMode: string(createArtifact.UsernsMode), + GroupAdd: spec.Process.User.AdditionalGids, + ContainerIDFile: createArtifact.CidFile, + AutoRemove: createArtifact.Rm, + CapAdd: createArtifact.CapAdd, + CapDrop: createArtifact.CapDrop, + DNS: createArtifact.DNSServers, + DNSOptions: createArtifact.DNSOpt, + DNSSearch: createArtifact.DNSSearch, + PidMode: string(createArtifact.PidMode), + CgroupParent: createArtifact.CgroupParent, + ShmSize: createArtifact.Resources.ShmSize, + Memory: createArtifact.Resources.Memory, + Ulimits: createArtifact.Resources.Ulimit, + SecurityOpt: createArtifact.SecurityOpts, + Tmpfs: createArtifact.Tmpfs, + }, + &InspectContainerConfig{ + Hostname: spec.Hostname, + User: spec.Process.User, + Env: spec.Process.Env, + Image: config.RootfsImageName, + WorkingDir: spec.Process.Cwd, + Labels: config.Labels, + Annotations: spec.Annotations, + Tty: spec.Process.Terminal, + OpenStdin: config.Stdin, + StopSignal: config.StopSignal, + Cmd: config.Spec.Process.Args, + Entrypoint: strings.Join(createArtifact.Entrypoint, " "), + Healthcheck: config.HealthCheckConfig, + }, + } + return data, nil +} + +func getCPUInfo(spec *specs.Spec) (string, string, *uint64, *int64, *uint64, *int64, *uint64) { + if spec.Linux.Resources == nil { + return "", "", nil, nil, nil, nil, nil + } + cpu := spec.Linux.Resources.CPU + if cpu == nil { + return "", "", nil, nil, nil, nil, nil + } + return cpu.Cpus, cpu.Mems, cpu.Period, cpu.Quota, cpu.RealtimePeriod, cpu.RealtimeRuntime, cpu.Shares +} + +func getBLKIOInfo(spec *specs.Spec) (*uint16, []specs.LinuxWeightDevice, []specs.LinuxThrottleDevice, []specs.LinuxThrottleDevice, []specs.LinuxThrottleDevice, []specs.LinuxThrottleDevice) { + if spec.Linux.Resources == nil { + return nil, nil, nil, nil, nil, nil + } + blkio := spec.Linux.Resources.BlockIO + if blkio == nil { + return nil, nil, nil, nil, nil, nil + } + return blkio.Weight, blkio.WeightDevice, blkio.ThrottleReadBpsDevice, blkio.ThrottleWriteBpsDevice, blkio.ThrottleReadIOPSDevice, blkio.ThrottleWriteIOPSDevice +} + +func getMemoryInfo(spec *specs.Spec) (*int64, *int64, *int64, *uint64, *bool) { + if spec.Linux.Resources == nil { + return nil, nil, nil, nil, nil + } + memory := spec.Linux.Resources.Memory + if memory == nil { + return nil, nil, nil, nil, nil + } + return memory.Kernel, memory.Reservation, memory.Swap, memory.Swappiness, memory.DisableOOMKiller +} + +func getPidsInfo(spec *specs.Spec) *int64 { + if spec.Linux.Resources == nil { + return nil + } + pids := spec.Linux.Resources.Pids + if pids == nil { + return nil + } + return &pids.Limit +} + +func getCgroup(spec *specs.Spec) string { + cgroup := "host" + for _, ns := range spec.Linux.Namespaces { + if ns.Type == specs.CgroupNamespace && ns.Path != "" { + cgroup = "container" + } + } + return cgroup +} diff --git a/cmd/podman/shared/create.go b/cmd/podman/shared/create.go index 3c9b17804..7cf230605 100644 --- a/cmd/podman/shared/create.go +++ b/cmd/podman/shared/create.go @@ -603,6 +603,11 @@ func ParseCreateOpts(ctx context.Context, c *GenericCLIResults, runtime *libpod. memorySwappiness := c.Int64("memory-swappiness") + logDriver := libpod.KubernetesLogging + if c.Changed("log-driver") { + logDriver = c.String("log-driver") + } + config := &cc.CreateConfig{ Annotations: annotations, BuiltinImgVolumes: ImageVolumes, @@ -635,7 +640,7 @@ func ParseCreateOpts(ctx context.Context, c *GenericCLIResults, runtime *libpod. IPAddress: c.String("ip"), Labels: labels, //LinkLocalIP: c.StringSlice("link-local-ip"), // Not implemented yet - LogDriver: c.String("log-driver"), + LogDriver: logDriver, LogDriverOpt: c.StringSlice("log-opt"), MacAddress: c.String("mac-address"), Name: c.String("name"), diff --git a/cmd/podman/varlink/io.podman.varlink b/cmd/podman/varlink/io.podman.varlink index ed7b49c68..5b3d5ae4c 100644 --- a/cmd/podman/varlink/io.podman.varlink +++ b/cmd/podman/varlink/io.podman.varlink @@ -802,8 +802,8 @@ method DeleteUnusedImages() -> (images: []string) # attributes: _CMD, ENTRYPOINT, ENV, EXPOSE, LABEL, ONBUILD, STOPSIGNAL, USER, VOLUME, and WORKDIR_. To pause the # container while it is being committed, pass a _true_ bool for the pause argument. If the container cannot # be found by the ID or name provided, a (ContainerNotFound)[#ContainerNotFound] error will be returned; otherwise, -# the resulting image's ID will be returned as a string. -method Commit(name: string, image_name: string, changes: []string, author: string, message: string, pause: bool, manifestType: string) -> (image: string) +# the resulting image's ID will be returned as a string inside a MoreResponse. +method Commit(name: string, image_name: string, changes: []string, author: string, message: string, pause: bool, manifestType: string) -> (reply: MoreResponse) # ImportImage imports an image from a source (like tarball) into local storage. The image can have additional # descriptions added to it using the message and changes options. See also [ExportImage](ExportImage). diff --git a/completions/bash/podman b/completions/bash/podman index 60d5fde52..efb8a6a9b 100644 --- a/completions/bash/podman +++ b/completions/bash/podman @@ -486,6 +486,7 @@ __podman_complete_log_drivers() { none splunk syslog + k8s-file " -- "$cur" ) ) } @@ -500,6 +501,7 @@ __podman_complete_log_options() { local logentries_options="logentries-token" local syslog_options="env labels syslog-address syslog-facility syslog-format syslog-tls-ca-cert syslog-tls-cert syslog-tls-key syslog-tls-skip-verify tag" local splunk_options="env labels splunk-caname splunk-capath splunk-format splunk-gzip splunk-gzip-level splunk-index splunk-insecureskipverify splunk-source splunk-sourcetype splunk-token splunk-url splunk-verify-connection tag" + local k8s_file_options="env labels max-file max-size" local all_options="$fluentd_options $gcplogs_options $gelf_options $journald_options $logentries_options $json_file_options $syslog_options $splunk_options" @@ -525,6 +527,9 @@ __podman_complete_log_options() { json-file) COMPREPLY=( $( compgen -W "$json_file_options" -S = -- "$cur" ) ) ;; + k8s-file) + COMPREPLY=( $( compgen -W "$k8s_file_options" -S = -- "$cur" ) ) + ;; logentries) COMPREPLY=( $( compgen -W "$logentries_options" -S = -- "$cur" ) ) ;; @@ -737,6 +742,10 @@ _podman_container_attach() { } _podman_container_checkpoint() { + local options_with_args=" + -e + --export + " local boolean_options=" -a --all @@ -750,9 +759,15 @@ _podman_container_checkpoint() { --leave-running --tcp-established " + case "$prev" in + -e|--export) + _filedir + return + ;; + esac case "$cur" in -*) - COMPREPLY=($(compgen -W "$boolean_options" -- "$cur")) + COMPREPLY=($(compgen -W "$boolean_options $options_with_args" -- "$cur")) ;; *) __podman_complete_containers_running @@ -764,6 +779,10 @@ _podman_container_commit() { _podman_commit } +_podman_container_cp() { + _podman_cp +} + _podman_container_create() { _podman_create } @@ -835,6 +854,12 @@ _podman_container_restart() { } _podman_container_restore() { + local options_with_args=" + -i + --import + -n + --name + " local boolean_options=" -a --all @@ -846,9 +871,15 @@ _podman_container_restore() { --latest --tcp-established " + case "$prev" in + -i|--import) + _filedir + return + ;; + esac case "$cur" in -*) - COMPREPLY=($(compgen -W "$boolean_options" -- "$cur")) + COMPREPLY=($(compgen -W "$boolean_options $options_with_args" -- "$cur")) ;; *) __podman_complete_containers_created @@ -961,6 +992,7 @@ _podman_container() { attach checkpoint commit + cp create diff exec @@ -2378,6 +2410,7 @@ _podman_cp() { --help -h --extract + --pause " _complete_ "$boolean_options" } diff --git a/contrib/cirrus/README.md b/contrib/cirrus/README.md index 69d8653fe..94494a558 100644 --- a/contrib/cirrus/README.md +++ b/contrib/cirrus/README.md @@ -63,41 +63,6 @@ task (pass or fail) is set based on the exit status of the last script to execut Total execution time is capped at 2-hours (includes all the above) but this script normally completes in less than an hour. -### ``special_testing`` Task - -This task exercises podman under specialized environments or conditions. -The specific differences from the ``testing`` task depend upon the -contents of the ``$SPECIALMODE`` environment variable. - -| Value | Meaning | -| rootless | Setup a regular user to build/run integration tests. | -| in_podman | Setup a container image, build/run integration tests inside container | - -***N/B: Steps below are performed by automation*** - -1. After `gating` passes, spin up one VM per - `matrix: image_name` item. - -2. ``setup_environment.sh``: Mostly the same as - in ``testing`` task, then specialized depending on ``$SPECIALMODE``. - -3. Which tests and how they execute depends on ``$SPECIALMODE``. - - -### ``optional_testing`` Task - -***N/B: Steps below are performed by automation*** - -1. Optionally executes in parallel with ``testing``. Requires - **prior** to job-start, the magic string ``***CIRRUS: SYSTEM TEST***`` - is found in the pull-request *description*. The *description* is the first - text-box under the main *summary* line in the github WebUI. - -2. ``setup_environment.sh``: Same as for other tasks. - -3. ``system_test.sh``: Build both dependencies and libpod, install them, - then execute `make localsystem` from the repository root. - ### ``test_build_cache_images_task`` Task diff --git a/contrib/cirrus/container_test.sh b/contrib/cirrus/container_test.sh index 1fd9551db..27baf0ad7 100644 --- a/contrib/cirrus/container_test.sh +++ b/contrib/cirrus/container_test.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -xeuo pipefail +set -xeo pipefail export GOPATH=/var/tmp/go export PATH=$HOME/gopath/bin:$PATH:$GOPATH/bin @@ -32,10 +32,10 @@ integrationtest=0 unittest=0 validate=0 options=0 -noremote=0 +remote=0 install_tools_made=0 -while getopts "biptuv" opt; do +while getopts "bituv" opt; do case "$opt" in b) build=1 options=1 @@ -46,9 +46,6 @@ while getopts "biptuv" opt; do t) integrationtest=1 options=1 ;; - n) noremote=1 - options=1 - ;; u) unittest=1 options=1 ;; @@ -58,6 +55,12 @@ while getopts "biptuv" opt; do esac done +# The TEST_REMOTE_CLIENT environment variable decides whether +# to test varlinke +if [[ "$TEST_REMOTE_CLIENT" == "true" ]]; then + remote=1 +fi + # If no options are passed, do everything if [ $options -eq 0 ]; then build=1 @@ -130,8 +133,8 @@ fi if [ $integrationtest -eq 1 ]; then make TAGS="${TAGS}" test-binaries make varlink_generate - make ginkgo $INTEGRATION_TEST_ENVS - if [ $noremote -eq 0 ]; then - make ginkgo-remote $INTEGRATION_TEST_ENVS + make localintegration $INTEGRATION_TEST_ENVS + if [ $remote -eq 1 ]; then + make remoteintegration $INTEGRATION_TEST_ENVS fi fi diff --git a/contrib/cirrus/integration_test.sh b/contrib/cirrus/integration_test.sh index e7f582b42..f9ba010cd 100755 --- a/contrib/cirrus/integration_test.sh +++ b/contrib/cirrus/integration_test.sh @@ -5,6 +5,14 @@ source $(dirname $0)/lib.sh req_env_var GOSRC SCRIPT_BASE OS_RELEASE_ID OS_RELEASE_VER CONTAINER_RUNTIME +# Our name must be of the form xxxx_test or xxxx_test.sh, where xxxx is +# the test suite to run; currently (2019-05) the only option is 'integration' +# but pr2947 intends to add 'system'. +TESTSUITE=$(expr $(basename $0) : '\(.*\)_test') +if [[ -z $TESTSUITE ]]; then + die 1 "Script name is not of the form xxxx_test.sh" +fi + cd "$GOSRC" if [[ "$SPECIALMODE" == "in_podman" ]] @@ -19,7 +27,7 @@ then -e "CONMON_BINARY=/usr/libexec/podman/conmon" \ -e "DIST=$OS_RELEASE_ID" \ -e "CONTAINER_RUNTIME=$CONTAINER_RUNTIME" \ - ${OS_RELEASE_ID}podmanbuild bash $GOSRC/$SCRIPT_BASE/container_test.sh -b -i -t -n + $IN_PODMAN_IMAGE bash $GOSRC/$SCRIPT_BASE/container_test.sh -b -i -t exit $? elif [[ "$SPECIALMODE" == "rootless" ]] @@ -28,11 +36,11 @@ then if [[ "$USER" == "$ROOTLESS_USER" ]] then - $GOSRC/$SCRIPT_BASE/rootless_test.sh + $GOSRC/$SCRIPT_BASE/rootless_test.sh ${TESTSUITE} else ssh $ROOTLESS_USER@localhost \ -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o CheckHostIP=no \ - $GOSRC/$SCRIPT_BASE/rootless_test.sh + $GOSRC/$SCRIPT_BASE/rootless_test.sh ${TESTSUITE} fi else make @@ -40,9 +48,9 @@ else make test-binaries if [[ "$TEST_REMOTE_CLIENT" == "true" ]] then - make remoteintegration + make remote${TESTSUITE} else - make localintegration + make local${TESTSUITE} fi exit $? fi diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh index 334202aa9..462fa332a 100644 --- a/contrib/cirrus/lib.sh +++ b/contrib/cirrus/lib.sh @@ -66,7 +66,7 @@ PRIOR_FEDORA_BASE_IMAGE="fedora-cloud-base-28-1-1-1544474897" BUILT_IMAGE_SUFFIX="${BUILT_IMAGE_SUFFIX:--$CIRRUS_REPO_NAME-${CIRRUS_BUILD_ID}}" # Safe env. vars. to transfer from root -> $ROOTLESS_USER (go env handled separetly) -ROOTLESS_ENV_RE='(CIRRUS_.+)|(ROOTLESS_.+)|(.+_IMAGE.*)|(.+_BASE)|(.*DIRPATH)|(.*FILEPATH)|(SOURCE.*)|(DEPEND.*)|(.+_DEPS_.+)|(OS_REL.*)|(.+_ENV_RE)|(TRAVIS)|(CI.+)' +ROOTLESS_ENV_RE='(CIRRUS_.+)|(ROOTLESS_.+)|(.+_IMAGE.*)|(.+_BASE)|(.*DIRPATH)|(.*FILEPATH)|(SOURCE.*)|(DEPEND.*)|(.+_DEPS_.+)|(OS_REL.*)|(.+_ENV_RE)|(TRAVIS)|(CI.+)|(TEST_REMOTE.*)' # Unsafe env. vars for display SECRET_ENV_RE='(IRCID)|(ACCOUNT)|(^GC[EP]..+)|(SSH)' @@ -74,6 +74,9 @@ SPECIALMODE="${SPECIALMODE:-none}" TEST_REMOTE_CLIENT="${TEST_REMOTE_CLIENT:-false}" export CONTAINER_RUNTIME=${CONTAINER_RUNTIME:-podman} +# IN_PODMAN container image +IN_PODMAN_IMAGE="quay.io/libpod/in_podman:latest" + # When running as root, this may be empty or not, as a user, it MUST be set. if [[ "$USER" == "root" ]] then diff --git a/contrib/cirrus/rootless_test.sh b/contrib/cirrus/rootless_test.sh index 3b668034b..b5744671b 100755 --- a/contrib/cirrus/rootless_test.sh +++ b/contrib/cirrus/rootless_test.sh @@ -2,6 +2,14 @@ set -e +remote=0 + +# The TEST_REMOTE_CLIENT environment variable decides whether +# to test varlinke +if [[ "$TEST_REMOTE_CLIENT" == "true" ]]; then + remote=1 +fi + source $(dirname $0)/lib.sh if [[ "$UID" == "0" ]] @@ -10,6 +18,12 @@ then exit 1 fi +# Which set of tests to run; possible alternative is "system" +TESTSUITE=integration +if [[ -n "$*" ]]; then + TESTSUITE="$1" +fi + # Ensure environment setup correctly req_env_var GOSRC ROOTLESS_USER @@ -25,5 +39,8 @@ cd "$GOSRC" make make varlink_generate make test-binaries -make ginkgo -make ginkgo-remote +if [ $remote -eq 0 ]; then + make local${TESTSUITE} +else + make remote${TESTSUITE} +fi diff --git a/contrib/cirrus/setup_container_environment.sh b/contrib/cirrus/setup_container_environment.sh index eda6f6167..c268c162e 100755 --- a/contrib/cirrus/setup_container_environment.sh +++ b/contrib/cirrus/setup_container_environment.sh @@ -5,9 +5,6 @@ source $(dirname $0)/lib.sh req_env_var GOSRC OS_RELEASE_ID CONTAINER_RUNTIME -DIST=$OS_RELEASE_ID -IMAGE=${DIST}podmanbuild - # Since CRIU 3.11 has been pushed to Fedora 28 the checkpoint/restore # test cases are actually run. As CRIU uses iptables to lock and unlock # the network during checkpoint and restore it needs the following two @@ -15,5 +12,5 @@ IMAGE=${DIST}podmanbuild modprobe ip6table_nat || : modprobe iptable_nat || : -# Build the test image -${CONTAINER_RUNTIME} build -t ${IMAGE} -f Dockerfile.${DIST} . +# Pull the test image +${CONTAINER_RUNTIME} pull ${IN_PODMAN_IMAGE} diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh index f40405e8d..13bce506a 100755 --- a/contrib/cirrus/setup_environment.sh +++ b/contrib/cirrus/setup_environment.sh @@ -87,7 +87,7 @@ case "$SPECIALMODE" in fi ;; in_podman) # Assumed to be Fedora - dnf install -y podman buildah + dnf install -y podman $SCRIPT_BASE/setup_container_environment.sh ;; *) diff --git a/contrib/imgts/entrypoint.sh b/contrib/imgts/entrypoint.sh index 65a76d8e4..610e1f3b6 100755 --- a/contrib/imgts/entrypoint.sh +++ b/contrib/imgts/entrypoint.sh @@ -32,6 +32,7 @@ ARGS="--update-labels=last-used=$(date +%s)" # optional [[ -z "$BUILDID" ]] || ARGS="$ARGS --update-labels=build-id=$BUILDID" [[ -z "$REPOREF" ]] || ARGS="$ARGS --update-labels=repo-ref=$REPOREF" +[[ -z "$GCPPROJECT" ]] || ARGS="$ARGS --update-labels=project=$GCPPROJECT" gcloud config set account "$GCPNAME" gcloud config set project "$GCPPROJECT" diff --git a/contrib/podmanimage/README.md b/contrib/podmanimage/README.md new file mode 100644 index 000000000..79484d4a3 --- /dev/null +++ b/contrib/podmanimage/README.md @@ -0,0 +1,44 @@ + + +# podmanimage + +## Overview + +This directory contains the Dockerfiles necessary to create the three podmanimage container +images that are housed on quay.io under the podman account. All three repositories where +the images live are public and can be pulled without credentials. These container images are secured and the +resulting containers can run safely with privileges within the container. The container images are built +using the latest Fedora and then Podman is installed into them: + + * quay.io/podman/stable - This image is built using the latest stable version of Podman in a Fedora based container. Built with podman/stable/Dockerfile. + * quay.io/podman/upstream - This image is built using the latest code found in this GitHub repository. When someone creates a commit and pushes it, the image is created. Due to that the image changes frequently and is not guaranteed to be stable. Built with podmanimage/upstream/Dockerfile. + * quay.io/podman/testing - This image is built using the latest version of Podman that is or was in updates testing for Fedora. At times this may be the same as the stable image. This container image will primarily be used by the development teams for verification testing when a new package is created. Built with podmanimage/testing/Dockerfile. + +## Sample Usage + + +``` +podman pull docker://quay.io/podman/stable:latest + +podman run --privileged stable podman version + +# Create a directory on the host to mount the container's +# /var/lib/container directory to so containers can be +# run within the container. +mkdir /var/lib/mycontainer + +# Run the image detached using the host's network in a container name +# podmanctr, turn off label and seccomp confinement in the container +# and then do a little shell hackery to keep the container up and running. +podman run --detach --name=podmanctr --net=host --security-opt label=disable --security-opt seccomp=unconfined --device /dev/fuse:rw -v /var/lib/mycontainer:/var/lib/containers:Z --privileged stable sh -c 'while true ;do wait; done' + +podman exec -it podmanctr /bin/sh + +# Now inside of the container + +podman pull alpine + +podman images + +exit +``` diff --git a/contrib/podmanimage/stable/Dockerfile b/contrib/podmanimage/stable/Dockerfile new file mode 100644 index 000000000..056f62624 --- /dev/null +++ b/contrib/podmanimage/stable/Dockerfile @@ -0,0 +1,26 @@ +# stable/Dockerfile +# +# Build a Podman container image from the latest +# stable version of Podman on the Fedoras Updates System. +# https://bodhi.fedoraproject.org/updates/?search=podman +# This image can be used to create a secured container +# that runs safely with privileges within the container. +# +FROM fedora:latest + +# Don't include container-selinux and remove +# directories used by dnf that are just taking +# up space. +RUN yum -y install podman fuse-overlayfs --exclude container-selinux; rm -rf /var/cache /var/log/dnf* /var/log/yum.* + +# Adjust storage.conf to enable Fuse storage. +RUN sed -i -e 's|^#mount_program|mount_program|g' -e '/additionalimage.*/a "/var/lib/shared",' /etc/containers/storage.conf +RUN mkdir -p /var/lib/shared/overlay-images /var/lib/shared/overlay-layers; touch /var/lib/shared/overlay-images/images.lock; touch /var/lib/shared/overlay-layers/layers.lock + +# Adjust libpod.conf to write logging to a file +RUN sed -i 's/events_logger = "journald"/events_logger = "file"/g' /usr/share/containers/libpod.conf + +# Set up environment variables to note that this is +# not starting with usernamespace and default to +# isolate the filesystem with chroot. +ENV _BUILDAH_STARTED_IN_USERNS="" BUILDAH_ISOLATION=chroot diff --git a/contrib/podmanimage/testing/Dockerfile b/contrib/podmanimage/testing/Dockerfile new file mode 100644 index 000000000..50d8ed7f2 --- /dev/null +++ b/contrib/podmanimage/testing/Dockerfile @@ -0,0 +1,28 @@ +# testing/Dockerfile +# +# Build a Podman image using the latest +# version of Podman that is in updates-testing +# on the Fedoras Updates System. At times this +# may be the same the latest stable version. +# https://bodhi.fedoraproject.org/updates/?search=podman +# This image can be used to create a secured container +# that runs safely with privileges within the container. +# +FROM fedora:latest + +# Don't include container-selinux and remove +# directories used by dnf that are just taking +# up space. +RUN yum -y install podman fuse-overlayfs --exclude container-selinux --enablerepo updates-testing; rm -rf /var/cache /var/log/dnf* /var/log/yum.* + +# Adjust storage.conf to enable Fuse storage. +RUN sed -i -e 's|^#mount_program|mount_program|g' -e '/additionalimage.*/a "/var/lib/shared",' /etc/containers/storage.conf +RUN mkdir -p /var/lib/shared/overlay-images /var/lib/shared/overlay-layers; touch /var/lib/shared/overlay-images/images.lock; touch /var/lib/shared/overlay-layers/layers.lock + +# Adjust libpod.conf to write logging to a file +RUN sed -i 's/events_logger = "journald"/events_logger = "file"/g' /usr/share/containers/libpod.conf + +# Set up environment variables to note that this is +# not starting with usernamespace and default to +# isolate the filesystem with chroot. +ENV _BUILDAH_STARTED_IN_USERNS="" BUILDAH_ISOLATION=chroot diff --git a/contrib/podmanimage/upstream/Dockerfile b/contrib/podmanimage/upstream/Dockerfile new file mode 100644 index 000000000..3583e1c54 --- /dev/null +++ b/contrib/podmanimage/upstream/Dockerfile @@ -0,0 +1,77 @@ +# git/Dockerfile +# +# Build a Podman container image from the latest +# upstream version of Podman on GitHub. +# https://github.com/containers/libpod +# This image can be used to create a secured container +# that runs safely with privileges within the container. +# The containers created by this image also come with a +# Podman development environment in /root/podman. +# +FROM fedora:latest +ENV GOPATH=/root/podman + +# Install the software required to build Podman. +# Then create a directory and clone from the Podman +# GitHub repository, make and install Podman +# to the container. +# Finally remove the podman directory and a few other packages +# that are needed for building but not running Podman +RUN dnf -y install --exclude container-selinux \ + --enablerepo=updates-testing \ + atomic-registries \ + btrfs-progs-devel \ + containernetworking-cni \ + device-mapper-devel \ + git \ + glib2-devel \ + glibc-devel \ + glibc-static \ + go \ + golang-github-cpuguy83-go-md2man \ + gpgme-devel \ + iptables \ + libassuan-devel \ + libgpg-error-devel \ + libseccomp-devel \ + libselinux-devel \ + make \ + ostree-devel \ + pkgconfig \ + runc \ + fuse-overlayfs \ + fuse3 \ + containers-common; \ + mkdir /root/podman; \ + git clone https://github.com/containers/libpod /root/podman/src/github.com/containers/libpod; \ + cd /root/podman/src/github.com/containers/libpod; \ + make BUILDTAGS="selinux seccomp"; \ + make install PREFIX=/usr; \ + cd /root/podman; \ + git clone https://github.com/containers/conmon /root/podman/conmon; \ + cd conmon; \ + make; \ + install -D -m 755 bin/conmon /usr/libexec/podman/conmon; \ + git clone https://github.com/containernetworking/plugins.git $GOPATH/src/github.com/containernetworking/plugins; \ + cd $GOPATH/src/github.com/containernetworking/plugins; \ + ./build_linux.sh; \ + mkdir -p /usr/libexec/cni; \ + \cp -fR bin/* /usr/libexec/cni; \ + mkdir -p /etc/cni/net.d; \ + curl -qsSL https://raw.githubusercontent.com/containers/libpod/master/cni/87-podman-bridge.conflist | tee /etc/cni/net.d/99-loopback.conf; \ + mkdir -p /usr/share/containers; \ + cp $GOPATH/src/github.com/containers/libpod/libpod.conf /usr/share/containers; \ + # Adjust libpod.conf to write logging to a file + sed -i 's/events_logger = "journald"/events_logger = "file"/g' /usr/share/containers/libpod.conf; \ + rm -rf /root/podman/*; \ + dnf -y remove git golang go-md2man make; \ + dnf clean all; + +# Adjust storage.conf to enable Fuse storage. +RUN sed -i -e 's|^#mount_program|mount_program|g' -e '/additionalimage.*/a "/var/lib/shared",' /etc/containers/storage.conf +RUN mkdir -p /var/lib/shared/overlay-images /var/lib/shared/overlay-layers; touch /var/lib/shared/overlay-images/images.lock; touch /var/lib/shared/overlay-layers/layers.lock + +# Set up environment variables to note that this is +# not starting with usernamespace and default to +# isolate the filesystem with chroot. +ENV _BUILDAH_STARTED_IN_USERNS="" BUILDAH_ISOLATION=chroot diff --git a/contrib/spec/podman.spec.in b/contrib/spec/podman.spec.in index a1c11a5a6..8c2ccd4b0 100644 --- a/contrib/spec/podman.spec.in +++ b/contrib/spec/podman.spec.in @@ -39,7 +39,7 @@ %global shortcommit_conmon %(c=%{commit_conmon}; echo ${c:0:7}) Name: podman -Version: 1.3.2 +Version: 1.4.1 Release: #COMMITDATE#.git%{shortcommit0}%{?dist} Summary: Manage Pods, Containers and Container Images License: ASL 2.0 diff --git a/docs/podman-build.1.md b/docs/podman-build.1.md index 4a26c0981..1b86992d9 100644 --- a/docs/podman-build.1.md +++ b/docs/podman-build.1.md @@ -174,6 +174,18 @@ This is a Docker specific option to disable image verification to a Docker registry and is not supported by Podman. This flag is a NOOP and provided soley for scripting compatibility. +**--dns**=[] + +Set custom DNS servers + +**--dns-option**=[] + +Set custom DNS options + +**--dns-search**=[] + +Set custom DNS search domains + **--file, -f** *Dockerfile* Specifies a Dockerfile which contains instructions for building the image, @@ -517,7 +529,7 @@ Only the current container can use a private volume. `Overlay Volume Mounts` - The `:O` flag tells Buildah to mount the directory from the host as a temporary storage using the Overlay file system. The `RUN` command containers are allowed to modify contents within the mountpoint and are stored in the container storage in a separate directory. In Ovelay FS terms the source directory will be the lower, and the container storage directory will be the upper. Modifications to the mount point are destroyed when the `RUN` command finishes executing, similar to a tmpfs mount point. + The `:O` flag tells Buildah to mount the directory from the host as a temporary storage using the Overlay file system. The `RUN` command containers are allowed to modify contents within the mountpoint and are stored in the container storage in a separate directory. In Overlay FS terms the source directory will be the lower, and the container storage directory will be the upper. Modifications to the mount point are destroyed when the `RUN` command finishes executing, similar to a tmpfs mount point. Any subsequent execution of `RUN` commands sees the original source directory content, any changes from previous RUN commands no longer exists. diff --git a/docs/podman-container-checkpoint.1.md b/docs/podman-container-checkpoint.1.md index 79dc12261..afccdf59a 100644 --- a/docs/podman-container-checkpoint.1.md +++ b/docs/podman-container-checkpoint.1.md @@ -38,6 +38,12 @@ image contains established TCP connections, this options is required during restore. Defaults to not checkpointing containers with established TCP connections. +**--export, -e** + +Export the checkpoint to a tar.gz file. The exported checkpoint can be used +to import the container on another system and thus enabling container live +migration. + ## EXAMPLE podman container checkpoint mywebserver diff --git a/docs/podman-container-restore.1.md b/docs/podman-container-restore.1.md index e41f7c1d8..5efc280fe 100644 --- a/docs/podman-container-restore.1.md +++ b/docs/podman-container-restore.1.md @@ -42,6 +42,24 @@ If the checkpoint image does not contain established TCP connections this option is ignored. Defaults to not restoring containers with established TCP connections. +**--import, -i** + +Import a checkpoint tar.gz file, which was exported by Podman. This can be used +to import a checkpointed container from another host. It is not necessary to specify +a container when restoring from an exported checkpoint. + +**--name, -n** + +This is only available in combination with **--import, -i**. If a container is restored +from a checkpoint tar.gz file it is possible to rename it with **--name, -n**. This +way it is possible to restore a container from a checkpoint multiple times with different +names. + +If the **--name, -n** option is used, Podman will not attempt to assign the same IP +address to the container it was using before checkpointing as each IP address can only +be used once and the restored container will have another IP address. This also means +that **--name, -n** cannot be used in combination with **--tcp-established**. + ## EXAMPLE podman container restore mywebserver diff --git a/docs/podman-container.1.md b/docs/podman-container.1.md index 564d791fa..eb53149bd 100644 --- a/docs/podman-container.1.md +++ b/docs/podman-container.1.md @@ -17,6 +17,7 @@ The container command allows you to manage containers | checkpoint | [podman-container-checkpoint(1)](podman-container-checkpoint.1.md) | Checkpoints one or more containers. | | cleanup | [podman-container-cleanup(1)](podman-container-cleanup.1.md) | Cleanup containers network and mountpoints. | | commit | [podman-commit(1)](podman-commit.1.md) | Create new image based on the changed container. | +| cp | [podman-cp(1)](podman-cp.1.md) | Copy files/folders between a container and the local filesystem. | | create | [podman-create(1)](podman-create.1.md) | Create a new container. | | diff | [podman-diff(1)](podman-diff.1.md) | Inspect changes on a container or image's filesystem. | | exec | [podman-exec(1)](podman-exec.1.md) | Execute a command in a running container. | diff --git a/docs/podman-cp.1.md b/docs/podman-cp.1.md index 406dd51df..ee218d66a 100644 --- a/docs/podman-cp.1.md +++ b/docs/podman-cp.1.md @@ -61,6 +61,10 @@ If you use a : in a local machine path, you must be explicit with a relative or Extract the tar file into the destination directory. If the destination directory is not provided, extract the tar file into the root directory. +**--pause** + +Pause the container while copying into it to avoid potential security issues around symlinks. Defaults to *false*. + ## ALTERNATIVES Podman has much stronger capabilities than just `podman cp` to achieve copy files between host and container. diff --git a/docs/podman-create.1.md b/docs/podman-create.1.md index cbd6d9a99..eafc6e27f 100644 --- a/docs/podman-create.1.md +++ b/docs/podman-create.1.md @@ -380,7 +380,7 @@ Read in a line delimited file of labels Not implemented -**--log-driver**="*json-file*" +**--log-driver**="*k8s-file*" Logging driver for the container. Currently not supported. This flag is a NOOP provided soley for scripting compatibility. diff --git a/docs/podman-logs.1.md b/docs/podman-logs.1.md index ce5d890ce..7feae1b76 100644 --- a/docs/podman-logs.1.md +++ b/docs/podman-logs.1.md @@ -50,7 +50,7 @@ Show timestamps in the log outputs. The default is false To view a container's logs: ``` -podman logs b3f2436bdb978c1d33b1387afb5d7ba7e3243ed2ce908db431ac0069da86cb45 +podman logs -t b3f2436bdb978c1d33b1387afb5d7ba7e3243ed2ce908db431ac0069da86cb45 2017/08/07 10:16:21 Seeked /var/log/crio/pods/eb296bd56fab164d4d3cc46e5776b54414af3bf543d138746b25832c816b933b/c49f49788da14f776b7aa93fb97a2a71f9912f4e5a3e30397fca7dfe0ee0367b.log - &{Offset:0 Whence:0} 1:C 07 Aug 14:10:09.055 # oO0OoO0OoO0Oo Redis is starting oO0OoO0OoO0Oo @@ -68,13 +68,13 @@ To view only the last two lines in container's log: ``` podman logs --tail 2 b3f2436bdb97 -1:M 07 Aug 14:10:09.056 # WARNING: The TCP backlog setting of 511 cannot be enforced because /proc/sys/net/core/somaxconn is set to the lower value of 128. -1:M 07 Aug 14:10:09.056 # Server initialized +# WARNING: The TCP backlog setting of 511 cannot be enforced because /proc/sys/net/core/somaxconn is set to the lower value of 128. +# Server initialized ``` To view a containers logs since a certain time: ``` -podman logs --since 2017-08-07T10:10:09.055837383-04:00 myserver +podman logs -t --since 2017-08-07T10:10:09.055837383-04:00 myserver 1:M 07 Aug 14:10:09.055 # Server can't set maximum open files to 10032 because of OS error: Operation not permitted. 1:M 07 Aug 14:10:09.055 # Current maximum open files is 4096. maxclients has been reduced to 4064 to compensate for low ulimit. If you need higher maxclients increase 'ulimit -n'. @@ -87,8 +87,8 @@ To view a container's logs generated in the last 10 minutes: ``` podman logs --since 10m myserver -1:M 07 Aug 14:10:09.055 # Server can't set maximum open files to 10032 because of OS error: Operation not permitted. -1:M 07 Aug 14:10:09.055 # Current maximum open files is 4096. maxclients has been reduced to 4064 to compensate for low ulimit. If you need higher maxclients increase 'ulimit -n'. +# Server can't set maximum open files to 10032 because of OS error: Operation not permitted. +# Current maximum open files is 4096. maxclients has been reduced to 4064 to compensate for low ulimit. If you need higher maxclients increase 'ulimit -n'. ``` ## SEE ALSO diff --git a/docs/podman-remote.conf.5.md b/docs/podman-remote.conf.5.md new file mode 100644 index 000000000..3e1cffb02 --- /dev/null +++ b/docs/podman-remote.conf.5.md @@ -0,0 +1,47 @@ +% podman-remote.conf(5) + +## NAME +podman-remote.conf - configuration file for the podman remote client + +## DESCRIPTION +The libpod.conf file is the default configuration file for all tools using +libpod to manage containers. + +The podman-remote.conf file is the default configuration file for the podman +remote client. It is in the TOML format. It is primarily used to keep track +of the user's remote connections. + +## CONNECTION OPTIONS +**destination** = "" + The hostname or IP address of the remote system + +**username** = "" + The username to use when connecting to the remote system + +**default** = bool + Denotes whether the connection is the default connection for the user. The default connection + is used when the user does not specify a destination or connection name to `podman`. + + +## EXAMPLE + +The following example depicts a configuration file with two connections. One of the connections +is designated as the default connection. +``` +[connections] + [connections.host1] + destination = "host1" + username = "homer" + default = true + + [connections.host2] + destination = "192.168.122.133" + username = "fedora" +``` + +## FILES + `/$HOME/.config/containers/podman-remote.conf`, default location for the podman remote +configuration file + +## HISTORY +May 2019, Originally compiled by Brent Baude<bbaude@redhat.com> diff --git a/docs/podman-run.1.md b/docs/podman-run.1.md index 78e8a5d6e..a7091e89a 100644 --- a/docs/podman-run.1.md +++ b/docs/podman-run.1.md @@ -394,7 +394,7 @@ Read in a line delimited file of labels Not implemented -**--log-driver**="*json-file*" +**--log-driver**="*k8s-file*" Logging driver for the container. Currently not supported. This flag is a NOOP provided soley for scripting compatibility. diff --git a/docs/podman.1.md b/docs/podman.1.md index ff942a3c4..b51cdb854 100644 --- a/docs/podman.1.md +++ b/docs/podman.1.md @@ -177,8 +177,8 @@ the exit codes follow the `chroot` standard, see below: | [podman-umount(1)](podman-umount.1.md) | Unmount a working container's root filesystem. | | [podman-unpause(1)](podman-unpause.1.md) | Unpause one or more containers. | | [podman-unshare(1)](podman-unshare.1.md) | Run a command inside of a modified user namespace. | -| [podman-version(1)](podman-varlink.1.md) | Runs the varlink backend interface. | -| [podman-varlink(1)](podman-version.1.md) | Display the Podman version information. | +| [podman-varlink(1)](podman-varlink.1.md) | Runs the varlink backend interface. | +| [podman-version(1)](podman-version.1.md) | Display the Podman version information. | | [podman-volume(1)](podman-volume.1.md) | Manage Volumes. | | [podman-wait(1)](podman-wait.1.md) | Wait on one or more containers to stop and print their exit codes. | diff --git a/docs/tutorials/podman_tutorial.md b/docs/tutorials/podman_tutorial.md index 032b7c851..d2f8e08fa 100644 --- a/docs/tutorials/podman_tutorial.md +++ b/docs/tutorials/podman_tutorial.md @@ -10,7 +10,7 @@ root escalation is required. ## Installing Podman -For installing or building Podman, please see the [installation instructions](install.md). +For installing or building Podman, please see the [installation instructions](https://github.com/containers/libpod/blob/master/install.md). ## Familiarizing yourself with Podman @@ -96,6 +96,28 @@ After being restored, the container will answer requests again as it did before curl http://<IP_address>:8080 ``` +### Migrate the container +To live migrate a container from one host to another the container is checkpointed on the source +system of the migration, transferred to the destination system and then restored on the destination +system. When transferring the checkpoint, it is possible to specify an output-file. + +On the source system: +```console +sudo podman container checkpoint <container_id> -e /tmp/checkpoint.tar.gz +scp /tmp/checkpoint.tar.gz <destination_system>:/tmp +``` + +On the destination system: +```console +sudo podman container restore -i /tmp/checkpoint.tar.gz +``` + +After being restored, the container will answer requests again as it did before checkpointing. This +time the container will continue to run on the destination system. +```console +curl http://<IP_address>:8080 +``` + ### Stopping the container To stop the httpd container: ```console diff --git a/hack/get_ci_vm.sh b/hack/get_ci_vm.sh index aed0042fb..12dd211f4 100755 --- a/hack/get_ci_vm.sh +++ b/hack/get_ci_vm.sh @@ -11,7 +11,7 @@ ${YEL}WARNING: This will not work without local sudo access to run podman,${NOR} ${YEL}possession of the proper ssh private key is required.${NOR} " # TODO: Many/most of these values should come from .cirrus.yml -ZONE="us-central1-a" +ZONE="${ZONE:-us-central1-a}" CPUS="2" MEMORY="4Gb" DISK="200" @@ -238,10 +238,14 @@ showrun --background tar cjf $TMPDIR/$TARBALL --warning=no-file-changed --exclud trap delvm INT # Allow deleting VM if CTRL-C during create # This fails if VM already exists: permit this usage to re-init -echo -e "\n${YEL}Trying to creating a VM named $VMNAME\n${RED}(might take a minute/two. Errors ignored).${NOR}" +echo -e "\n${YEL}Trying to creating a VM named $VMNAME${NOR}\n${YEL}in GCE region/zone $ZONE${NOR}" +echo -e "For faster access, export ZONE='something-closer-<any letter>'" +echo 'List of regions and zones: https://cloud.google.com/compute/docs/regions-zones/' +echo -e "${RED}(might take a minute/two. Errors ignored).${NOR}" showrun $CREATE_CMD || true # allow re-running commands below when "delete: N" # Any subsequent failure should prompt for VM deletion +trap - INT trap delvm EXIT echo -e "\n${YEL}Waiting up to 30s for ssh port to open${NOR}" diff --git a/install.md b/install.md index 94de2ea74..4b2c5a119 100644 --- a/install.md +++ b/install.md @@ -273,7 +273,7 @@ First, ensure that the go version that is found first on the $PATH (in case you git clone https://github.com/containers/libpod/ $GOPATH/src/github.com/containers/libpod cd $GOPATH/src/github.com/containers/libpod make BUILDTAGS="selinux seccomp" -sudo make install PREFIX= +sudo make install PREFIX=/usr ``` #### Build Tags diff --git a/libpod/container.go b/libpod/container.go index c07f4c78d..c8ab42fc3 100644 --- a/libpod/container.go +++ b/libpod/container.go @@ -51,6 +51,15 @@ const CgroupfsDefaultCgroupParent = "/libpod_parent" // manager in libpod const SystemdDefaultCgroupParent = "machine.slice" +// JournaldLogging is the string conmon expects to specify journald logging +const JournaldLogging = "journald" + +// KubernetesLogging is the string conmon expects when specifying to use the kubernetes logging format +const KubernetesLogging = "k8s-file" + +// JSONLogging is the string conmon expects when specifying to use the json logging format +const JSONLogging = "json-file" + // DefaultWaitInterval is the default interval between container status checks // while waiting. const DefaultWaitInterval = 250 * time.Millisecond @@ -368,6 +377,8 @@ type ContainerConfig struct { CgroupParent string `json:"cgroupParent"` // LogPath log location LogPath string `json:"logPath"` + // LogDriver driver for logs + LogDriver string `json:"logDriver"` // File containing the conmon PID ConmonPidFile string `json:"conmonPidFile,omitempty"` // RestartPolicy indicates what action the container will take upon @@ -775,6 +786,11 @@ func (c *Container) RestartRetries() uint { return c.config.RestartRetries } +// LogDriver returns the log driver for this container +func (c *Container) LogDriver() string { + return c.config.LogDriver +} + // RuntimeName returns the name of the runtime func (c *Container) RuntimeName() string { return c.runtime.ociRuntime.name diff --git a/libpod/container_api.go b/libpod/container_api.go index eff5bfe5f..1894780de 100644 --- a/libpod/container_api.go +++ b/libpod/container_api.go @@ -10,9 +10,7 @@ import ( "sync" "time" - "github.com/containers/libpod/libpod/driver" "github.com/containers/libpod/libpod/events" - "github.com/containers/libpod/pkg/inspect" "github.com/containers/libpod/pkg/lookup" "github.com/containers/storage/pkg/stringid" "github.com/docker/docker/oci/caps" @@ -535,32 +533,6 @@ func (c *Container) RemoveArtifact(name string) error { return os.Remove(c.getArtifactPath(name)) } -// Inspect a container for low-level information -func (c *Container) Inspect(size bool) (*inspect.ContainerInspectData, error) { - if !c.batched { - c.lock.Lock() - defer c.lock.Unlock() - - if err := c.syncContainer(); err != nil { - return nil, err - } - } - - storeCtr, err := c.runtime.store.Container(c.ID()) - if err != nil { - return nil, errors.Wrapf(err, "error getting container from store %q", c.ID()) - } - layer, err := c.runtime.store.Layer(storeCtr.LayerID) - if err != nil { - return nil, errors.Wrapf(err, "error reading information about layer %q", storeCtr.LayerID) - } - driverData, err := driver.GetDriverData(c.runtime.store, layer.ID) - if err != nil { - return nil, errors.Wrapf(err, "error getting graph driver info %q", c.ID()) - } - return c.getContainerInspectData(size, driverData) -} - // Wait blocks until the container exits and returns its exit code. func (c *Container) Wait() (int32, error) { return c.WaitWithInterval(DefaultWaitInterval) @@ -815,11 +787,27 @@ type ContainerCheckpointOptions struct { // TCPEstablished tells the API to checkpoint a container // even if it contains established TCP connections TCPEstablished bool + // Export tells the API to write the checkpoint image to + // the filename set in TargetFile + // Import tells the API to read the checkpoint image from + // the filename set in TargetFile + TargetFile string + // Name tells the API that during restore from an exported + // checkpoint archive a new name should be used for the + // restored container + Name string } // Checkpoint checkpoints a container func (c *Container) Checkpoint(ctx context.Context, options ContainerCheckpointOptions) error { logrus.Debugf("Trying to checkpoint container %s", c.ID()) + + if options.TargetFile != "" { + if err := c.prepareCheckpointExport(); err != nil { + return err + } + } + if !c.batched { c.lock.Lock() defer c.lock.Unlock() diff --git a/libpod/container_inspect.go b/libpod/container_inspect.go index a7369bfdd..8e34e7088 100644 --- a/libpod/container_inspect.go +++ b/libpod/container_inspect.go @@ -2,14 +2,127 @@ package libpod import ( "strings" + "time" - "github.com/containers/libpod/pkg/inspect" + "github.com/containers/libpod/libpod/driver" "github.com/cri-o/ocicni/pkg/ocicni" specs "github.com/opencontainers/runtime-spec/specs-go" + "github.com/pkg/errors" "github.com/sirupsen/logrus" ) -func (c *Container) getContainerInspectData(size bool, driverData *inspect.Data) (*inspect.ContainerInspectData, error) { +// InspectContainerData provides a detailed record of a container's configuration +// and state as viewed by Libpod. +// Large portions of this structure are defined such that the output is +// compatible with `docker inspect` JSON, but additional fields have been added +// as required to share information not in the original output. +type InspectContainerData struct { + ID string `json:"Id"` + Created time.Time `json:"Created"` + Path string `json:"Path"` + Args []string `json:"Args"` + State *InspectContainerState `json:"State"` + ImageID string `json:"Image"` + ImageName string `json:"ImageName"` + Rootfs string `json:"Rootfs"` + ResolvConfPath string `json:"ResolvConfPath"` + HostnamePath string `json:"HostnamePath"` + HostsPath string `json:"HostsPath"` + StaticDir string `json:"StaticDir"` + LogPath string `json:"LogPath"` + ConmonPidFile string `json:"ConmonPidFile"` + Name string `json:"Name"` + RestartCount int32 `json:"RestartCount"` + Driver string `json:"Driver"` + MountLabel string `json:"MountLabel"` + ProcessLabel string `json:"ProcessLabel"` + AppArmorProfile string `json:"AppArmorProfile"` + EffectiveCaps []string `json:"EffectiveCaps"` + BoundingCaps []string `json:"BoundingCaps"` + ExecIDs []string `json:"ExecIDs"` + GraphDriver *driver.Data `json:"GraphDriver"` + SizeRw int64 `json:"SizeRw,omitempty"` + SizeRootFs int64 `json:"SizeRootFs,omitempty"` + Mounts []specs.Mount `json:"Mounts"` + Dependencies []string `json:"Dependencies"` + NetworkSettings *InspectNetworkSettings `json:"NetworkSettings"` //TODO + ExitCommand []string `json:"ExitCommand"` + Namespace string `json:"Namespace"` + IsInfra bool `json:"IsInfra"` +} + +// InspectContainerState provides a detailed record of a container's current +// state. It is returned as part of InspectContainerData. +// As with InspectContainerData, many portions of this struct are matched to +// Docker, but here we see more fields that are unused (nonsensical in the +// context of Libpod). +type InspectContainerState struct { + OciVersion string `json:"OciVersion"` + Status string `json:"Status"` + Running bool `json:"Running"` + Paused bool `json:"Paused"` + Restarting bool `json:"Restarting"` // TODO + OOMKilled bool `json:"OOMKilled"` + Dead bool `json:"Dead"` + Pid int `json:"Pid"` + ExitCode int32 `json:"ExitCode"` + Error string `json:"Error"` // TODO + StartedAt time.Time `json:"StartedAt"` + FinishedAt time.Time `json:"FinishedAt"` + Healthcheck HealthCheckResults `json:"Healthcheck,omitempty"` +} + +// InspectNetworkSettings holds information about the network settings of the +// container. +// Many fields are maintained only for compatibility with `docker inspect` and +// are unused within Libpod. +type InspectNetworkSettings struct { + Bridge string `json:"Bridge"` + SandboxID string `json:"SandboxID"` + HairpinMode bool `json:"HairpinMode"` + LinkLocalIPv6Address string `json:"LinkLocalIPv6Address"` + LinkLocalIPv6PrefixLen int `json:"LinkLocalIPv6PrefixLen"` + Ports []ocicni.PortMapping `json:"Ports"` + SandboxKey string `json:"SandboxKey"` + SecondaryIPAddresses []string `json:"SecondaryIPAddresses"` + SecondaryIPv6Addresses []string `json:"SecondaryIPv6Addresses"` + EndpointID string `json:"EndpointID"` + Gateway string `json:"Gateway"` + GlobalIPv6Address string `json:"GlobalIPv6Address"` + GlobalIPv6PrefixLen int `json:"GlobalIPv6PrefixLen"` + IPAddress string `json:"IPAddress"` + IPPrefixLen int `json:"IPPrefixLen"` + IPv6Gateway string `json:"IPv6Gateway"` + MacAddress string `json:"MacAddress"` +} + +// Inspect a container for low-level information +func (c *Container) Inspect(size bool) (*InspectContainerData, error) { + if !c.batched { + c.lock.Lock() + defer c.lock.Unlock() + + if err := c.syncContainer(); err != nil { + return nil, err + } + } + + storeCtr, err := c.runtime.store.Container(c.ID()) + if err != nil { + return nil, errors.Wrapf(err, "error getting container from store %q", c.ID()) + } + layer, err := c.runtime.store.Layer(storeCtr.LayerID) + if err != nil { + return nil, errors.Wrapf(err, "error reading information about layer %q", storeCtr.LayerID) + } + driverData, err := driver.GetDriverData(c.runtime.store, layer.ID) + if err != nil { + return nil, errors.Wrapf(err, "error getting graph driver info %q", c.ID()) + } + return c.getContainerInspectData(size, driverData) +} + +func (c *Container) getContainerInspectData(size bool, driverData *driver.Data) (*InspectContainerData, error) { config := c.config runtimeInfo := c.state spec, err := c.specFromState() @@ -65,12 +178,12 @@ func (c *Container) getContainerInspectData(size bool, driverData *inspect.Data) } } - data := &inspect.ContainerInspectData{ + data := &InspectContainerData{ ID: config.ID, Created: config.CreatedTime, Path: path, Args: args, - State: &inspect.ContainerInspectState{ + State: &InspectContainerState{ OciVersion: spec.Version, Status: runtimeInfo.State.String(), Running: runtimeInfo.State == ContainerStateRunning, @@ -106,7 +219,7 @@ func (c *Container) getContainerInspectData(size bool, driverData *inspect.Data) GraphDriver: driverData, Mounts: mounts, Dependencies: c.Dependencies(), - NetworkSettings: &inspect.NetworkSettings{ + NetworkSettings: &InspectNetworkSettings{ Bridge: "", // TODO SandboxID: "", // TODO - is this even relevant? HairpinMode: false, // TODO diff --git a/libpod/container_internal.go b/libpod/container_internal.go index 5f8dd1c72..c0b5e4302 100644 --- a/libpod/container_internal.go +++ b/libpod/container_internal.go @@ -21,6 +21,7 @@ import ( "github.com/containers/storage/pkg/archive" "github.com/containers/storage/pkg/mount" spec "github.com/opencontainers/runtime-spec/specs-go" + "github.com/opencontainers/runtime-tools/generate" "github.com/opencontainers/selinux/go-selinux/label" opentracing "github.com/opentracing/opentracing-go" "github.com/pkg/errors" @@ -1345,7 +1346,7 @@ func (c *Container) appendStringToRundir(destFile, output string) (string, error return filepath.Join(c.state.RunDir, destFile), nil } -// Save OCI spec to disk, replacing any existing specs for the container +// saveSpec saves the OCI spec to disk, replacing any existing specs for the container func (c *Container) saveSpec(spec *spec.Spec) error { // If the OCI spec already exists, we need to replace it // Cannot guarantee some things, e.g. network namespaces, have the same @@ -1501,3 +1502,40 @@ func (c *Container) checkReadyForRemoval() error { return nil } + +// writeJSONFile marshalls and writes the given data to a JSON file +// in the bundle path +func (c *Container) writeJSONFile(v interface{}, file string) (err error) { + fileJSON, err := json.MarshalIndent(v, "", " ") + if err != nil { + return errors.Wrapf(err, "error writing JSON to %s for container %s", file, c.ID()) + } + file = filepath.Join(c.bundlePath(), file) + if err := ioutil.WriteFile(file, fileJSON, 0644); err != nil { + return err + } + + return nil +} + +// prepareCheckpointExport writes the config and spec to +// JSON files for later export +func (c *Container) prepareCheckpointExport() (err error) { + // save live config + if err := c.writeJSONFile(c.Config(), "config.dump"); err != nil { + return err + } + + // save spec + jsonPath := filepath.Join(c.bundlePath(), "config.json") + g, err := generate.NewFromFile(jsonPath) + if err != nil { + logrus.Debugf("generating spec for container %q failed with %v", c.ID(), err) + return err + } + if err := c.writeJSONFile(g.Spec(), "spec.dump"); err != nil { + return err + } + + return nil +} diff --git a/libpod/container_internal_linux.go b/libpod/container_internal_linux.go index f25f76092..4acc77afa 100644 --- a/libpod/container_internal_linux.go +++ b/libpod/container_internal_linux.go @@ -5,6 +5,7 @@ package libpod import ( "context" "fmt" + "io" "io/ioutil" "net" "os" @@ -25,6 +26,7 @@ import ( "github.com/containers/libpod/pkg/lookup" "github.com/containers/libpod/pkg/resolvconf" "github.com/containers/libpod/pkg/rootless" + "github.com/containers/storage/pkg/archive" securejoin "github.com/cyphar/filepath-securejoin" "github.com/opencontainers/runc/libcontainer/user" spec "github.com/opencontainers/runtime-spec/specs-go" @@ -496,6 +498,45 @@ func (c *Container) addNamespaceContainer(g *generate.Generator, ns LinuxNS, ctr return nil } +func (c *Container) exportCheckpoint(dest string) (err error) { + if (len(c.config.NamedVolumes) > 0) || (len(c.Dependencies()) > 0) { + return errors.Errorf("Cannot export checkpoints of containers with named volumes or dependencies") + } + logrus.Debugf("Exporting checkpoint image of container %q to %q", c.ID(), dest) + input, err := archive.TarWithOptions(c.bundlePath(), &archive.TarOptions{ + Compression: archive.Gzip, + IncludeSourceDir: true, + IncludeFiles: []string{ + "checkpoint", + "artifacts", + "ctr.log", + "config.dump", + "spec.dump", + "network.status"}, + }) + + if err != nil { + return errors.Wrapf(err, "error reading checkpoint directory %q", c.ID()) + } + + outFile, err := os.Create(dest) + if err != nil { + return errors.Wrapf(err, "error creating checkpoint export file %q", dest) + } + defer outFile.Close() + + if err := os.Chmod(dest, 0600); err != nil { + return errors.Wrapf(err, "cannot chmod %q", dest) + } + + _, err = io.Copy(outFile, input) + if err != nil { + return err + } + + return nil +} + func (c *Container) checkpointRestoreSupported() (err error) { if !criu.CheckForCriu() { return errors.Errorf("Checkpoint/Restore requires at least CRIU %d", criu.MinCriuVersion) @@ -549,6 +590,12 @@ func (c *Container) checkpoint(ctx context.Context, options ContainerCheckpointO return err } + if options.TargetFile != "" { + if err = c.exportCheckpoint(options.TargetFile); err != nil { + return err + } + } + logrus.Debugf("Checkpointed container %s", c.ID()) if !options.KeepRunning { @@ -561,15 +608,50 @@ func (c *Container) checkpoint(ctx context.Context, options ContainerCheckpointO } if !options.Keep { - // Remove log file - os.Remove(filepath.Join(c.bundlePath(), "dump.log")) - // Remove statistic file - os.Remove(filepath.Join(c.bundlePath(), "stats-dump")) + cleanup := []string{ + "dump.log", + "stats-dump", + "config.dump", + "spec.dump", + } + for _, delete := range cleanup { + file := filepath.Join(c.bundlePath(), delete) + os.Remove(file) + } } return c.save() } +func (c *Container) importCheckpoint(input string) (err error) { + archiveFile, err := os.Open(input) + if err != nil { + return errors.Wrapf(err, "Failed to open checkpoint archive %s for import", input) + } + + defer archiveFile.Close() + options := &archive.TarOptions{ + ExcludePatterns: []string{ + // config.dump and spec.dump are only required + // container creation + "config.dump", + "spec.dump", + }, + } + err = archive.Untar(archiveFile, c.bundlePath(), options) + if err != nil { + return errors.Wrapf(err, "Unpacking of checkpoint archive %s failed", input) + } + + // Make sure the newly created config.json exists on disk + g := generate.NewFromSpec(c.config.Spec) + if err = c.saveSpec(g.Spec()); err != nil { + return errors.Wrap(err, "Saving imported container specification for restore failed") + } + + return nil +} + func (c *Container) restore(ctx context.Context, options ContainerCheckpointOptions) (err error) { if err := c.checkpointRestoreSupported(); err != nil { @@ -580,6 +662,12 @@ func (c *Container) restore(ctx context.Context, options ContainerCheckpointOpti return errors.Wrapf(ErrCtrStateInvalid, "container %s is running or paused, cannot restore", c.ID()) } + if options.TargetFile != "" { + if err = c.importCheckpoint(options.TargetFile); err != nil { + return err + } + } + // Let's try to stat() CRIU's inventory file. If it does not exist, it makes // no sense to try a restore. This is a minimal check if a checkpoint exist. if _, err := os.Stat(filepath.Join(c.CheckpointPath(), "inventory.img")); os.IsNotExist(err) { @@ -593,7 +681,13 @@ func (c *Container) restore(ctx context.Context, options ContainerCheckpointOpti // Read network configuration from checkpoint // Currently only one interface with one IP is supported. networkStatusFile, err := os.Open(filepath.Join(c.bundlePath(), "network.status")) - if err == nil { + // If the restored container should get a new name, the IP address of + // the container will not be restored. This assumes that if a new name is + // specified, the container is restored multiple times. + // TODO: This implicit restoring with or without IP depending on an + // unrelated restore parameter (--name) does not seem like the + // best solution. + if err == nil && options.Name == "" { // The file with the network.status does exist. Let's restore the // container with the same IP address as during checkpointing. defer networkStatusFile.Close() @@ -637,23 +731,44 @@ func (c *Container) restore(ctx context.Context, options ContainerCheckpointOpti return err } + // Restoring from an import means that we are doing migration + if options.TargetFile != "" { + g.SetRootPath(c.state.Mountpoint) + } + // We want to have the same network namespace as before. if c.config.CreateNetNS { g.AddOrReplaceLinuxNamespace(spec.NetworkNamespace, c.state.NetNS.Path()) } - // Save the OCI spec to disk - if err := c.saveSpec(g.Spec()); err != nil { + if err := c.makeBindMounts(); err != nil { return err } - if err := c.makeBindMounts(); err != nil { - return err + if options.TargetFile != "" { + for dstPath, srcPath := range c.state.BindMounts { + newMount := spec.Mount{ + Type: "bind", + Source: srcPath, + Destination: dstPath, + Options: []string{"bind", "private"}, + } + if c.IsReadOnly() && dstPath != "/dev/shm" { + newMount.Options = append(newMount.Options, "ro", "nosuid", "noexec", "nodev") + } + if !MountExists(g.Mounts(), dstPath) { + g.AddMount(newMount) + } + } } // Cleanup for a working restore. c.removeConmonFiles() + // Save the OCI spec to disk + if err := c.saveSpec(g.Spec()); err != nil { + return err + } if err := c.runtime.ociRuntime.createContainer(c, c.config.CgroupParent, &options); err != nil { return err } diff --git a/libpod/container_log.go b/libpod/container_log.go index e998ad316..374e5a1fc 100644 --- a/libpod/container_log.go +++ b/libpod/container_log.go @@ -19,6 +19,13 @@ const ( // zeroes are not trimmed, taken from // https://github.com/golang/go/issues/19635 logTimeFormat = "2006-01-02T15:04:05.000000000Z07:00" + + // partialLogType signifies a log line that exceeded the buffer + // length and needed to spill into a new line + partialLogType = "P" + + // fullLogType signifies a log line is full + fullLogType = "F" ) // LogOptions is the options you can use for logs @@ -53,6 +60,15 @@ func (r *Runtime) Log(containers []*Container, options *LogOptions, logChannel c // ReadLog reads a containers log based on the input options and returns loglines over a channel func (c *Container) ReadLog(options *LogOptions, logChannel chan *LogLine) error { + // TODO Skip sending logs until journald logs can be read + // TODO make this not a magic string + if c.LogDriver() == JournaldLogging { + return c.readFromJournal(options, logChannel) + } + return c.readFromLogFile(options, logChannel) +} + +func (c *Container) readFromLogFile(options *LogOptions, logChannel chan *LogLine) error { t, tailLog, err := getLogFile(c.LogPath(), options) if err != nil { // If the log file does not exist, this is not fatal. @@ -191,7 +207,7 @@ func newLogLine(line string) (*LogLine, error) { if len(splitLine) < 4 { return nil, errors.Errorf("'%s' is not a valid container log line", line) } - logTime, err := time.Parse(time.RFC3339Nano, splitLine[0]) + logTime, err := time.Parse(logTimeFormat, splitLine[0]) if err != nil { return nil, errors.Wrapf(err, "unable to convert time %s from container log", splitLine[0]) } @@ -206,7 +222,7 @@ func newLogLine(line string) (*LogLine, error) { // Partial returns a bool if the log line is a partial log type func (l *LogLine) Partial() bool { - if l.ParseLogType == "P" { + if l.ParseLogType == partialLogType { return true } return false diff --git a/libpod/container_log_linux.go b/libpod/container_log_linux.go new file mode 100644 index 000000000..e549673a6 --- /dev/null +++ b/libpod/container_log_linux.go @@ -0,0 +1,143 @@ +//+build linux +//+build systemd + +package libpod + +import ( + "fmt" + "io" + "strings" + "time" + + journal "github.com/coreos/go-systemd/sdjournal" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +const ( + // journaldLogOut is the journald priority signifying stdout + journaldLogOut = "6" + + // journaldLogErr is the journald priority signifying stderr + journaldLogErr = "3" + + // bufLen is the length of the buffer to read from a k8s-file + // formatted log line + // let's set it as 2k just to be safe if k8s-file format ever changes + bufLen = 16384 +) + +func (c *Container) readFromJournal(options *LogOptions, logChannel chan *LogLine) error { + var config journal.JournalReaderConfig + config.NumFromTail = options.Tail + config.Formatter = journalFormatter + defaultTime := time.Time{} + if options.Since != defaultTime { + // coreos/go-systemd/sdjournal doesn't correctly handle requests for data in the future + // return nothing instead of fasely printing + if time.Now().Before(options.Since) { + return nil + } + config.Since = time.Since(options.Since) + } + config.Matches = append(config.Matches, journal.Match{ + Field: "CONTAINER_ID_FULL", + Value: c.ID(), + }) + options.WaitGroup.Add(1) + + r, err := journal.NewJournalReader(config) + if err != nil { + return err + } + if r == nil { + return errors.Errorf("journal reader creation failed") + } + if options.Tail == 0 { + r.Rewind() + } + + if options.Follow { + go func() { + follower := FollowBuffer{logChannel} + err := r.Follow(nil, follower) + if err != nil { + logrus.Debugf(err.Error()) + } + r.Close() + options.WaitGroup.Done() + return + }() + return nil + } + + go func() { + bytes := make([]byte, bufLen) + // /me complains about no do-while in go + ec, err := r.Read(bytes) + for ec != 0 && err == nil { + // because we are reusing bytes, we need to make + // sure the old data doesn't get into the new line + bytestr := string(bytes[:ec]) + logLine, err2 := newLogLine(bytestr) + if err2 != nil { + logrus.Error(err2) + continue + } + logChannel <- logLine + ec, err = r.Read(bytes) + } + if err != nil && err != io.EOF { + logrus.Error(err) + } + r.Close() + options.WaitGroup.Done() + }() + return nil +} + +func journalFormatter(entry *journal.JournalEntry) (string, error) { + usec := entry.RealtimeTimestamp + tsString := time.Unix(0, int64(usec)*int64(time.Microsecond)).Format(logTimeFormat) + output := fmt.Sprintf("%s ", tsString) + priority, ok := entry.Fields["PRIORITY"] + if !ok { + return "", errors.Errorf("no PRIORITY field present in journal entry") + } + if priority == journaldLogOut { + output += "stdout " + } else if priority == journaldLogErr { + output += "stderr " + } else { + return "", errors.Errorf("unexpected PRIORITY field in journal entry") + } + + // if CONTAINER_PARTIAL_MESSAGE is defined, the log type is "P" + if _, ok := entry.Fields["CONTAINER_PARTIAL_MESSAGE"]; ok { + output += fmt.Sprintf("%s ", partialLogType) + } else { + output += fmt.Sprintf("%s ", fullLogType) + } + + // Finally, append the message + msg, ok := entry.Fields["MESSAGE"] + if !ok { + return "", fmt.Errorf("no MESSAGE field present in journal entry") + } + output += strings.TrimSpace(msg) + return output, nil +} + +type FollowBuffer struct { + logChannel chan *LogLine +} + +func (f FollowBuffer) Write(p []byte) (int, error) { + bytestr := string(p) + logLine, err := newLogLine(bytestr) + if err != nil { + return -1, err + } + f.logChannel <- logLine + return len(p), nil +} diff --git a/libpod/container_log_unsupported.go b/libpod/container_log_unsupported.go new file mode 100644 index 000000000..0ec5740e2 --- /dev/null +++ b/libpod/container_log_unsupported.go @@ -0,0 +1,11 @@ +//+build !linux !systemd + +package libpod + +import ( + "github.com/pkg/errors" +) + +func (c *Container) readFromJournal(options *LogOptions, logChannel chan *LogLine) error { + return errors.Wrapf(ErrOSNotSupported, "Journald logging only enabled with systemd on linux") +} diff --git a/libpod/driver/driver.go b/libpod/driver/driver.go index 717ac2a4d..f9442fa21 100644 --- a/libpod/driver/driver.go +++ b/libpod/driver/driver.go @@ -1,10 +1,15 @@ package driver import ( - "github.com/containers/libpod/pkg/inspect" cstorage "github.com/containers/storage" ) +// Data handles the data for a storage driver +type Data struct { + Name string `json:"Name"` + Data map[string]string `json:"Data"` +} + // GetDriverName returns the name of the driver for the given store func GetDriverName(store cstorage.Store) (string, error) { driver, err := store.GraphDriver() @@ -24,7 +29,7 @@ func GetDriverMetadata(store cstorage.Store, layerID string) (map[string]string, } // GetDriverData returns the Data struct with information of the driver used by the store -func GetDriverData(store cstorage.Store, layerID string) (*inspect.Data, error) { +func GetDriverData(store cstorage.Store, layerID string) (*Data, error) { name, err := GetDriverName(store) if err != nil { return nil, err @@ -33,7 +38,7 @@ func GetDriverData(store cstorage.Store, layerID string) (*inspect.Data, error) if err != nil { return nil, err } - return &inspect.Data{ + return &Data{ Name: name, Data: metaData, }, nil diff --git a/libpod/healthcheck.go b/libpod/healthcheck.go index 5c48cc8ee..3e36a2c95 100644 --- a/libpod/healthcheck.go +++ b/libpod/healthcheck.go @@ -9,7 +9,6 @@ import ( "strings" "time" - "github.com/containers/libpod/pkg/inspect" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -53,6 +52,28 @@ const ( HealthCheckStarting string = "starting" ) +// HealthCheckResults describes the results/logs from a healthcheck +type HealthCheckResults struct { + // Status healthy or unhealthy + Status string `json:"Status"` + // FailingStreak is the number of consecutive failed healthchecks + FailingStreak int `json:"FailingStreak"` + // Log describes healthcheck attempts and results + Log []HealthCheckLog `json:"Log"` +} + +// HealthCheckLog describes the results of a single healthcheck +type HealthCheckLog struct { + // Start time as string + Start string `json:"Start"` + // End time as a string + End string `json:"End"` + // Exitcode is 0 or 1 + ExitCode int `json:"ExitCode"` + // Output is the stdout/stderr from the healthcheck command + Output string `json:"Output"` +} + // hcWriteCloser allows us to use bufio as a WriteCloser type hcWriteCloser struct { *bufio.Writer @@ -157,8 +178,8 @@ func checkHealthCheckCanBeRun(c *Container) (HealthCheckStatus, error) { return HealthCheckDefined, nil } -func newHealthCheckLog(start, end time.Time, exitCode int, log string) inspect.HealthCheckLog { - return inspect.HealthCheckLog{ +func newHealthCheckLog(start, end time.Time, exitCode int, log string) HealthCheckLog { + return HealthCheckLog{ Start: start.Format(time.RFC3339Nano), End: end.Format(time.RFC3339Nano), ExitCode: exitCode, @@ -182,7 +203,7 @@ func (c *Container) updateHealthStatus(status string) error { } // UpdateHealthCheckLog parses the health check results and writes the log -func (c *Container) updateHealthCheckLog(hcl inspect.HealthCheckLog, inStartPeriod bool) error { +func (c *Container) updateHealthCheckLog(hcl HealthCheckLog, inStartPeriod bool) error { healthCheck, err := c.GetHealthCheckLog() if err != nil { return err @@ -223,8 +244,8 @@ func (c *Container) healthCheckLogPath() string { // GetHealthCheckLog returns HealthCheck results by reading the container's // health check log file. If the health check log file does not exist, then // an empty healthcheck struct is returned -func (c *Container) GetHealthCheckLog() (inspect.HealthCheckResults, error) { - var healthCheck inspect.HealthCheckResults +func (c *Container) GetHealthCheckLog() (HealthCheckResults, error) { + var healthCheck HealthCheckResults if _, err := os.Stat(c.healthCheckLogPath()); os.IsNotExist(err) { return healthCheck, nil } diff --git a/libpod/image/image.go b/libpod/image/image.go index b965a4640..89a68a1bd 100644 --- a/libpod/image/image.go +++ b/libpod/image/image.go @@ -659,7 +659,7 @@ func (i *Image) Size(ctx context.Context) (*uint64, error) { } // DriverData gets the driver data from the store on a layer -func (i *Image) DriverData() (*inspect.Data, error) { +func (i *Image) DriverData() (*driver.Data, error) { topLayer, err := i.Layer() if err != nil { return nil, err diff --git a/libpod/networking_linux.go b/libpod/networking_linux.go index b8a916de3..ed9ad5f0d 100644 --- a/libpod/networking_linux.go +++ b/libpod/networking_linux.go @@ -17,7 +17,6 @@ import ( cnitypes "github.com/containernetworking/cni/pkg/types/current" "github.com/containernetworking/plugins/pkg/ns" "github.com/containers/libpod/pkg/firewall" - "github.com/containers/libpod/pkg/inspect" "github.com/containers/libpod/pkg/netns" "github.com/containers/libpod/pkg/rootless" "github.com/cri-o/ocicni/pkg/ocicni" @@ -470,7 +469,7 @@ func getContainerNetIO(ctr *Container) (*netlink.LinkStatistics, error) { return netStats, err } -func (c *Container) getContainerNetworkInfo(data *inspect.ContainerInspectData) *inspect.ContainerInspectData { +func (c *Container) getContainerNetworkInfo(data *InspectContainerData) *InspectContainerData { if c.state.NetNS != nil && len(c.state.NetworkStatus) > 0 { // Report network settings from the first pod network result := c.state.NetworkStatus[0] diff --git a/libpod/networking_unsupported.go b/libpod/networking_unsupported.go index 3a8ac4455..1e46ca40b 100644 --- a/libpod/networking_unsupported.go +++ b/libpod/networking_unsupported.go @@ -2,10 +2,6 @@ package libpod -import ( - "github.com/containers/libpod/pkg/inspect" -) - func (r *Runtime) setupRootlessNetNS(ctr *Container) (err error) { return ErrNotImplemented } @@ -22,6 +18,6 @@ func (r *Runtime) createNetNS(ctr *Container) (err error) { return ErrNotImplemented } -func (c *Container) getContainerNetworkInfo(data *inspect.ContainerInspectData) *inspect.ContainerInspectData { +func (c *Container) getContainerNetworkInfo(data *InspectContainerData) *InspectContainerData { return nil } diff --git a/libpod/oci.go b/libpod/oci.go index abc6214b9..7138108c5 100644 --- a/libpod/oci.go +++ b/libpod/oci.go @@ -367,8 +367,6 @@ func (r *OCIRuntime) execContainer(c *Container, cmd, capAdd, env []string, tty args := []string{} // TODO - should we maintain separate logpaths for exec sessions? - args = append(args, "--log", c.LogPath()) - args = append(args, "exec") if cwd != "" { @@ -402,7 +400,7 @@ func (r *OCIRuntime) execContainer(c *Container, cmd, capAdd, env []string, tty args = append(args, "--env", envVar) } - // Append container ID and command + // Append container ID, name and command args = append(args, c.ID()) args = append(args, cmd...) diff --git a/libpod/oci_linux.go b/libpod/oci_linux.go index 1c1e4a203..7c1c18052 100644 --- a/libpod/oci_linux.go +++ b/libpod/oci_linux.go @@ -214,10 +214,10 @@ func (r *OCIRuntime) createOCIContainer(ctr *Container, cgroupParent string, res } args = append(args, "-c", ctr.ID()) args = append(args, "-u", ctr.ID()) + args = append(args, "-n", ctr.Name()) args = append(args, "-r", r.path) args = append(args, "-b", ctr.bundlePath()) args = append(args, "-p", filepath.Join(ctr.state.RunDir, "pidfile")) - args = append(args, "-l", ctr.LogPath()) args = append(args, "--exit-dir", r.exitsDir) if ctr.config.ConmonPidFile != "" { args = append(args, "--conmon-pidfile", ctr.config.ConmonPidFile) @@ -237,6 +237,13 @@ func (r *OCIRuntime) createOCIContainer(ctr *Container, cgroupParent string, res if r.logSizeMax >= 0 { args = append(args, "--log-size-max", fmt.Sprintf("%v", r.logSizeMax)) } + + logDriver := KubernetesLogging + if ctr.LogDriver() != "" { + logDriver = ctr.LogDriver() + } + args = append(args, "-l", fmt.Sprintf("%s:%s", logDriver, ctr.LogPath())) + if r.noPivot { args = append(args, "--no-pivot") } diff --git a/libpod/options.go b/libpod/options.go index 7ec7dfe63..20aa51981 100644 --- a/libpod/options.go +++ b/libpod/options.go @@ -979,6 +979,27 @@ func WithStaticIP(ip net.IP) CtrCreateOption { } } +// WithLogDriver sets the log driver for the container +func WithLogDriver(driver string) CtrCreateOption { + return func(ctr *Container) error { + if ctr.valid { + return ErrCtrFinalized + } + switch driver { + case "": + return errors.Wrapf(ErrInvalidArg, "log driver must be set") + case JournaldLogging, KubernetesLogging, JSONLogging: + break + default: + return errors.Wrapf(ErrInvalidArg, "invalid log driver") + } + + ctr.config.LogDriver = driver + + return nil + } +} + // WithLogPath sets the path to the log file. func WithLogPath(path string) CtrCreateOption { return func(ctr *Container) error { diff --git a/libpod/runtime.go b/libpod/runtime.go index 1f8dd98b4..098607b63 100644 --- a/libpod/runtime.go +++ b/libpod/runtime.go @@ -250,6 +250,7 @@ type runtimeConfiguredFrom struct { volPathSet bool conmonPath bool conmonEnvVars bool + initPath bool ociRuntimes bool runtimePath bool cniPluginDir bool @@ -475,6 +476,9 @@ func newRuntimeFromConfig(ctx context.Context, userConfigPath string, options .. if tmpConfig.ConmonEnvVars != nil { runtime.configuredFrom.conmonEnvVars = true } + if tmpConfig.InitPath != "" { + runtime.configuredFrom.initPath = true + } if tmpConfig.OCIRuntimes != nil { runtime.configuredFrom.ociRuntimes = true } @@ -512,6 +516,9 @@ func newRuntimeFromConfig(ctx context.Context, userConfigPath string, options .. if !runtime.configuredFrom.conmonEnvVars { runtime.config.ConmonEnvVars = tmpConfig.ConmonEnvVars } + if !runtime.configuredFrom.initPath { + runtime.config.InitPath = tmpConfig.InitPath + } if !runtime.configuredFrom.ociRuntimes { runtime.config.OCIRuntimes = tmpConfig.OCIRuntimes } diff --git a/libpod/runtime_ctr.go b/libpod/runtime_ctr.go index c7758055f..cf1f5701d 100644 --- a/libpod/runtime_ctr.go +++ b/libpod/runtime_ctr.go @@ -14,6 +14,7 @@ import ( "github.com/containers/storage" "github.com/containers/storage/pkg/stringid" spec "github.com/opencontainers/runtime-spec/specs-go" + "github.com/opencontainers/runtime-tools/generate" opentracing "github.com/opentracing/opentracing-go" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -34,7 +35,7 @@ type CtrCreateOption func(*Container) error // A true return will include the container, a false return will exclude it. type ContainerFilter func(*Container) bool -// NewContainer creates a new container from a given OCI config +// NewContainer creates a new container from a given OCI config. func (r *Runtime) NewContainer(ctx context.Context, rSpec *spec.Spec, options ...CtrCreateOption) (c *Container, err error) { r.lock.Lock() defer r.lock.Unlock() @@ -44,20 +45,46 @@ func (r *Runtime) NewContainer(ctx context.Context, rSpec *spec.Spec, options .. return r.newContainer(ctx, rSpec, options...) } -func (r *Runtime) newContainer(ctx context.Context, rSpec *spec.Spec, options ...CtrCreateOption) (c *Container, err error) { - span, _ := opentracing.StartSpanFromContext(ctx, "newContainer") - span.SetTag("type", "runtime") - defer span.Finish() +// RestoreContainer re-creates a container from an imported checkpoint +func (r *Runtime) RestoreContainer(ctx context.Context, rSpec *spec.Spec, config *ContainerConfig) (c *Container, err error) { + r.lock.Lock() + defer r.lock.Unlock() + if !r.valid { + return nil, ErrRuntimeStopped + } + ctr, err := r.initContainerVariables(rSpec, config) + if err != nil { + return nil, errors.Wrapf(err, "error initializing container variables") + } + return r.setupContainer(ctx, ctr, true) +} + +func (r *Runtime) initContainerVariables(rSpec *spec.Spec, config *ContainerConfig) (c *Container, err error) { if rSpec == nil { return nil, errors.Wrapf(ErrInvalidArg, "must provide a valid runtime spec to create container") } - ctr := new(Container) ctr.config = new(ContainerConfig) ctr.state = new(ContainerState) - ctr.config.ID = stringid.GenerateNonCryptoID() + if config == nil { + ctr.config.ID = stringid.GenerateNonCryptoID() + ctr.config.ShmSize = DefaultShmSize + } else { + // This is a restore from an imported checkpoint + if err := JSONDeepCopy(config, ctr.config); err != nil { + return nil, errors.Wrapf(err, "error copying container config for restore") + } + // If the ID is empty a new name for the restored container was requested + if ctr.config.ID == "" { + ctr.config.ID = stringid.GenerateNonCryptoID() + // Fixup ExitCommand with new ID + ctr.config.ExitCommand[len(ctr.config.ExitCommand)-1] = ctr.config.ID + } + // Reset the log path to point to the default + ctr.config.LogPath = "" + } ctr.config.Spec = new(spec.Spec) if err := JSONDeepCopy(rSpec, ctr.config.Spec); err != nil { @@ -65,8 +92,6 @@ func (r *Runtime) newContainer(ctx context.Context, rSpec *spec.Spec, options .. } ctr.config.CreatedTime = time.Now() - ctr.config.ShmSize = DefaultShmSize - ctr.state.BindMounts = make(map[string]string) ctr.config.StopTimeout = CtrRemoveTimeout @@ -80,12 +105,29 @@ func (r *Runtime) newContainer(ctx context.Context, rSpec *spec.Spec, options .. } ctr.runtime = r + + return ctr, nil +} + +func (r *Runtime) newContainer(ctx context.Context, rSpec *spec.Spec, options ...CtrCreateOption) (c *Container, err error) { + span, _ := opentracing.StartSpanFromContext(ctx, "newContainer") + span.SetTag("type", "runtime") + defer span.Finish() + + ctr, err := r.initContainerVariables(rSpec, nil) + if err != nil { + return nil, errors.Wrapf(err, "error initializing container variables") + } + for _, option := range options { if err := option(ctr); err != nil { return nil, errors.Wrapf(err, "error running container create option") } } + return r.setupContainer(ctx, ctr, false) +} +func (r *Runtime) setupContainer(ctx context.Context, ctr *Container, restore bool) (c *Container, err error) { // Allocate a lock for the container lock, err := r.lockManager.AllocateLock() if err != nil { @@ -154,6 +196,19 @@ func (r *Runtime) newContainer(ctx context.Context, rSpec *spec.Spec, options .. return nil, errors.Wrapf(ErrInvalidArg, "unsupported CGroup manager: %s - cannot validate cgroup parent", r.config.CgroupManager) } + if restore { + // Remove information about bind mount + // for new container from imported checkpoint + g := generate.Generator{Config: ctr.config.Spec} + g.RemoveMount("/dev/shm") + ctr.config.ShmDir = "" + g.RemoveMount("/etc/resolv.conf") + g.RemoveMount("/etc/hostname") + g.RemoveMount("/etc/hosts") + g.RemoveMount("/run/.containerenv") + g.RemoveMount("/run/secrets") + } + // Set up storage for the container if err := ctr.setupStorage(ctx); err != nil { return nil, err @@ -167,7 +222,7 @@ func (r *Runtime) newContainer(ctx context.Context, rSpec *spec.Spec, options .. }() if rootless.IsRootless() && ctr.config.ConmonPidFile == "" { - ctr.config.ConmonPidFile = filepath.Join(ctr.config.StaticDir, "conmon.pid") + ctr.config.ConmonPidFile = filepath.Join(ctr.state.RunDir, "conmon.pid") } // Go through named volumes and add them. @@ -196,7 +251,7 @@ func (r *Runtime) newContainer(ctx context.Context, rSpec *spec.Spec, options .. } } - if ctr.config.LogPath == "" { + if ctr.config.LogPath == "" && ctr.config.LogDriver != JournaldLogging { ctr.config.LogPath = filepath.Join(ctr.config.StaticDir, "ctr.log") } diff --git a/pkg/adapter/checkpoint_restore.go b/pkg/adapter/checkpoint_restore.go new file mode 100644 index 000000000..97ba5ecf7 --- /dev/null +++ b/pkg/adapter/checkpoint_restore.go @@ -0,0 +1,145 @@ +// +build !remoteclient + +package adapter + +import ( + "context" + "github.com/containers/libpod/libpod" + "github.com/containers/libpod/libpod/image" + "github.com/containers/storage/pkg/archive" + jsoniter "github.com/json-iterator/go" + spec "github.com/opencontainers/runtime-spec/specs-go" + "github.com/pkg/errors" + "io" + "io/ioutil" + "os" + "path/filepath" +) + +// Prefixing the checkpoint/restore related functions with 'cr' + +// crImportFromJSON imports the JSON files stored in the exported +// checkpoint tarball +func crImportFromJSON(filePath string, v interface{}) error { + jsonFile, err := os.Open(filePath) + if err != nil { + return errors.Wrapf(err, "Failed to open container definition %s for restore", filePath) + } + defer jsonFile.Close() + + content, err := ioutil.ReadAll(jsonFile) + if err != nil { + return errors.Wrapf(err, "Failed to read container definition %s for restore", filePath) + } + json := jsoniter.ConfigCompatibleWithStandardLibrary + if err = json.Unmarshal([]byte(content), v); err != nil { + return errors.Wrapf(err, "Failed to unmarshal container definition %s for restore", filePath) + } + + return nil +} + +// crImportCheckpoint it the function which imports the information +// from checkpoint tarball and re-creates the container from that information +func crImportCheckpoint(ctx context.Context, runtime *libpod.Runtime, input string, name string) ([]*libpod.Container, error) { + // First get the container definition from the + // tarball to a temporary directory + archiveFile, err := os.Open(input) + if err != nil { + return nil, errors.Wrapf(err, "Failed to open checkpoint archive %s for import", input) + } + defer archiveFile.Close() + options := &archive.TarOptions{ + // Here we only need the files config.dump and spec.dump + ExcludePatterns: []string{ + "checkpoint", + "artifacts", + "ctr.log", + "network.status", + }, + } + dir, err := ioutil.TempDir("", "checkpoint") + if err != nil { + return nil, err + } + defer os.RemoveAll(dir) + err = archive.Untar(archiveFile, dir, options) + if err != nil { + return nil, errors.Wrapf(err, "Unpacking of checkpoint archive %s failed", input) + } + + // Load spec.dump from temporary directory + spec := new(spec.Spec) + if err := crImportFromJSON(filepath.Join(dir, "spec.dump"), spec); err != nil { + return nil, err + } + + // Load config.dump from temporary directory + config := new(libpod.ContainerConfig) + if err = crImportFromJSON(filepath.Join(dir, "config.dump"), config); err != nil { + return nil, err + } + + // This should not happen as checkpoints with these options are not exported. + if (len(config.Dependencies) > 0) || (len(config.NamedVolumes) > 0) { + return nil, errors.Errorf("Cannot import checkpoints of containers with named volumes or dependencies") + } + + ctrID := config.ID + newName := false + + // Check if the restored container gets a new name + if name != "" { + config.ID = "" + config.Name = name + newName = true + } + + ctrName := config.Name + + // The code to load the images is copied from create.go + var writer io.Writer + // In create.go this only set if '--quiet' does not exist. + writer = os.Stderr + rtc, err := runtime.GetConfig() + if err != nil { + return nil, err + } + + _, err = runtime.ImageRuntime().New(ctx, config.RootfsImageName, rtc.SignaturePolicyPath, "", writer, nil, image.SigningOptions{}, false, nil) + if err != nil { + return nil, err + } + + // Now create a new container from the just loaded information + container, err := runtime.RestoreContainer(ctx, spec, config) + if err != nil { + return nil, err + } + + var containers []*libpod.Container + if container == nil { + return nil, nil + } + + containerConfig := container.Config() + if containerConfig.Name != ctrName { + return nil, errors.Errorf("Name of restored container (%s) does not match requested name (%s)", containerConfig.Name, ctrName) + } + + if newName == false { + // Only check ID for a restore with the same name. + // Using -n to request a new name for the restored container, will also create a new ID + if containerConfig.ID != ctrID { + return nil, errors.Errorf("ID of restored container (%s) does not match requested ID (%s)", containerConfig.ID, ctrID) + } + } + + // Check if the ExitCommand points to the correct container ID + if containerConfig.ExitCommand[len(containerConfig.ExitCommand)-1] != containerConfig.ID { + return nil, errors.Errorf("'ExitCommandID' uses ID %s instead of container ID %s", containerConfig.ExitCommand[len(containerConfig.ExitCommand)-1], containerConfig.ID) + } + + containers = append(containers, container) + return containers, nil +} diff --git a/pkg/adapter/client.go b/pkg/adapter/client.go index 01914834f..69aa3220a 100644 --- a/pkg/adapter/client.go +++ b/pkg/adapter/client.go @@ -6,42 +6,52 @@ import ( "fmt" "os" + "github.com/containers/libpod/cmd/podman/remoteclientconfig" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "github.com/varlink/go/varlink" ) var remoteEndpoint *Endpoint func (r RemoteRuntime) RemoteEndpoint() (remoteEndpoint *Endpoint, err error) { - if remoteEndpoint == nil { - remoteEndpoint = &Endpoint{Unknown, ""} - } else { - return remoteEndpoint, nil - } + remoteConfigConnections, _ := remoteclientconfig.ReadRemoteConfig(r.config) - // I'm leaving this here for now as a document of the birdge format. It can be removed later once the bridge - // function is more flushed out. - // bridge := `ssh -T root@192.168.122.1 "/usr/bin/varlink -A '/usr/bin/podman varlink \$VARLINK_ADDRESS' bridge"` - if len(r.cmd.RemoteHost) > 0 { - // The user has provided a remote host endpoint + // If the user defines an env variable for podman_varlink_bridge + // we use that as passed. + if bridge := os.Getenv("PODMAN_VARLINK_BRIDGE"); bridge != "" { + logrus.Debug("creating a varlink bridge based on env variable") + remoteEndpoint, err = newBridgeConnection(bridge, nil, r.cmd.LogLevel) + // if an environment variable for podman_varlink_address is defined, + // we used that as passed + } else if address := os.Getenv("PODMAN_VARLINK_ADDRESS"); address != "" { + logrus.Debug("creating a varlink address based on env variable: %s", address) + remoteEndpoint, err = newSocketConnection(address) + // if the user provides a remote host, we use it to configure a bridge connection + } else if len(r.cmd.RemoteHost) > 0 { + logrus.Debug("creating a varlink bridge based on user input") if len(r.cmd.RemoteUserName) < 1 { return nil, errors.New("you must provide a username when providing a remote host name") } - remoteEndpoint.Type = BridgeConnection - remoteEndpoint.Connection = fmt.Sprintf( - `ssh -T %s@%s /usr/bin/varlink -A \'/usr/bin/podman --log-level=%s varlink \\\$VARLINK_ADDRESS\' bridge`, - r.cmd.RemoteUserName, r.cmd.RemoteHost, r.cmd.LogLevel) - - } else if bridge := os.Getenv("PODMAN_VARLINK_BRIDGE"); bridge != "" { - remoteEndpoint.Type = BridgeConnection - remoteEndpoint.Connection = bridge - } else { - address := os.Getenv("PODMAN_VARLINK_ADDRESS") - if address == "" { - address = DefaultAddress + rc := remoteclientconfig.RemoteConnection{r.cmd.RemoteHost, r.cmd.RemoteUserName, false} + remoteEndpoint, err = newBridgeConnection("", &rc, r.cmd.LogLevel) + // if the user has a config file with connections in it + } else if len(remoteConfigConnections.Connections) > 0 { + logrus.Debug("creating a varlink bridge based configuration file") + var rc *remoteclientconfig.RemoteConnection + if len(r.cmd.ConnectionName) > 0 { + rc, err = remoteConfigConnections.GetRemoteConnection(r.cmd.ConnectionName) + } else { + rc, err = remoteConfigConnections.GetDefault() + } + if err != nil { + return nil, err } - remoteEndpoint.Type = DirectConnection - remoteEndpoint.Connection = address + remoteEndpoint, err = newBridgeConnection("", rc, r.cmd.LogLevel) + // last resort is to make a socket connection with the default varlink address for root user + } else { + logrus.Debug("creating a varlink address based default root address") + remoteEndpoint, err = newSocketConnection(DefaultAddress) } return } @@ -72,3 +82,12 @@ func (r RemoteRuntime) RefreshConnection() error { r.Conn = newConn return nil } + +// newSocketConnection returns an endpoint for a uds based connection +func newSocketConnection(address string) (*Endpoint, error) { + endpoint := Endpoint{ + Type: DirectConnection, + Connection: address, + } + return &endpoint, nil +} diff --git a/pkg/adapter/client_unix.go b/pkg/adapter/client_unix.go new file mode 100644 index 000000000..e0406567c --- /dev/null +++ b/pkg/adapter/client_unix.go @@ -0,0 +1,30 @@ +// +build linux darwin +// +build remoteclient + +package adapter + +import ( + "fmt" + + "github.com/containers/libpod/cmd/podman/remoteclientconfig" + "github.com/pkg/errors" +) + +// newBridgeConnection creates a bridge type endpoint with username, destination, and log-level +func newBridgeConnection(formattedBridge string, remoteConn *remoteclientconfig.RemoteConnection, logLevel string) (*Endpoint, error) { + endpoint := Endpoint{ + Type: BridgeConnection, + } + + if len(formattedBridge) < 1 && remoteConn == nil { + return nil, errors.New("bridge connections must either be created by string or remoteconnection") + } + if len(formattedBridge) > 0 { + endpoint.Connection = formattedBridge + return &endpoint, nil + } + endpoint.Connection = fmt.Sprintf( + `ssh -T %s@%s -- /usr/bin/varlink -A \'/usr/bin/podman --log-level=%s varlink \\\$VARLINK_ADDRESS\' bridge`, + remoteConn.Username, remoteConn.Destination, logLevel) + return &endpoint, nil +} diff --git a/pkg/adapter/client_windows.go b/pkg/adapter/client_windows.go new file mode 100644 index 000000000..088550667 --- /dev/null +++ b/pkg/adapter/client_windows.go @@ -0,0 +1,15 @@ +// +build remoteclient + +package adapter + +import ( + "github.com/containers/libpod/cmd/podman/remoteclientconfig" + "github.com/containers/libpod/libpod" +) + +func newBridgeConnection(formattedBridge string, remoteConn *remoteclientconfig.RemoteConnection, logLevel string) (*Endpoint, error) { + // TODO + // Unix and Windows appear to quote their ssh implementations differently therefore once we figure out what + // windows ssh is doing here, we can then get the format correct. + return nil, libpod.ErrNotImplemented +} diff --git a/pkg/adapter/containers.go b/pkg/adapter/containers.go index ff7b6377a..29297fbd5 100644 --- a/pkg/adapter/containers.go +++ b/pkg/adapter/containers.go @@ -6,6 +6,7 @@ import ( "bufio" "context" "fmt" + "io" "io/ioutil" "os" "path/filepath" @@ -15,9 +16,12 @@ import ( "syscall" "time" + "github.com/containers/buildah" + "github.com/containers/image/manifest" "github.com/containers/libpod/cmd/podman/cliconfig" "github.com/containers/libpod/cmd/podman/shared" "github.com/containers/libpod/libpod" + "github.com/containers/libpod/libpod/image" "github.com/containers/libpod/pkg/adapter/shortcuts" "github.com/containers/libpod/pkg/systemdgen" "github.com/containers/psgo" @@ -522,7 +526,7 @@ func (r *LocalRuntime) Checkpoint(c *cliconfig.CheckpointValues, options libpod. } // Restore one or more containers -func (r *LocalRuntime) Restore(c *cliconfig.RestoreValues, options libpod.ContainerCheckpointOptions) error { +func (r *LocalRuntime) Restore(ctx context.Context, c *cliconfig.RestoreValues, options libpod.ContainerCheckpointOptions) error { var ( containers []*libpod.Container err, lastError error @@ -534,7 +538,9 @@ func (r *LocalRuntime) Restore(c *cliconfig.RestoreValues, options libpod.Contai return state == libpod.ContainerStateExited }) - if c.All { + if c.Import != "" { + containers, err = crImportCheckpoint(ctx, r.Runtime, c.Import, c.Name) + } else if c.All { containers, err = r.GetContainers(filterFuncs...) } else { containers, err = shortcuts.GetContainersByContext(false, c.Latest, c.InputArgs, r.Runtime) @@ -1030,3 +1036,55 @@ func (r *LocalRuntime) GenerateSystemd(c *cliconfig.GenerateSystemdValues) (stri func (r *LocalRuntime) GetNamespaces(container shared.PsContainerOutput) *shared.Namespace { return shared.GetNamespaces(container.Pid) } + +// Commit creates a local image from a container +func (r *LocalRuntime) Commit(ctx context.Context, c *cliconfig.CommitValues, container, imageName string) (string, error) { + var ( + writer io.Writer + mimeType string + ) + switch c.Format { + case "oci": + mimeType = buildah.OCIv1ImageManifest + if c.Flag("message").Changed { + return "", errors.Errorf("messages are only compatible with the docker image format (-f docker)") + } + case "docker": + mimeType = manifest.DockerV2Schema2MediaType + default: + return "", errors.Errorf("unrecognized image format %q", c.Format) + } + if !c.Quiet { + writer = os.Stderr + } + ctr, err := r.Runtime.LookupContainer(container) + if err != nil { + return "", errors.Wrapf(err, "error looking up container %q", container) + } + + rtc, err := r.Runtime.GetConfig() + if err != nil { + return "", err + } + + sc := image.GetSystemContext(rtc.SignaturePolicyPath, "", false) + coptions := buildah.CommitOptions{ + SignaturePolicyPath: rtc.SignaturePolicyPath, + ReportWriter: writer, + SystemContext: sc, + PreferredManifestType: mimeType, + } + options := libpod.ContainerCommitOptions{ + CommitOptions: coptions, + Pause: c.Pause, + IncludeVolumes: c.IncludeVolumes, + Message: c.Message, + Changes: c.Change, + Author: c.Author, + } + newImage, err := ctr.Commit(ctx, imageName, options) + if err != nil { + return "", err + } + return newImage.ID(), nil +} diff --git a/pkg/adapter/containers_remote.go b/pkg/adapter/containers_remote.go index c34495b3d..cf0b90b3a 100644 --- a/pkg/adapter/containers_remote.go +++ b/pkg/adapter/containers_remote.go @@ -16,7 +16,6 @@ import ( "github.com/containers/libpod/cmd/podman/shared" iopodman "github.com/containers/libpod/cmd/podman/varlink" "github.com/containers/libpod/libpod" - "github.com/containers/libpod/pkg/inspect" "github.com/containers/libpod/pkg/varlinkapi/virtwriter" "github.com/cri-o/ocicni/pkg/ocicni" "github.com/docker/docker/pkg/term" @@ -29,12 +28,12 @@ import ( ) // Inspect returns an inspect struct from varlink -func (c *Container) Inspect(size bool) (*inspect.ContainerInspectData, error) { +func (c *Container) Inspect(size bool) (*libpod.InspectContainerData, error) { reply, err := iopodman.ContainerInspectData().Call(c.Runtime.Conn, c.ID(), size) if err != nil { return nil, err } - data := inspect.ContainerInspectData{} + data := libpod.InspectContainerData{} if err := json.Unmarshal([]byte(reply), &data); err != nil { return nil, err } @@ -583,7 +582,15 @@ func (r *LocalRuntime) attach(ctx context.Context, stdin, stdout *os.File, cid s } // TODO add detach keys support - _, err = iopodman.Attach().Send(r.Conn, varlink.Upgrade, cid, detachKeys, start) + reply, err := iopodman.Attach().Send(r.Conn, varlink.Upgrade, cid, detachKeys, start) + if err != nil { + restoreTerminal(oldTermState) + return nil, err + } + + // See if the server accepts the upgraded connection or returns an error + _, err = reply() + if err != nil { restoreTerminal(oldTermState) return nil, err @@ -656,6 +663,10 @@ func (r *LocalRuntime) Attach(ctx context.Context, c *cliconfig.AttachValues) er // Checkpoint one or more containers func (r *LocalRuntime) Checkpoint(c *cliconfig.CheckpointValues, options libpod.ContainerCheckpointOptions) error { + if c.Export != "" { + return errors.New("the remote client does not support exporting checkpoints") + } + var lastError error ids, err := iopodman.GetContainersByContext().Call(r.Conn, c.All, c.Latest, c.InputArgs) if err != nil { @@ -691,7 +702,11 @@ func (r *LocalRuntime) Checkpoint(c *cliconfig.CheckpointValues, options libpod. } // Restore one or more containers -func (r *LocalRuntime) Restore(c *cliconfig.RestoreValues, options libpod.ContainerCheckpointOptions) error { +func (r *LocalRuntime) Restore(ctx context.Context, c *cliconfig.RestoreValues, options libpod.ContainerCheckpointOptions) error { + if c.Import != "" { + return errors.New("the remote client does not support importing checkpoints") + } + var lastError error ids, err := iopodman.GetContainersByContext().Call(r.Conn, c.All, c.Latest, c.InputArgs) if err != nil { @@ -986,3 +1001,26 @@ func (r *LocalRuntime) GetNamespaces(container shared.PsContainerOutput) *shared } return &ns } + +// Commit creates a local image from a container +func (r *LocalRuntime) Commit(ctx context.Context, c *cliconfig.CommitValues, container, imageName string) (string, error) { + var iid string + reply, err := iopodman.Commit().Send(r.Conn, varlink.More, container, imageName, c.Change, c.Author, c.Message, c.Pause, c.Format) + if err != nil { + return "", err + } + for { + responses, flags, err := reply() + if err != nil { + return "", err + } + for _, line := range responses.Logs { + fmt.Fprintln(os.Stderr, line) + } + iid = responses.Id + if flags&varlink.Continues == 0 { + break + } + } + return iid, nil +} diff --git a/pkg/adapter/runtime_remote.go b/pkg/adapter/runtime_remote.go index e0c0898bd..a1d358f68 100644 --- a/pkg/adapter/runtime_remote.go +++ b/pkg/adapter/runtime_remote.go @@ -20,6 +20,7 @@ import ( "github.com/containers/image/docker/reference" "github.com/containers/image/types" "github.com/containers/libpod/cmd/podman/cliconfig" + "github.com/containers/libpod/cmd/podman/remoteclientconfig" "github.com/containers/libpod/cmd/podman/varlink" "github.com/containers/libpod/libpod" "github.com/containers/libpod/libpod/events" @@ -40,6 +41,7 @@ type RemoteRuntime struct { Conn *varlink.Connection Remote bool cmd cliconfig.MainFlags + config io.Reader } // LocalRuntime describes a typical libpod runtime @@ -49,10 +51,35 @@ type LocalRuntime struct { // GetRuntime returns a LocalRuntime struct with the actual runtime embedded in it func GetRuntime(ctx context.Context, c *cliconfig.PodmanCommand) (*LocalRuntime, error) { + var ( + customConfig bool + err error + f *os.File + ) runtime := RemoteRuntime{ Remote: true, cmd: c.GlobalFlags, } + configPath := remoteclientconfig.GetConfigFilePath() + if len(c.GlobalFlags.RemoteConfigFilePath) > 0 { + configPath = c.GlobalFlags.RemoteConfigFilePath + customConfig = true + } + + f, err = os.Open(configPath) + if err != nil { + // If user does not explicitly provide a configuration file path and we cannot + // find a default, no error should occur. + if os.IsNotExist(err) && !customConfig { + logrus.Debugf("unable to load configuration file at %s", configPath) + runtime.config = nil + } else { + return nil, errors.Wrapf(err, "unable to load configuration file at %s", configPath) + } + } else { + // create the io reader for the remote client + runtime.config = bufio.NewReader(f) + } conn, err := runtime.Connect() if err != nil { return nil, err diff --git a/pkg/inspect/inspect.go b/pkg/inspect/inspect.go index 693755aa8..ec3d98613 100644 --- a/pkg/inspect/inspect.go +++ b/pkg/inspect/inspect.go @@ -3,110 +3,11 @@ package inspect import ( "time" - "github.com/containers/image/manifest" - "github.com/cri-o/ocicni/pkg/ocicni" - "github.com/docker/go-connections/nat" + "github.com/containers/libpod/libpod/driver" "github.com/opencontainers/go-digest" "github.com/opencontainers/image-spec/specs-go/v1" - "github.com/opencontainers/runtime-spec/specs-go" ) -// ContainerData holds the podman inspect data for a container -type ContainerData struct { - *ContainerInspectData - HostConfig *HostConfig `json:"HostConfig"` - Config *CtrConfig `json:"Config"` -} - -// HostConfig represents the host configuration for the container -type HostConfig struct { - ContainerIDFile string `json:"ContainerIDFile"` - LogConfig *LogConfig `json:"LogConfig"` //TODO - NetworkMode string `json:"NetworkMode"` - PortBindings nat.PortMap `json:"PortBindings"` //TODO - AutoRemove bool `json:"AutoRemove"` - CapAdd []string `json:"CapAdd"` - CapDrop []string `json:"CapDrop"` - DNS []string `json:"DNS"` - DNSOptions []string `json:"DNSOptions"` - DNSSearch []string `json:"DNSSearch"` - ExtraHosts []string `json:"ExtraHosts"` - GroupAdd []uint32 `json:"GroupAdd"` - IpcMode string `json:"IpcMode"` - Cgroup string `json:"Cgroup"` - OomScoreAdj *int `json:"OomScoreAdj"` - PidMode string `json:"PidMode"` - Privileged bool `json:"Privileged"` - PublishAllPorts bool `json:"PublishAllPorts"` //TODO - ReadOnlyRootfs bool `json:"ReadonlyRootfs"` - ReadOnlyTmpfs bool `json:"ReadonlyTmpfs"` - SecurityOpt []string `json:"SecurityOpt"` - UTSMode string `json:"UTSMode"` - UsernsMode string `json:"UsernsMode"` - ShmSize int64 `json:"ShmSize"` - Runtime string `json:"Runtime"` - ConsoleSize *specs.Box `json:"ConsoleSize"` - CPUShares *uint64 `json:"CpuShares"` - Memory int64 `json:"Memory"` - NanoCPUs int `json:"NanoCpus"` - CgroupParent string `json:"CgroupParent"` - BlkioWeight *uint16 `json:"BlkioWeight"` - BlkioWeightDevice []specs.LinuxWeightDevice `json:"BlkioWeightDevice"` - BlkioDeviceReadBps []specs.LinuxThrottleDevice `json:"BlkioDeviceReadBps"` - BlkioDeviceWriteBps []specs.LinuxThrottleDevice `json:"BlkioDeviceWriteBps"` - BlkioDeviceReadIOps []specs.LinuxThrottleDevice `json:"BlkioDeviceReadIOps"` - BlkioDeviceWriteIOps []specs.LinuxThrottleDevice `json:"BlkioDeviceWriteIOps"` - CPUPeriod *uint64 `json:"CpuPeriod"` - CPUQuota *int64 `json:"CpuQuota"` - CPURealtimePeriod *uint64 `json:"CpuRealtimePeriod"` - CPURealtimeRuntime *int64 `json:"CpuRealtimeRuntime"` - CPUSetCPUs string `json:"CpuSetCpus"` - CPUSetMems string `json:"CpuSetMems"` - Devices []specs.LinuxDevice `json:"Devices"` - DiskQuota int `json:"DiskQuota"` //check type, TODO - KernelMemory *int64 `json:"KernelMemory"` - MemoryReservation *int64 `json:"MemoryReservation"` - MemorySwap *int64 `json:"MemorySwap"` - MemorySwappiness *uint64 `json:"MemorySwappiness"` - OomKillDisable *bool `json:"OomKillDisable"` - PidsLimit *int64 `json:"PidsLimit"` - Ulimits []string `json:"Ulimits"` - CPUCount int `json:"CpuCount"` - CPUPercent int `json:"CpuPercent"` - IOMaximumIOps int `json:"IOMaximumIOps"` //check type, TODO - IOMaximumBandwidth int `json:"IOMaximumBandwidth"` //check type, TODO - Tmpfs []string `json:"Tmpfs"` -} - -// CtrConfig holds information about the container configuration -type CtrConfig struct { - Hostname string `json:"Hostname"` - DomainName string `json:"Domainname"` //TODO - User specs.User `json:"User"` - AttachStdin bool `json:"AttachStdin"` //TODO - AttachStdout bool `json:"AttachStdout"` //TODO - AttachStderr bool `json:"AttachStderr"` //TODO - Tty bool `json:"Tty"` - OpenStdin bool `json:"OpenStdin"` - StdinOnce bool `json:"StdinOnce"` //TODO - Env []string `json:"Env"` - Cmd []string `json:"Cmd"` - Image string `json:"Image"` - Volumes map[string]struct{} `json:"Volumes"` - WorkingDir string `json:"WorkingDir"` - Entrypoint string `json:"Entrypoint"` - Labels map[string]string `json:"Labels"` - Annotations map[string]string `json:"Annotations"` - StopSignal uint `json:"StopSignal"` - Healthcheck *manifest.Schema2HealthConfig `json:"Healthcheck,omitempty"` -} - -// LogConfig holds the log information for a container -type LogConfig struct { - Type string `json:"Type"` // TODO - Config map[string]string `json:"Config"` //idk type, TODO -} - // ImageData holds the inspect information of an image type ImageData struct { ID string `json:"Id"` @@ -123,7 +24,7 @@ type ImageData struct { Os string `json:"Os"` Size int64 `json:"Size"` VirtualSize int64 `json:"VirtualSize"` - GraphDriver *Data `json:"GraphDriver"` + GraphDriver *driver.Data `json:"GraphDriver"` RootFS *RootFS `json:"RootFS"` Labels map[string]string `json:"Labels"` Annotations map[string]string `json:"Annotations"` @@ -138,86 +39,6 @@ type RootFS struct { Layers []digest.Digest `json:"Layers"` } -// Data handles the data for a storage driver -type Data struct { - Name string `json:"Name"` - Data map[string]string `json:"Data"` -} - -// ContainerInspectData handles the data used when inspecting a container -type ContainerInspectData struct { - ID string `json:"ID"` - Created time.Time `json:"Created"` - Path string `json:"Path"` - Args []string `json:"Args"` - State *ContainerInspectState `json:"State"` - ImageID string `json:"Image"` - ImageName string `json:"ImageName"` - Rootfs string `json:"Rootfs"` - ResolvConfPath string `json:"ResolvConfPath"` - HostnamePath string `json:"HostnamePath"` - HostsPath string `json:"HostsPath"` - StaticDir string `json:"StaticDir"` - LogPath string `json:"LogPath"` - ConmonPidFile string `json:"ConmonPidFile"` - Name string `json:"Name"` - RestartCount int32 `json:"RestartCount"` - Driver string `json:"Driver"` - MountLabel string `json:"MountLabel"` - ProcessLabel string `json:"ProcessLabel"` - AppArmorProfile string `json:"AppArmorProfile"` - EffectiveCaps []string `json:"EffectiveCaps"` - BoundingCaps []string `json:"BoundingCaps"` - ExecIDs []string `json:"ExecIDs"` - GraphDriver *Data `json:"GraphDriver"` - SizeRw int64 `json:"SizeRw,omitempty"` - SizeRootFs int64 `json:"SizeRootFs,omitempty"` - Mounts []specs.Mount `json:"Mounts"` - Dependencies []string `json:"Dependencies"` - NetworkSettings *NetworkSettings `json:"NetworkSettings"` //TODO - ExitCommand []string `json:"ExitCommand"` - Namespace string `json:"Namespace"` - IsInfra bool `json:"IsInfra"` -} - -// ContainerInspectState represents the state of a container. -type ContainerInspectState struct { - OciVersion string `json:"OciVersion"` - Status string `json:"Status"` - Running bool `json:"Running"` - Paused bool `json:"Paused"` - Restarting bool `json:"Restarting"` // TODO - OOMKilled bool `json:"OOMKilled"` - Dead bool `json:"Dead"` - Pid int `json:"Pid"` - ExitCode int32 `json:"ExitCode"` - Error string `json:"Error"` // TODO - StartedAt time.Time `json:"StartedAt"` - FinishedAt time.Time `json:"FinishedAt"` - Healthcheck HealthCheckResults `json:"Healthcheck,omitempty"` -} - -// NetworkSettings holds information about the newtwork settings of the container -type NetworkSettings struct { - Bridge string `json:"Bridge"` - SandboxID string `json:"SandboxID"` - HairpinMode bool `json:"HairpinMode"` - LinkLocalIPv6Address string `json:"LinkLocalIPv6Address"` - LinkLocalIPv6PrefixLen int `json:"LinkLocalIPv6PrefixLen"` - Ports []ocicni.PortMapping `json:"Ports"` - SandboxKey string `json:"SandboxKey"` - SecondaryIPAddresses []string `json:"SecondaryIPAddresses"` - SecondaryIPv6Addresses []string `json:"SecondaryIPv6Addresses"` - EndpointID string `json:"EndpointID"` - Gateway string `json:"Gateway"` - GlobalIPv6Address string `json:"GlobalIPv6Address"` - GlobalIPv6PrefixLen int `json:"GlobalIPv6PrefixLen"` - IPAddress string `json:"IPAddress"` - IPPrefixLen int `json:"IPPrefixLen"` - IPv6Gateway string `json:"IPv6Gateway"` - MacAddress string `json:"MacAddress"` -} - // ImageResult is used for podman images for collection and output type ImageResult struct { Tag string @@ -232,25 +53,3 @@ type ImageResult struct { Labels map[string]string Dangling bool } - -// HealthCheckResults describes the results/logs from a healthcheck -type HealthCheckResults struct { - // Status healthy or unhealthy - Status string `json:"Status"` - // FailingStreak is the number of consecutive failed healthchecks - FailingStreak int `json:"FailingStreak"` - // Log describes healthcheck attempts and results - Log []HealthCheckLog `json:"Log"` -} - -// HealthCheckLog describes the results of a single healthcheck -type HealthCheckLog struct { - // Start time as string - Start string `json:"Start"` - // End time as a string - End string `json:"End"` - // Exitcode is 0 or 1 - ExitCode int `json:"ExitCode"` - // Output is the stdout/stderr from the healthcheck command - Output string `json:"Output"` -} diff --git a/pkg/rootless/rootless_linux.c b/pkg/rootless/rootless_linux.c index 098ca7830..eb62d55e9 100644 --- a/pkg/rootless/rootless_linux.c +++ b/pkg/rootless/rootless_linux.c @@ -34,6 +34,15 @@ int renameat2 (int olddirfd, const char *oldpath, int newdirfd, const char *newp } #endif +#ifndef TEMP_FAILURE_RETRY +#define TEMP_FAILURE_RETRY(expression) \ + (__extension__ \ + ({ long int __result; \ + do __result = (long int) (expression); \ + while (__result == -1L && errno == EINTR); \ + __result; })) +#endif + static const char *_max_user_namespaces = "/proc/sys/user/max_user_namespaces"; static const char *_unprivileged_user_namespaces = "/proc/sys/kernel/unprivileged_userns_clone"; @@ -69,6 +78,19 @@ rootless_gid () static void do_pause () { + int i; + struct sigaction act; + int const sig[] = + { + SIGALRM, SIGHUP, SIGINT, SIGPIPE, SIGQUIT, SIGTERM, SIGPOLL, + SIGPROF, SIGVTALRM, SIGXCPU, SIGXFSZ, 0 + }; + + act.sa_handler = SIG_IGN; + + for (i = 0; sig[i]; i++) + sigaction (sig[i], &act, NULL); + prctl (PR_SET_NAME, "podman pause", NULL, NULL, NULL); while (1) pause (); @@ -100,9 +122,7 @@ get_cmd_line_args (pid_t pid) return NULL; for (;;) { - do - ret = read (fd, buffer + used, allocated - used); - while (ret < 0 && errno == EINTR); + ret = TEMP_FAILURE_RETRY (read (fd, buffer + used, allocated - used)); if (ret < 0) { free (buffer); @@ -167,7 +187,7 @@ can_use_shortcut () argv = get_cmd_line_args (0); if (argv == NULL) - return NULL; + return false; for (argc = 0; argv[argc]; argc++) { @@ -256,13 +276,15 @@ static void __attribute__((constructor)) init() return; } - r = read (fd, buf, sizeof (buf)); + r = TEMP_FAILURE_RETRY (read (fd, buf, sizeof (buf))); close (fd); if (r < 0) { free (cwd); return; } + buf[r] = '\0'; + pid = strtol (buf, NULL, 10); if (pid == LONG_MAX) { @@ -333,6 +355,23 @@ syscall_clone (unsigned long flags, void *child_stack) #endif } +int +reexec_in_user_namespace_wait (int pid, int options) +{ + pid_t p; + int status; + + p = TEMP_FAILURE_RETRY (waitpid (pid, &status, 0)); + if (p < 0) + return -1; + + if (WIFEXITED (status)) + return WEXITSTATUS (status); + if (WIFSIGNALED (status)) + return 128 + WTERMSIG (status); + return -1; +} + static int create_pause_process (const char *pause_pid_file_path, char **argv) { @@ -351,11 +390,11 @@ create_pause_process (const char *pause_pid_file_path, char **argv) close (p[1]); /* Block until we write the pid file. */ - do - r = read (p[0], &b, 1); - while (r < 0 && errno == EINTR); + r = TEMP_FAILURE_RETRY (read (p[0], &b, 1)); close (p[0]); + reexec_in_user_namespace_wait (r, 0); + return r == 1 && b == '0' ? 0 : -1; } else @@ -391,9 +430,7 @@ create_pause_process (const char *pause_pid_file_path, char **argv) _exit (EXIT_FAILURE); } - do - r = write (fd, pid_str, strlen (pid_str)); - while (r < 0 && errno == EINTR); + r = TEMP_FAILURE_RETRY (write (fd, pid_str, strlen (pid_str))); if (r < 0) { kill (pid, SIGKILL); @@ -410,9 +447,7 @@ create_pause_process (const char *pause_pid_file_path, char **argv) _exit (EXIT_FAILURE); } - do - r = write (p[1], "0", 1); - while (r < 0 && errno == EINTR); + r = TEMP_FAILURE_RETRY (write (p[1], "0", 1)); close (p[1]); _exit (EXIT_SUCCESS); @@ -454,6 +489,7 @@ reexec_userns_join (int userns, int mountns, char *pause_pid_file_path) char **argv; int pid; char *cwd = getcwd (NULL, 0); + sigset_t sigset, oldsigset; if (cwd == NULL) { @@ -487,6 +523,22 @@ reexec_userns_join (int userns, int mountns, char *pause_pid_file_path) return pid; } + if (sigfillset (&sigset) < 0) + { + fprintf (stderr, "cannot fill sigset: %s\n", strerror (errno)); + _exit (EXIT_FAILURE); + } + if (sigdelset (&sigset, SIGCHLD) < 0) + { + fprintf (stderr, "cannot sigdelset(SIGCHLD): %s\n", strerror (errno)); + _exit (EXIT_FAILURE); + } + if (sigprocmask (SIG_BLOCK, &sigset, &oldsigset) < 0) + { + fprintf (stderr, "cannot block signals: %s\n", strerror (errno)); + _exit (EXIT_FAILURE); + } + setenv ("_CONTAINERS_USERNS_CONFIGURED", "init", 1); setenv ("_CONTAINERS_ROOTLESS_UID", uid, 1); setenv ("_CONTAINERS_ROOTLESS_GID", gid, 1); @@ -535,6 +587,11 @@ reexec_userns_join (int userns, int mountns, char *pause_pid_file_path) /* We ignore errors here as we didn't create the namespace anyway. */ create_pause_process (pause_pid_file_path, argv); } + if (sigprocmask (SIG_SETMASK, &oldsigset, NULL) < 0) + { + fprintf (stderr, "cannot block signals: %s\n", strerror (errno)); + _exit (EXIT_FAILURE); + } execvp (argv[0], argv); @@ -560,8 +617,47 @@ check_proc_sys_userns_file (const char *path) } } +static int +copy_file_to_fd (const char *file_to_read, int outfd) +{ + char buf[512]; + int fd; + + fd = open (file_to_read, O_RDONLY); + if (fd < 0) + return fd; + + for (;;) + { + ssize_t r, w, t = 0; + + r = TEMP_FAILURE_RETRY (read (fd, buf, sizeof buf)); + if (r < 0) + { + close (fd); + return r; + } + + if (r == 0) + break; + + while (t < r) + { + w = TEMP_FAILURE_RETRY (write (outfd, &buf[t], r - t)); + if (w < 0) + { + close (fd); + return w; + } + t += w; + } + } + close (fd); + return 0; +} + int -reexec_in_user_namespace (int ready, char *pause_pid_file_path) +reexec_in_user_namespace (int ready, char *pause_pid_file_path, char *file_to_read, int outputfd) { int ret; pid_t pid; @@ -574,6 +670,7 @@ reexec_in_user_namespace (int ready, char *pause_pid_file_path) char *listen_pid = NULL; bool do_socket_activation = false; char *cwd = getcwd (NULL, 0); + sigset_t sigset, oldsigset; if (cwd == NULL) { @@ -584,11 +681,11 @@ reexec_in_user_namespace (int ready, char *pause_pid_file_path) listen_pid = getenv("LISTEN_PID"); listen_fds = getenv("LISTEN_FDS"); - if (listen_pid != NULL && listen_fds != NULL) { - if (strtol(listen_pid, NULL, 10) == getpid()) { - do_socket_activation = true; + if (listen_pid != NULL && listen_fds != NULL) + { + if (strtol(listen_pid, NULL, 10) == getpid()) + do_socket_activation = true; } - } sprintf (uid, "%d", geteuid ()); sprintf (gid, "%d", getegid ()); @@ -621,6 +718,22 @@ reexec_in_user_namespace (int ready, char *pause_pid_file_path) return pid; } + if (sigfillset (&sigset) < 0) + { + fprintf (stderr, "cannot fill sigset: %s\n", strerror (errno)); + _exit (EXIT_FAILURE); + } + if (sigdelset (&sigset, SIGCHLD) < 0) + { + fprintf (stderr, "cannot sigdelset(SIGCHLD): %s\n", strerror (errno)); + _exit (EXIT_FAILURE); + } + if (sigprocmask (SIG_BLOCK, &sigset, &oldsigset) < 0) + { + fprintf (stderr, "cannot block signals: %s\n", strerror (errno)); + _exit (EXIT_FAILURE); + } + argv = get_cmd_line_args (ppid); if (argv == NULL) { @@ -628,19 +741,18 @@ reexec_in_user_namespace (int ready, char *pause_pid_file_path) _exit (EXIT_FAILURE); } - if (do_socket_activation) { - char s[32]; - sprintf (s, "%d", getpid()); - setenv ("LISTEN_PID", s, true); - } + if (do_socket_activation) + { + char s[32]; + sprintf (s, "%d", getpid()); + setenv ("LISTEN_PID", s, true); + } setenv ("_CONTAINERS_USERNS_CONFIGURED", "init", 1); setenv ("_CONTAINERS_ROOTLESS_UID", uid, 1); setenv ("_CONTAINERS_ROOTLESS_GID", gid, 1); - do - ret = read (ready, &b, 1) < 0; - while (ret < 0 && errno == EINTR); + ret = TEMP_FAILURE_RETRY (read (ready, &b, 1)); if (ret < 0) { fprintf (stderr, "cannot read from sync pipe: %s\n", strerror (errno)); @@ -652,21 +764,21 @@ reexec_in_user_namespace (int ready, char *pause_pid_file_path) if (syscall_setresgid (0, 0, 0) < 0) { fprintf (stderr, "cannot setresgid: %s\n", strerror (errno)); - write (ready, "1", 1); + TEMP_FAILURE_RETRY (write (ready, "1", 1)); _exit (EXIT_FAILURE); } if (syscall_setresuid (0, 0, 0) < 0) { fprintf (stderr, "cannot setresuid: %s\n", strerror (errno)); - write (ready, "1", 1); + TEMP_FAILURE_RETRY (write (ready, "1", 1)); _exit (EXIT_FAILURE); } if (chdir (cwd) < 0) { fprintf (stderr, "cannot chdir: %s\n", strerror (errno)); - write (ready, "1", 1); + TEMP_FAILURE_RETRY (write (ready, "1", 1)); _exit (EXIT_FAILURE); } free (cwd); @@ -675,37 +787,28 @@ reexec_in_user_namespace (int ready, char *pause_pid_file_path) { if (create_pause_process (pause_pid_file_path, argv) < 0) { - write (ready, "2", 1); + TEMP_FAILURE_RETRY (write (ready, "2", 1)); _exit (EXIT_FAILURE); } } - do - ret = write (ready, "0", 1) < 0; - while (ret < 0 && errno == EINTR); + ret = TEMP_FAILURE_RETRY (write (ready, "0", 1)); close (ready); - execvp (argv[0], argv); - - _exit (EXIT_FAILURE); -} - -int -reexec_in_user_namespace_wait (int pid) -{ - pid_t p; - int status; + if (sigprocmask (SIG_SETMASK, &oldsigset, NULL) < 0) + { + fprintf (stderr, "cannot block signals: %s\n", strerror (errno)); + _exit (EXIT_FAILURE); + } - do - p = waitpid (pid, &status, 0); - while (p < 0 && errno == EINTR); + if (file_to_read && file_to_read[0]) + { + ret = copy_file_to_fd (file_to_read, outputfd); + close (outputfd); + _exit (ret == 0 ? EXIT_SUCCESS : EXIT_FAILURE); + } - if (p < 0) - return -1; + execvp (argv[0], argv); - if (WIFEXITED (status)) - return WEXITSTATUS (status); - if (WIFSIGNALED (status)) - return 128 + WTERMSIG (status); - return -1; + _exit (EXIT_FAILURE); } diff --git a/pkg/rootless/rootless_linux.go b/pkg/rootless/rootless_linux.go index 9132c0fe5..3f78ffc67 100644 --- a/pkg/rootless/rootless_linux.go +++ b/pkg/rootless/rootless_linux.go @@ -24,10 +24,11 @@ import ( /* #cgo remoteclient CFLAGS: -DDISABLE_JOIN_SHORTCUT #include <stdlib.h> +#include <sys/types.h> extern uid_t rootless_uid(); extern uid_t rootless_gid(); -extern int reexec_in_user_namespace(int ready, char *pause_pid_file_path); -extern int reexec_in_user_namespace_wait(int pid); +extern int reexec_in_user_namespace(int ready, char *pause_pid_file_path, char *file_to_read, int fd); +extern int reexec_in_user_namespace_wait(int pid, int options); extern int reexec_userns_join(int userns, int mountns, char *pause_pid_file_path); */ import "C" @@ -169,6 +170,9 @@ func getUserNSFirstChild(fd uintptr) (*os.File, error) { for { nextFd, err := getParentUserNs(fd) if err != nil { + if err == syscall.ENOTTY { + return os.NewFile(fd, "userns child"), nil + } return nil, errors.Wrapf(err, "cannot get parent user namespace") } @@ -194,10 +198,24 @@ func getUserNSFirstChild(fd uintptr) (*os.File, error) { } } -// JoinUserAndMountNS re-exec podman in a new userNS and join the user and mount +func enableLinger(pausePid string) { + if pausePid == "" { + return + } + // If we are trying to write a pause pid file, make sure we can leave processes + // running longer than the user session. + err := exec.Command("loginctl", "enable-linger", fmt.Sprintf("%d", GetRootlessUID())).Run() + if err != nil { + logrus.Warnf("cannot run `loginctl enable-linger` for the current user: %v", err) + } +} + +// joinUserAndMountNS re-exec podman in a new userNS and join the user and mount // namespace of the specified PID without looking up its parent. Useful to join directly // the conmon process. -func JoinUserAndMountNS(pid uint, pausePid string) (bool, int, error) { +func joinUserAndMountNS(pid uint, pausePid string) (bool, int, error) { + enableLinger(pausePid) + if os.Geteuid() == 0 || os.Getenv("_CONTAINERS_USERNS_CONFIGURED") != "" { return false, -1, nil } @@ -226,7 +244,7 @@ func JoinUserAndMountNS(pid uint, pausePid string) (bool, int, error) { return false, -1, errors.Errorf("cannot re-exec process") } - ret := C.reexec_in_user_namespace_wait(pidC) + ret := C.reexec_in_user_namespace_wait(pidC, 0) if ret < 0 { return false, -1, errors.New("error waiting for the re-exec process") } @@ -234,11 +252,7 @@ func JoinUserAndMountNS(pid uint, pausePid string) (bool, int, error) { return true, int(ret), nil } -// BecomeRootInUserNS re-exec podman in a new userNS. It returns whether podman was re-executed -// into a new user namespace and the return code from the re-executed podman process. -// If podman was re-executed the caller needs to propagate the error code returned by the child -// process. -func BecomeRootInUserNS(pausePid string) (bool, int, error) { +func becomeRootInUserNS(pausePid, fileToRead string, fileOutput *os.File) (bool, int, error) { if os.Geteuid() == 0 || os.Getenv("_CONTAINERS_USERNS_CONFIGURED") != "" { if os.Getenv("_CONTAINERS_USERNS_CONFIGURED") == "init" { return false, 0, runInUser() @@ -249,6 +263,13 @@ func BecomeRootInUserNS(pausePid string) (bool, int, error) { cPausePid := C.CString(pausePid) defer C.free(unsafe.Pointer(cPausePid)) + cFileToRead := C.CString(fileToRead) + defer C.free(unsafe.Pointer(cFileToRead)) + var fileOutputFD C.int + if fileOutput != nil { + fileOutputFD = C.int(fileOutput.Fd()) + } + runtime.LockOSThread() defer runtime.UnlockOSThread() @@ -262,7 +283,7 @@ func BecomeRootInUserNS(pausePid string) (bool, int, error) { defer w.Close() defer w.Write([]byte("0")) - pidC := C.reexec_in_user_namespace(C.int(r.Fd()), cPausePid) + pidC := C.reexec_in_user_namespace(C.int(r.Fd()), cPausePid, cFileToRead, fileOutputFD) pid := int(pidC) if pid < 0 { return false, -1, errors.Errorf("cannot re-exec process") @@ -328,6 +349,10 @@ func BecomeRootInUserNS(pausePid string) (bool, int, error) { return false, -1, errors.Wrapf(err, "read from sync pipe") } + if fileOutput != nil { + return true, 0, nil + } + if b[0] == '2' { // We have lost the race for writing the PID file, as probably another // process created a namespace and wrote the PID. @@ -336,7 +361,7 @@ func BecomeRootInUserNS(pausePid string) (bool, int, error) { if err == nil { pid, err := strconv.ParseUint(string(data), 10, 0) if err == nil { - return JoinUserAndMountNS(uint(pid), "") + return joinUserAndMountNS(uint(pid), "") } } return false, -1, errors.Wrapf(err, "error setting up the process") @@ -368,10 +393,96 @@ func BecomeRootInUserNS(pausePid string) (bool, int, error) { } }() - ret := C.reexec_in_user_namespace_wait(pidC) + ret := C.reexec_in_user_namespace_wait(pidC, 0) if ret < 0 { return false, -1, errors.New("error waiting for the re-exec process") } return true, int(ret), nil } + +// BecomeRootInUserNS re-exec podman in a new userNS. It returns whether podman was re-executed +// into a new user namespace and the return code from the re-executed podman process. +// If podman was re-executed the caller needs to propagate the error code returned by the child +// process. +func BecomeRootInUserNS(pausePid string) (bool, int, error) { + enableLinger(pausePid) + return becomeRootInUserNS(pausePid, "", nil) +} + +// TryJoinFromFilePaths attempts to join the namespaces of the pid files in paths. +// This is useful when there are already running containers and we +// don't have a pause process yet. We can use the paths to the conmon +// processes to attempt joining their namespaces. +// If needNewNamespace is set, the file is read from a temporary user +// namespace, this is useful for containers that are running with a +// different uidmap and the unprivileged user has no way to read the +// file owned by the root in the container. +func TryJoinFromFilePaths(pausePidPath string, needNewNamespace bool, paths []string) (bool, int, error) { + if len(paths) == 0 { + return BecomeRootInUserNS(pausePidPath) + } + + var lastErr error + var pausePid int + + for _, path := range paths { + if !needNewNamespace { + data, err := ioutil.ReadFile(path) + if err != nil { + lastErr = err + continue + } + + pausePid, err = strconv.Atoi(string(data)) + if err != nil { + lastErr = errors.Wrapf(err, "cannot parse file %s", path) + continue + } + + lastErr = nil + break + } else { + fds, err := syscall.Socketpair(syscall.AF_UNIX, syscall.SOCK_DGRAM, 0) + if err != nil { + lastErr = err + continue + } + + r, w := os.NewFile(uintptr(fds[0]), "read file"), os.NewFile(uintptr(fds[1]), "write file") + + defer w.Close() + defer r.Close() + + if _, _, err := becomeRootInUserNS("", path, w); err != nil { + lastErr = err + continue + } + + w.Close() + defer func() { + r.Close() + C.reexec_in_user_namespace_wait(-1, 0) + }() + + b := make([]byte, 32) + + n, err := r.Read(b) + if err != nil { + lastErr = errors.Wrapf(err, "cannot read %s\n", path) + continue + } + + pausePid, err = strconv.Atoi(string(b[:n])) + if err == nil { + lastErr = nil + break + } + } + } + if lastErr != nil { + return false, 0, lastErr + } + + return joinUserAndMountNS(uint(pausePid), pausePidPath) +} diff --git a/pkg/rootless/rootless_unsupported.go b/pkg/rootless/rootless_unsupported.go index 221baff97..c063adee5 100644 --- a/pkg/rootless/rootless_unsupported.go +++ b/pkg/rootless/rootless_unsupported.go @@ -29,10 +29,14 @@ func GetRootlessGID() int { return -1 } -// JoinUserAndMountNS re-exec podman in a new userNS and join the user and mount -// namespace of the specified PID without looking up its parent. Useful to join directly -// the conmon process. It is a convenience function for JoinUserAndMountNSWithOpts -// with a default configuration. -func JoinUserAndMountNS(pid uint, pausePid string) (bool, int, error) { +// TryJoinFromFilePaths attempts to join the namespaces of the pid files in paths. +// This is useful when there are already running containers and we +// don't have a pause process yet. We can use the paths to the conmon +// processes to attempt joining their namespaces. +// If needNewNamespace is set, the file is read from a temporary user +// namespace, this is useful for containers that are running with a +// different uidmap and the unprivileged user has no way to read the +// file owned by the root in the container. +func TryJoinFromFilePaths(pausePidPath string, needNewNamespace bool, paths []string) (bool, int, error) { return false, -1, errors.New("this function is not supported on this os") } diff --git a/pkg/spec/createconfig.go b/pkg/spec/createconfig.go index 9979e773c..e4501aaac 100644 --- a/pkg/spec/createconfig.go +++ b/pkg/spec/createconfig.go @@ -319,6 +319,9 @@ func (c *CreateConfig) getContainerCreateOptions(runtime *libpod.Runtime, pod *l if logPath != "" { options = append(options, libpod.WithLogPath(logPath)) } + + options = append(options, libpod.WithLogDriver(c.LogDriver)) + if c.IPAddress != "" { ip := net.ParseIP(c.IPAddress) if ip == nil { diff --git a/pkg/spec/storage.go b/pkg/spec/storage.go index dcc149b55..e221b5cb5 100644 --- a/pkg/spec/storage.go +++ b/pkg/spec/storage.go @@ -797,7 +797,7 @@ func initFSMounts(inputMounts []spec.Mount) []spec.Mount { if m.Type == TypeBind { m.Options = util.ProcessOptions(m.Options) } - if m.Type == TypeTmpfs { + if m.Type == TypeTmpfs && filepath.Clean(m.Destination) != "/dev" { m.Options = append(m.Options, "tmpcopyup") } mounts = append(mounts, m) diff --git a/pkg/varlinkapi/attach.go b/pkg/varlinkapi/attach.go index 2234899a5..8051f07be 100644 --- a/pkg/varlinkapi/attach.go +++ b/pkg/varlinkapi/attach.go @@ -60,7 +60,10 @@ func (i *LibpodAPI) Attach(call iopodman.VarlinkCall, name string, detachKeys st if !start && state != libpod.ContainerStateRunning { return call.ReplyErrorOccurred("container must be running to attach") } - call.Reply(nil) + + // ACK the client upgrade request + call.ReplyAttach() + reader, writer, _, pw, streams := setupStreams(call) go func() { diff --git a/pkg/varlinkapi/images.go b/pkg/varlinkapi/images.go index fa1a0a109..1abc4f086 100644 --- a/pkg/varlinkapi/images.go +++ b/pkg/varlinkapi/images.go @@ -371,7 +371,6 @@ func (i *LibpodAPI) PushImage(call iopodman.VarlinkCall, name, tag string, compr done = true default: if !call.WantsMore() { - time.Sleep(1 * time.Second) break } br := iopodman.MoreResponse{ @@ -495,6 +494,9 @@ func (i *LibpodAPI) DeleteUnusedImages(call iopodman.VarlinkCall) error { // Commit ... func (i *LibpodAPI) Commit(call iopodman.VarlinkCall, name, imageName string, changes []string, author, message string, pause bool, manifestType string) error { + var newImage *image.Image + + output := bytes.NewBuffer([]byte{}) ctr, err := i.Runtime.LookupContainer(name) if err != nil { return call.ReplyContainerNotFound(name, err.Error()) @@ -515,7 +517,7 @@ func (i *LibpodAPI) Commit(call iopodman.VarlinkCall, name, imageName string, ch } coptions := buildah.CommitOptions{ SignaturePolicyPath: rtc.SignaturePolicyPath, - ReportWriter: nil, + ReportWriter: output, SystemContext: sc, PreferredManifestType: mimeType, } @@ -527,11 +529,61 @@ func (i *LibpodAPI) Commit(call iopodman.VarlinkCall, name, imageName string, ch Author: author, } - newImage, err := ctr.Commit(getContext(), imageName, options) - if err != nil { - return call.ReplyErrorOccurred(err.Error()) + if call.WantsMore() { + call.Continues = true } - return call.ReplyCommit(newImage.ID()) + + c := make(chan error) + + go func() { + newImage, err = ctr.Commit(getContext(), imageName, options) + if err != nil { + c <- err + } + c <- nil + close(c) + }() + + var log []string + done := false + for { + line, err := output.ReadString('\n') + if err == nil { + log = append(log, line) + continue + } else if err == io.EOF { + select { + case err := <-c: + if err != nil { + logrus.Errorf("reading of output during commit failed for %s", name) + return call.ReplyErrorOccurred(err.Error()) + } + done = true + default: + if !call.WantsMore() { + break + } + br := iopodman.MoreResponse{ + Logs: log, + } + call.ReplyCommit(br) + log = []string{} + } + } else { + return call.ReplyErrorOccurred(err.Error()) + } + if done { + break + } + } + call.Continues = false + + br := iopodman.MoreResponse{ + Logs: log, + Id: newImage.ID(), + } + + return call.ReplyCommit(br) } // ImportImage imports an image from a tarball to the image store @@ -633,7 +685,6 @@ func (i *LibpodAPI) PullImage(call iopodman.VarlinkCall, name string) error { done = true default: if !call.WantsMore() { - time.Sleep(1 * time.Second) break } br := iopodman.MoreResponse{ @@ -764,7 +815,6 @@ func (i *LibpodAPI) ImageSave(call iopodman.VarlinkCall, options iopodman.ImageS done = true default: if !call.WantsMore() { - time.Sleep(1 * time.Second) break } br := iopodman.MoreResponse{ @@ -844,7 +894,6 @@ func (i *LibpodAPI) LoadImage(call iopodman.VarlinkCall, name, inputFile string, done = true default: if !call.WantsMore() { - time.Sleep(1 * time.Second) break } br := iopodman.MoreResponse{ diff --git a/pkg/varlinkapi/transfers.go b/pkg/varlinkapi/transfers.go index 96f76bcdc..24a91a86f 100644 --- a/pkg/varlinkapi/transfers.go +++ b/pkg/varlinkapi/transfers.go @@ -29,6 +29,12 @@ func (i *LibpodAPI) SendFile(call iopodman.VarlinkCall, ftype string, length int return call.ReplyErrorOccurred(err.Error()) } + // FIXME return parameter + if err = call.ReplySendFile("FIXME_file_handle"); err != nil { + // If an error occurs while sending the reply, return the error + return err + } + writer := bufio.NewWriter(outputFile) defer writer.Flush() @@ -60,9 +66,10 @@ func (i *LibpodAPI) ReceiveFile(call iopodman.VarlinkCall, filepath string, dele } // Send the file length down to client - // Varlink connection upraded + // Varlink connection upgraded if err = call.ReplyReceiveFile(fileInfo.Size()); err != nil { - return call.ReplyErrorOccurred(err.Error()) + // If an error occurs while sending the reply, return the error + return err } reader := bufio.NewReader(fs) diff --git a/pkg/varlinkapi/virtwriter/virtwriter.go b/pkg/varlinkapi/virtwriter/virtwriter.go index 3adaf6e17..e747984c7 100644 --- a/pkg/varlinkapi/virtwriter/virtwriter.go +++ b/pkg/varlinkapi/virtwriter/virtwriter.go @@ -91,65 +91,65 @@ func (v VirtWriteCloser) Write(input []byte) (int, error) { // Reader decodes the content that comes over the wire and directs it to the proper destination. func Reader(r *bufio.Reader, output, errput *os.File, input *io.PipeWriter, resize chan remotecommand.TerminalSize) error { - var saveb []byte - var eom int + var messageSize int64 + headerBytes := make([]byte, 8) + for { - readb := make([]byte, 32*1024) - n, err := r.Read(readb) - // TODO, later may be worth checking in len of the read is 0 + n, err := io.ReadFull(r, headerBytes) if err != nil { return err } - b := append(saveb, readb[0:n]...) - // no sense in reading less than the header len - for len(b) > 7 { - eom = int(binary.BigEndian.Uint32(b[4:8])) + 8 - // The message and header are togther - if len(b) >= eom { - out := append([]byte{}, b[8:eom]...) - - switch IntToSocketDest(int(b[0])) { - case ToStdout: - n, err := output.Write(out) - if err != nil { - return err - } - if n < len(out) { - return errors.New("short write error occurred on stdout") - } - case ToStderr: - n, err := errput.Write(out) - if err != nil { - return err - } - if n < len(out) { - return errors.New("short write error occurred on stderr") - } - case ToStdin: - n, err := input.Write(out) - if err != nil { - return err - } - if n < len(out) { - return errors.New("short write error occurred on stdin") - } - case TerminalResize: - // Resize events come over in bytes, need to be reserialized - resizeEvent := remotecommand.TerminalSize{} - if err := json.Unmarshal(out, &resizeEvent); err != nil { - return err - } - resize <- resizeEvent - case Quit: - return nil + if n < 8 { + return errors.New("short read and no full header read") + } + + messageSize = int64(binary.BigEndian.Uint32(headerBytes[4:8])) + + switch IntToSocketDest(int(headerBytes[0])) { + case ToStdout: + _, err := io.CopyN(output, r, messageSize) + if err != nil { + return err + } + case ToStderr: + _, err := io.CopyN(errput, r, messageSize) + if err != nil { + return err + } + case ToStdin: + _, err := io.CopyN(input, r, messageSize) + if err != nil { + return err + } + case TerminalResize: + out := make([]byte, messageSize) + if messageSize > 0 { + _, err = io.ReadFull(r, out) + + if err != nil { + return err } - b = b[eom:] - } else { - // We do not have the header and full message, need to slurp again - saveb = b - break } + // Resize events come over in bytes, need to be reserialized + resizeEvent := remotecommand.TerminalSize{} + if err := json.Unmarshal(out, &resizeEvent); err != nil { + return err + } + resize <- resizeEvent + case Quit: + out := make([]byte, messageSize) + if messageSize > 0 { + _, err = io.ReadFull(r, out) + + if err != nil { + return err + } + } + return nil + + default: + // Something really went wrong + return errors.New("Unknown multiplex destination") } } - return nil } diff --git a/test/e2e/attach_test.go b/test/e2e/attach_test.go index a843fe7ff..7233d169c 100644 --- a/test/e2e/attach_test.go +++ b/test/e2e/attach_test.go @@ -26,7 +26,7 @@ var _ = Describe("Podman attach", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() }) AfterEach(func() { diff --git a/test/e2e/checkpoint_test.go b/test/e2e/checkpoint_test.go index c2f5a592c..d452a062b 100644 --- a/test/e2e/checkpoint_test.go +++ b/test/e2e/checkpoint_test.go @@ -28,7 +28,7 @@ var _ = Describe("Podman checkpoint", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() // Check if the runtime implements checkpointing. Currently only // runc's checkpoint/restore implementation is supported. cmd := exec.Command(podmanTest.OCIRuntime, "checkpoint", "-h") @@ -221,7 +221,6 @@ var _ = Describe("Podman checkpoint", func() { }) It("podman checkpoint container with established tcp connections", func() { - podmanTest.RestoreArtifact(redis) session := podmanTest.Podman([]string{"run", "-it", "--security-opt", "seccomp=unconfined", "-d", redis}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) @@ -348,4 +347,49 @@ var _ = Describe("Podman checkpoint", func() { Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0)) }) + // This test does the same steps which are necessary for migrating + // a container from one host to another + It("podman checkpoint container with export (migration)", func() { + // CRIU does not work with seccomp correctly on RHEL7 + session := podmanTest.Podman([]string{"run", "-it", "--security-opt", "seccomp=unconfined", "-d", ALPINE, "top"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1)) + + result := podmanTest.Podman([]string{"container", "checkpoint", "-l", "-e", "/tmp/checkpoint.tar.gz"}) + result.WaitWithDefaultTimeout() + + Expect(result.ExitCode()).To(Equal(0)) + Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0)) + Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Exited")) + + // Remove all containers to simulate migration + result = podmanTest.Podman([]string{"rm", "-fa"}) + result.WaitWithDefaultTimeout() + Expect(result.ExitCode()).To(Equal(0)) + Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0)) + + result = podmanTest.Podman([]string{"container", "restore", "-i", "/tmp/checkpoint.tar.gz"}) + result.WaitWithDefaultTimeout() + + Expect(result.ExitCode()).To(Equal(0)) + Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1)) + Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Up")) + + // Restore container a second time with different name + result = podmanTest.Podman([]string{"container", "restore", "-i", "/tmp/checkpoint.tar.gz", "-n", "restore_again"}) + result.WaitWithDefaultTimeout() + + Expect(result.ExitCode()).To(Equal(0)) + Expect(podmanTest.NumberOfContainersRunning()).To(Equal(2)) + Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Up")) + + result = podmanTest.Podman([]string{"rm", "-fa"}) + result.WaitWithDefaultTimeout() + Expect(result.ExitCode()).To(Equal(0)) + Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0)) + + // Remove exported checkpoint + os.Remove("/tmp/checkpoint.tar.gz") + }) }) diff --git a/test/e2e/commit_test.go b/test/e2e/commit_test.go index bf20ac999..e9d274649 100644 --- a/test/e2e/commit_test.go +++ b/test/e2e/commit_test.go @@ -1,5 +1,3 @@ -// +build !remoteclient - package integration import ( @@ -24,7 +22,7 @@ var _ = Describe("Podman commit", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() }) AfterEach(func() { @@ -174,6 +172,9 @@ var _ = Describe("Podman commit", func() { }) It("podman commit with volume mounts and --include-volumes", func() { + // We need to figure out how volumes are going to work correctly with the remote + // client. This does not currently work. + SkipIfRemote() s := podmanTest.Podman([]string{"run", "--name", "test1", "-v", "/tmp:/foo", "alpine", "date"}) s.WaitWithDefaultTimeout() Expect(s.ExitCode()).To(Equal(0)) diff --git a/test/e2e/common_test.go b/test/e2e/common_test.go index 3c7675b35..9529346b4 100644 --- a/test/e2e/common_test.go +++ b/test/e2e/common_test.go @@ -11,6 +11,7 @@ import ( "strings" "testing" + "github.com/containers/libpod/cmd/podman/shared" "github.com/containers/libpod/libpod" "github.com/containers/libpod/pkg/inspect" "github.com/containers/libpod/pkg/rootless" @@ -100,12 +101,23 @@ var _ = SynchronizedBeforeSuite(func() []byte { } } + // make cache dir + if err := os.MkdirAll(ImageCacheDir, 0777); err != nil { + fmt.Printf("%q\n", err) + os.Exit(1) + } + for _, image := range CACHE_IMAGES { if err := podman.CreateArtifact(image); err != nil { fmt.Printf("%q\n", err) os.Exit(1) } } + + // If running localized tests, the cache dir is created and populated. if the + // tests are remote, this is a no-op + populateCache(podman) + host := GetHostDistributionInfo() if host.Distribution == "rhel" && strings.HasPrefix(host.Version, "7") { f, err := os.OpenFile("/proc/sys/user/max_user_namespaces", os.O_WRONLY, 0644) @@ -136,53 +148,29 @@ func (p *PodmanTestIntegration) Setup() { p.ArtifactPath = ARTIFACT_DIR } -// var _ = BeforeSuite(func() { -// cwd, _ := os.Getwd() -// INTEGRATION_ROOT = filepath.Join(cwd, "../../") -// podman := PodmanTestCreate("/tmp") -// podman.ArtifactPath = ARTIFACT_DIR -// if _, err := os.Stat(ARTIFACT_DIR); os.IsNotExist(err) { -// if err = os.Mkdir(ARTIFACT_DIR, 0777); err != nil { -// fmt.Printf("%q\n", err) -// os.Exit(1) -// } -// } -// }) -// for _, image := range CACHE_IMAGES { -// if err := podman.CreateArtifact(image); err != nil { -// fmt.Printf("%q\n", err) -// os.Exit(1) -// } -// } -// host := GetHostDistributionInfo() -// if host.Distribution == "rhel" && strings.HasPrefix(host.Version, "7") { -// f, err := os.OpenFile("/proc/sys/user/max_user_namespaces", os.O_WRONLY, 0644) -// if err != nil { -// fmt.Println("Unable to enable userspace on RHEL 7") -// os.Exit(1) -// } -// _, err = f.WriteString("15000") -// if err != nil { -// fmt.Println("Unable to enable userspace on RHEL 7") -// os.Exit(1) -// } -// f.Close() -// } -// path, err := ioutil.TempDir("", "libpodlock") -// if err != nil { -// fmt.Println(err) -// os.Exit(1) -// } -// LockTmpDir = path -// }) - -var _ = AfterSuite(func() { - sort.Sort(testResultsSortedLength{testResults}) - fmt.Println("integration timing results") - for _, result := range testResults { - fmt.Printf("%s\t\t%f\n", result.name, result.length) - } -}) +var _ = SynchronizedAfterSuite(func() {}, + func() { + sort.Sort(testResultsSortedLength{testResults}) + fmt.Println("integration timing results") + for _, result := range testResults { + fmt.Printf("%s\t\t%f\n", result.name, result.length) + } + + // previous crio-run + tempdir, err := CreateTempDirInTempDir() + if err != nil { + os.Exit(1) + } + podmanTest := PodmanTestCreate(tempdir) + + if err := os.RemoveAll(podmanTest.CrioRoot); err != nil { + fmt.Printf("%q\n", err) + } + + // for localized tests, this removes the image cache dir and for remote tests + // this is a no-op + removeCache() + }) // PodmanTestCreate creates a PodmanTestIntegration instance for the tests func PodmanTestCreateUtil(tempDir string, remote bool) *PodmanTestIntegration { @@ -244,12 +232,18 @@ func PodmanTestCreateUtil(tempDir string, remote bool) *PodmanTestIntegration { os.Setenv("DISABLE_HC_SYSTEMD", "true") CNIConfigDir := "/etc/cni/net.d" + storageFs := STORAGE_FS + if rootless.IsRootless() { + storageFs = ROOTLESS_STORAGE_FS + } p := &PodmanTestIntegration{ PodmanTest: PodmanTest{ - PodmanBinary: podmanBinary, - ArtifactPath: ARTIFACT_DIR, - TempDir: tempDir, - RemoteTest: remote, + PodmanBinary: podmanBinary, + ArtifactPath: ARTIFACT_DIR, + TempDir: tempDir, + RemoteTest: remote, + ImageCacheFS: storageFs, + ImageCacheDir: ImageCacheDir, }, ConmonBinary: conmonBinary, CrioRoot: filepath.Join(tempDir, "crio"), @@ -304,10 +298,10 @@ func (p *PodmanTestIntegration) CreateArtifact(image string) error { dest := strings.Split(image, "/") destName := fmt.Sprintf("/tmp/%s.tar", strings.Replace(strings.Join(strings.Split(dest[len(dest)-1], "/"), ""), ":", "-", -1)) if _, err := os.Stat(destName); os.IsNotExist(err) { - pull := p.Podman([]string{"pull", image}) + pull := p.PodmanNoCache([]string{"pull", image}) pull.Wait(90) - save := p.Podman([]string{"save", "-o", destName, image}) + save := p.PodmanNoCache([]string{"save", "-o", destName, image}) save.Wait(90) fmt.Printf("\n") } else { @@ -326,7 +320,7 @@ func (s *PodmanSessionIntegration) InspectImageJSON() []inspect.ImageData { } // InspectContainer returns a container's inspect data in JSON format -func (p *PodmanTestIntegration) InspectContainer(name string) []inspect.ContainerData { +func (p *PodmanTestIntegration) InspectContainer(name string) []shared.InspectContainer { cmd := []string{"inspect", name} session := p.Podman(cmd) session.WaitWithDefaultTimeout() @@ -390,7 +384,7 @@ func (p *PodmanTestIntegration) BuildImage(dockerfile, imageName string, layers dockerfilePath := filepath.Join(p.TempDir, "Dockerfile") err := ioutil.WriteFile(dockerfilePath, []byte(dockerfile), 0755) Expect(err).To(BeNil()) - session := p.Podman([]string{"build", "--layers=" + layers, "-t", imageName, "--file", dockerfilePath, p.TempDir}) + session := p.PodmanNoCache([]string{"build", "--layers=" + layers, "-t", imageName, "--file", dockerfilePath, p.TempDir}) session.Wait(120) Expect(session.ExitCode()).To(Equal(0)) } @@ -465,7 +459,7 @@ func (p *PodmanTestIntegration) PullImages(images []string) error { // PullImage pulls a single image // TODO should the timeout be configurable? func (p *PodmanTestIntegration) PullImage(image string) error { - session := p.Podman([]string{"pull", image}) + session := p.PodmanNoCache([]string{"pull", image}) session.Wait(60) Expect(session.ExitCode()).To(Equal(0)) return nil @@ -473,8 +467,8 @@ func (p *PodmanTestIntegration) PullImage(image string) error { // InspectContainerToJSON takes the session output of an inspect // container and returns json -func (s *PodmanSessionIntegration) InspectContainerToJSON() []inspect.ContainerData { - var i []inspect.ContainerData +func (s *PodmanSessionIntegration) InspectContainerToJSON() []shared.InspectContainer { + var i []shared.InspectContainer err := json.Unmarshal(s.Out.Contents(), &i) Expect(err).To(BeNil()) return i @@ -508,3 +502,9 @@ func (p *PodmanTestIntegration) RunTopContainerInPod(name, pod string) *PodmanSe podmanArgs = append(podmanArgs, "-d", ALPINE, "top") return p.Podman(podmanArgs) } + +func (p *PodmanTestIntegration) ImageExistsInMainStore(idOrName string) bool { + results := p.PodmanNoCache([]string{"image", "exists", idOrName}) + results.WaitWithDefaultTimeout() + return Expect(results.ExitCode()).To(Equal(0)) +} diff --git a/test/e2e/config.go b/test/e2e/config.go index 3fdb9e116..5e0000e09 100644 --- a/test/e2e/config.go +++ b/test/e2e/config.go @@ -7,4 +7,5 @@ var ( infra = "k8s.gcr.io/pause:3.1" BB = "docker.io/library/busybox:latest" healthcheck = "docker.io/libpod/alpine_healthcheck:latest" + ImageCacheDir = "/tmp/podman/imagecachedir" ) diff --git a/test/e2e/config_amd64.go b/test/e2e/config_amd64.go index d02de7a6e..01aa8445a 100644 --- a/test/e2e/config_amd64.go +++ b/test/e2e/config_amd64.go @@ -1,7 +1,9 @@ package integration var ( + STORAGE_FS = "vfs" STORAGE_OPTIONS = "--storage-driver vfs" + ROOTLESS_STORAGE_FS = "vfs" ROOTLESS_STORAGE_OPTIONS = "--storage-driver vfs" CACHE_IMAGES = []string{ALPINE, BB, fedoraMinimal, nginx, redis, registry, infra, labels, healthcheck} nginx = "quay.io/libpod/alpine_nginx:latest" diff --git a/test/e2e/config_ppc64le.go b/test/e2e/config_ppc64le.go index d1737fa6f..569a34efb 100644 --- a/test/e2e/config_ppc64le.go +++ b/test/e2e/config_ppc64le.go @@ -1,7 +1,9 @@ package integration var ( + STORAGE_FS = "overlay" STORAGE_OPTIONS = "--storage-driver overlay" + ROOTLESS_STORAGE_FS = "vfs" ROOTLESS_STORAGE_OPTIONS = "--storage-driver vfs" CACHE_IMAGES = []string{ALPINE, BB, fedoraMinimal, nginx, redis, infra, labels} nginx = "quay.io/libpod/alpine_nginx-ppc64le:latest" diff --git a/test/e2e/cp_test.go b/test/e2e/cp_test.go index f8df5d3d0..597d0aef7 100644 --- a/test/e2e/cp_test.go +++ b/test/e2e/cp_test.go @@ -28,7 +28,7 @@ var _ = Describe("Podman cp", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() }) AfterEach(func() { @@ -42,14 +42,15 @@ var _ = Describe("Podman cp", func() { srcPath := filepath.Join(podmanTest.RunRoot, "cp_test.txt") dstPath := filepath.Join(podmanTest.RunRoot, "cp_from_container") fromHostToContainer := []byte("copy from host to container") - err := ioutil.WriteFile(srcPath, fromHostToContainer, 0644) - Expect(err).To(BeNil()) session := podmanTest.Podman([]string{"create", ALPINE, "cat", "foo"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) name := session.OutputToString() + err := ioutil.WriteFile(srcPath, fromHostToContainer, 0644) + Expect(err).To(BeNil()) + session = podmanTest.Podman([]string{"cp", srcPath, name + ":foo"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) @@ -63,16 +64,17 @@ var _ = Describe("Podman cp", func() { srcPath := filepath.Join(podmanTest.RunRoot, "cp_test.txt") dstDir := filepath.Join(podmanTest.RunRoot, "receive") fromHostToContainer := []byte("copy from host to container directory") - err := ioutil.WriteFile(srcPath, fromHostToContainer, 0644) - Expect(err).To(BeNil()) - err = os.Mkdir(dstDir, 0755) - Expect(err).To(BeNil()) session := podmanTest.Podman([]string{"create", ALPINE, "ls", "foodir/"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) name := session.OutputToString() + err := ioutil.WriteFile(srcPath, fromHostToContainer, 0644) + Expect(err).To(BeNil()) + err = os.Mkdir(dstDir, 0755) + Expect(err).To(BeNil()) + session = podmanTest.Podman([]string{"cp", srcPath, name + ":foodir/"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) @@ -87,14 +89,15 @@ var _ = Describe("Podman cp", func() { It("podman cp dir to dir", func() { testDirPath := filepath.Join(podmanTest.RunRoot, "TestDir") - err := os.Mkdir(testDirPath, 0755) - Expect(err).To(BeNil()) session := podmanTest.Podman([]string{"create", ALPINE, "ls", "/foodir"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) name := session.OutputToString() + err := os.Mkdir(testDirPath, 0755) + Expect(err).To(BeNil()) + session = podmanTest.Podman([]string{"cp", testDirPath, name + ":/foodir"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) @@ -105,6 +108,11 @@ var _ = Describe("Podman cp", func() { }) It("podman cp stdin/stdout", func() { + session := podmanTest.Podman([]string{"create", ALPINE, "ls", "foo"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + name := session.OutputToString() + testDirPath := filepath.Join(podmanTest.RunRoot, "TestDir") err := os.Mkdir(testDirPath, 0755) Expect(err).To(BeNil()) @@ -112,11 +120,6 @@ var _ = Describe("Podman cp", func() { _, err = cmd.Output() Expect(err).To(BeNil()) - session := podmanTest.Podman([]string{"create", ALPINE, "ls", "foo"}) - session.WaitWithDefaultTimeout() - Expect(session.ExitCode()).To(Equal(0)) - name := session.OutputToString() - data, err := ioutil.ReadFile("foo.tar.gz") reader := strings.NewReader(string(data)) cmd.Stdin = reader @@ -133,6 +136,10 @@ var _ = Describe("Podman cp", func() { }) It("podman cp tar", func() { + session := podmanTest.Podman([]string{"create", "--name", "testctr", ALPINE, "ls", "-l", "foo"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + path, err := os.Getwd() Expect(err).To(BeNil()) testDirPath := filepath.Join(path, "TestDir") @@ -142,10 +149,6 @@ var _ = Describe("Podman cp", func() { _, err = cmd.Output() Expect(err).To(BeNil()) - session := podmanTest.Podman([]string{"create", "--name", "testctr", ALPINE, "ls", "-l", "foo"}) - session.WaitWithDefaultTimeout() - Expect(session.ExitCode()).To(Equal(0)) - session = podmanTest.Podman([]string{"cp", "file.tar", "testctr:/foo/"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) @@ -158,4 +161,27 @@ var _ = Describe("Podman cp", func() { os.Remove("file.tar") os.RemoveAll(testDirPath) }) + + It("podman cp symlink", func() { + session := podmanTest.Podman([]string{"run", "-d", ALPINE, "top"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + name := session.OutputToString() + + srcPath := filepath.Join(podmanTest.RunRoot, "cp_test.txt") + fromHostToContainer := []byte("copy from host to container") + err := ioutil.WriteFile(srcPath, fromHostToContainer, 0644) + Expect(err).To(BeNil()) + + session = podmanTest.Podman([]string{"exec", name, "ln", "-s", "/tmp", "/test"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + + session = podmanTest.Podman([]string{"cp", "--pause=false", srcPath, name + ":/test"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + + _, err = os.Stat("/tmp/cp_test.txt") + Expect(err).To(Not(BeNil())) + }) }) diff --git a/test/e2e/create_staticip_test.go b/test/e2e/create_staticip_test.go index 6c4ca1cb8..11301856b 100644 --- a/test/e2e/create_staticip_test.go +++ b/test/e2e/create_staticip_test.go @@ -25,7 +25,7 @@ var _ = Describe("Podman create with --ip flag", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() // Cleanup the CNI networks used by the tests os.RemoveAll("/var/lib/cni/networks/podman") }) diff --git a/test/e2e/create_test.go b/test/e2e/create_test.go index f3367337e..e2b4a7cf4 100644 --- a/test/e2e/create_test.go +++ b/test/e2e/create_test.go @@ -24,7 +24,7 @@ var _ = Describe("Podman create", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() }) AfterEach(func() { diff --git a/test/e2e/diff_test.go b/test/e2e/diff_test.go index 920b920c0..0f53d9bc8 100644 --- a/test/e2e/diff_test.go +++ b/test/e2e/diff_test.go @@ -23,7 +23,7 @@ var _ = Describe("Podman diff", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() }) AfterEach(func() { diff --git a/test/e2e/events_test.go b/test/e2e/events_test.go index 5ac5c9860..c5eedda3c 100644 --- a/test/e2e/events_test.go +++ b/test/e2e/events_test.go @@ -23,7 +23,7 @@ var _ = Describe("Podman events", func() { os.Exit(1) } podmanTest = PodmanTestCreate(tempdir) - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() }) AfterEach(func() { diff --git a/test/e2e/exec_test.go b/test/e2e/exec_test.go index 2a10e52b1..f42831ebb 100644 --- a/test/e2e/exec_test.go +++ b/test/e2e/exec_test.go @@ -24,7 +24,7 @@ var _ = Describe("Podman exec", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() }) AfterEach(func() { @@ -123,7 +123,6 @@ var _ = Describe("Podman exec", func() { }) It("podman exec with user only in container", func() { - podmanTest.RestoreArtifact(fedoraMinimal) testUser := "test123" setup := podmanTest.Podman([]string{"run", "--name", "test1", "-d", fedoraMinimal, "sleep", "60"}) setup.WaitWithDefaultTimeout() diff --git a/test/e2e/exists_test.go b/test/e2e/exists_test.go index 71c6c1820..1486427c5 100644 --- a/test/e2e/exists_test.go +++ b/test/e2e/exists_test.go @@ -48,7 +48,6 @@ var _ = Describe("Podman image|container exists", func() { Expect(session.ExitCode()).To(Equal(1)) }) It("podman container exists in local storage by name", func() { - SkipIfRemote() setup := podmanTest.RunTopContainer("foobar") setup.WaitWithDefaultTimeout() Expect(setup.ExitCode()).To(Equal(0)) @@ -58,7 +57,6 @@ var _ = Describe("Podman image|container exists", func() { Expect(session.ExitCode()).To(Equal(0)) }) It("podman container exists in local storage by container ID", func() { - SkipIfRemote() setup := podmanTest.RunTopContainer("") setup.WaitWithDefaultTimeout() Expect(setup.ExitCode()).To(Equal(0)) @@ -69,7 +67,6 @@ var _ = Describe("Podman image|container exists", func() { Expect(session.ExitCode()).To(Equal(0)) }) It("podman container exists in local storage by short container ID", func() { - SkipIfRemote() setup := podmanTest.RunTopContainer("") setup.WaitWithDefaultTimeout() Expect(setup.ExitCode()).To(Equal(0)) @@ -80,14 +77,12 @@ var _ = Describe("Podman image|container exists", func() { Expect(session.ExitCode()).To(Equal(0)) }) It("podman container does not exist in local storage", func() { - SkipIfRemote() session := podmanTest.Podman([]string{"container", "exists", "foobar"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(1)) }) It("podman pod exists in local storage by name", func() { - SkipIfRemote() setup, rc, _ := podmanTest.CreatePod("foobar") setup.WaitWithDefaultTimeout() Expect(rc).To(Equal(0)) @@ -97,7 +92,6 @@ var _ = Describe("Podman image|container exists", func() { Expect(session.ExitCode()).To(Equal(0)) }) It("podman pod exists in local storage by container ID", func() { - SkipIfRemote() setup, rc, podID := podmanTest.CreatePod("") setup.WaitWithDefaultTimeout() Expect(rc).To(Equal(0)) @@ -107,7 +101,6 @@ var _ = Describe("Podman image|container exists", func() { Expect(session.ExitCode()).To(Equal(0)) }) It("podman pod exists in local storage by short container ID", func() { - SkipIfRemote() setup, rc, podID := podmanTest.CreatePod("") setup.WaitWithDefaultTimeout() Expect(rc).To(Equal(0)) @@ -117,6 +110,7 @@ var _ = Describe("Podman image|container exists", func() { Expect(session.ExitCode()).To(Equal(0)) }) It("podman pod does not exist in local storage", func() { + // The exit code for non-existing pod is incorrect (125 vs 1) SkipIfRemote() session := podmanTest.Podman([]string{"pod", "exists", "foobar"}) session.WaitWithDefaultTimeout() diff --git a/test/e2e/export_test.go b/test/e2e/export_test.go index 71ddb518a..8406b0e73 100644 --- a/test/e2e/export_test.go +++ b/test/e2e/export_test.go @@ -23,7 +23,7 @@ var _ = Describe("Podman export", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() }) AfterEach(func() { diff --git a/test/e2e/generate_kube_test.go b/test/e2e/generate_kube_test.go index 2f0af7e5f..95d46476d 100644 --- a/test/e2e/generate_kube_test.go +++ b/test/e2e/generate_kube_test.go @@ -25,7 +25,7 @@ var _ = Describe("Podman generate kube", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() }) AfterEach(func() { diff --git a/test/e2e/generate_systemd_test.go b/test/e2e/generate_systemd_test.go index 940e894bc..5bb040206 100644 --- a/test/e2e/generate_systemd_test.go +++ b/test/e2e/generate_systemd_test.go @@ -23,7 +23,7 @@ var _ = Describe("Podman generate systemd", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() }) AfterEach(func() { diff --git a/test/e2e/healthcheck_run_test.go b/test/e2e/healthcheck_run_test.go index 60be86ebc..125002bf9 100644 --- a/test/e2e/healthcheck_run_test.go +++ b/test/e2e/healthcheck_run_test.go @@ -5,6 +5,7 @@ package integration import ( "fmt" "os" + "time" . "github.com/containers/libpod/test/utils" . "github.com/onsi/ginkgo" @@ -24,7 +25,8 @@ var _ = Describe("Podman healthcheck run", func() { os.Exit(1) } podmanTest = PodmanTestCreate(tempdir) - podmanTest.RestoreAllArtifacts() + podmanTest.Setup() + podmanTest.SeedImages() }) AfterEach(func() { @@ -42,14 +44,24 @@ var _ = Describe("Podman healthcheck run", func() { }) It("podman healthcheck on valid container", func() { - podmanTest.RestoreArtifact(healthcheck) + Skip("Extremely consistent flake - reenable on debugging") session := podmanTest.Podman([]string{"run", "-dt", "--name", "hc", healthcheck}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) - hc := podmanTest.Podman([]string{"healthcheck", "run", "hc"}) - hc.WaitWithDefaultTimeout() - Expect(hc.ExitCode()).To(Equal(0)) + exitCode := 999 + + // Buy a little time to get container running + for i := 0; i < 5; i++ { + hc := podmanTest.Podman([]string{"healthcheck", "run", "hc"}) + hc.WaitWithDefaultTimeout() + exitCode = hc.ExitCode() + if exitCode == 0 || i == 4 { + break + } + time.Sleep(1 * time.Second) + } + Expect(exitCode).To(Equal(0)) }) It("podman healthcheck that should fail", func() { @@ -63,7 +75,6 @@ var _ = Describe("Podman healthcheck run", func() { }) It("podman healthcheck on stopped container", func() { - podmanTest.RestoreArtifact(healthcheck) session := podmanTest.Podman([]string{"run", "-dt", "--name", "hc", healthcheck, "ls"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) diff --git a/test/e2e/history_test.go b/test/e2e/history_test.go index 9e519dd9c..231e8b856 100644 --- a/test/e2e/history_test.go +++ b/test/e2e/history_test.go @@ -22,7 +22,7 @@ var _ = Describe("Podman history", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() }) AfterEach(func() { diff --git a/test/e2e/images_test.go b/test/e2e/images_test.go index 23455163b..07d61e885 100644 --- a/test/e2e/images_test.go +++ b/test/e2e/images_test.go @@ -25,7 +25,7 @@ var _ = Describe("Podman images", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() }) AfterEach(func() { @@ -44,14 +44,15 @@ var _ = Describe("Podman images", func() { }) It("podman images with no images prints header", func() { - rmi := podmanTest.Podman([]string{"rmi", "-a"}) + rmi := podmanTest.PodmanNoCache([]string{"rmi", "-a"}) rmi.WaitWithDefaultTimeout() Expect(rmi.ExitCode()).To(Equal(0)) - session := podmanTest.Podman([]string{"images"}) + session := podmanTest.PodmanNoCache([]string{"images"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) Expect(len(session.OutputToStringArray())).To(Equal(1)) + Expect(session.LineInOutputContains("REPOSITORY")).To(BeTrue()) }) It("podman image List", func() { @@ -65,15 +66,16 @@ var _ = Describe("Podman images", func() { It("podman images with multiple tags", func() { // tag "docker.io/library/alpine:latest" to "foo:{a,b,c}" - session := podmanTest.Podman([]string{"tag", ALPINE, "foo:a", "foo:b", "foo:c"}) + podmanTest.RestoreAllArtifacts() + session := podmanTest.PodmanNoCache([]string{"tag", ALPINE, "foo:a", "foo:b", "foo:c"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) // tag "foo:c" to "bar:{a,b}" - session = podmanTest.Podman([]string{"tag", "foo:c", "bar:a", "bar:b"}) + session = podmanTest.PodmanNoCache([]string{"tag", "foo:c", "bar:a", "bar:b"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) // check all previous and the newly tagged images - session = podmanTest.Podman([]string{"images"}) + session = podmanTest.PodmanNoCache([]string{"images"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) session.LineInOutputContainsTag("docker.io/library/alpine", "latest") @@ -83,7 +85,7 @@ var _ = Describe("Podman images", func() { session.LineInOutputContainsTag("foo", "c") session.LineInOutputContainsTag("bar", "a") session.LineInOutputContainsTag("bar", "b") - session = podmanTest.Podman([]string{"images", "-qn"}) + session = podmanTest.PodmanNoCache([]string{"images", "-qn"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) Expect(len(session.OutputToStringArray())).To(BeNumerically("==", 2)) @@ -119,19 +121,20 @@ var _ = Describe("Podman images", func() { }) It("podman images filter by image name", func() { - session := podmanTest.Podman([]string{"images", "-q", ALPINE}) + podmanTest.RestoreAllArtifacts() + session := podmanTest.PodmanNoCache([]string{"images", "-q", ALPINE}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) Expect(len(session.OutputToStringArray())).To(Equal(1)) - session = podmanTest.Podman([]string{"tag", ALPINE, "foo:a"}) + session = podmanTest.PodmanNoCache([]string{"tag", ALPINE, "foo:a"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) - session = podmanTest.Podman([]string{"tag", BB, "foo:b"}) + session = podmanTest.PodmanNoCache([]string{"tag", BB, "foo:b"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) - session = podmanTest.Podman([]string{"images", "-q", "foo"}) + session = podmanTest.PodmanNoCache([]string{"images", "-q", "foo"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) Expect(len(session.OutputToStringArray())).To(Equal(2)) @@ -141,24 +144,25 @@ var _ = Describe("Podman images", func() { if podmanTest.RemoteTest { Skip("Does not work on remote client") } - result := podmanTest.Podman([]string{"images", "-q", "-f", "reference=docker.io*"}) + podmanTest.RestoreAllArtifacts() + result := podmanTest.PodmanNoCache([]string{"images", "-q", "-f", "reference=docker.io*"}) result.WaitWithDefaultTimeout() Expect(result.ExitCode()).To(Equal(0)) Expect(len(result.OutputToStringArray())).To(Equal(2)) - retapline := podmanTest.Podman([]string{"images", "-f", "reference=a*pine"}) + retapline := podmanTest.PodmanNoCache([]string{"images", "-f", "reference=a*pine"}) retapline.WaitWithDefaultTimeout() Expect(retapline.ExitCode()).To(Equal(0)) Expect(len(retapline.OutputToStringArray())).To(Equal(2)) Expect(retapline.LineInOutputContains("alpine")) - retapline = podmanTest.Podman([]string{"images", "-f", "reference=alpine"}) + retapline = podmanTest.PodmanNoCache([]string{"images", "-f", "reference=alpine"}) retapline.WaitWithDefaultTimeout() Expect(retapline.ExitCode()).To(Equal(0)) Expect(len(retapline.OutputToStringArray())).To(Equal(2)) Expect(retapline.LineInOutputContains("alpine")) - retnone := podmanTest.Podman([]string{"images", "-q", "-f", "reference=bogus"}) + retnone := podmanTest.PodmanNoCache([]string{"images", "-q", "-f", "reference=bogus"}) retnone.WaitWithDefaultTimeout() Expect(retnone.ExitCode()).To(Equal(0)) Expect(len(retnone.OutputToStringArray())).To(Equal(0)) @@ -182,14 +186,15 @@ RUN apk update && apk add man if podmanTest.RemoteTest { Skip("Does not work on remote client") } - rmi := podmanTest.Podman([]string{"rmi", "busybox"}) + podmanTest.RestoreAllArtifacts() + rmi := podmanTest.PodmanNoCache([]string{"rmi", "busybox"}) rmi.WaitWithDefaultTimeout() Expect(rmi.ExitCode()).To(Equal(0)) dockerfile := `FROM docker.io/library/alpine:latest ` podmanTest.BuildImage(dockerfile, "foobar.com/before:latest", "false") - result := podmanTest.Podman([]string{"images", "-q", "-f", "after=docker.io/library/alpine:latest"}) + result := podmanTest.PodmanNoCache([]string{"images", "-q", "-f", "after=docker.io/library/alpine:latest"}) result.WaitWithDefaultTimeout() Expect(result.ExitCode()).To(Equal(0)) Expect(len(result.OutputToStringArray())).To(Equal(0)) @@ -199,14 +204,15 @@ RUN apk update && apk add man if podmanTest.RemoteTest { Skip("Does not work on remote client") } - rmi := podmanTest.Podman([]string{"image", "rm", "busybox"}) + podmanTest.RestoreAllArtifacts() + rmi := podmanTest.PodmanNoCache([]string{"image", "rm", "busybox"}) rmi.WaitWithDefaultTimeout() Expect(rmi.ExitCode()).To(Equal(0)) dockerfile := `FROM docker.io/library/alpine:latest ` podmanTest.BuildImage(dockerfile, "foobar.com/before:latest", "false") - result := podmanTest.Podman([]string{"image", "list", "-q", "-f", "after=docker.io/library/alpine:latest"}) + result := podmanTest.PodmanNoCache([]string{"image", "list", "-q", "-f", "after=docker.io/library/alpine:latest"}) result.WaitWithDefaultTimeout() Expect(result.ExitCode()).To(Equal(0)) Expect(len(result.OutputToStringArray())).To(Equal(0)) @@ -282,18 +288,19 @@ RUN apk update && apk add man if podmanTest.RemoteTest { Skip("Does not work on remote client") } + podmanTest.RestoreAllArtifacts() dockerfile := `FROM docker.io/library/alpine:latest RUN mkdir hello RUN touch test.txt ENV foo=bar ` podmanTest.BuildImage(dockerfile, "test", "true") - session := podmanTest.Podman([]string{"images"}) + session := podmanTest.PodmanNoCache([]string{"images"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) Expect(len(session.OutputToStringArray())).To(Equal(4)) - session2 := podmanTest.Podman([]string{"images", "--all"}) + session2 := podmanTest.PodmanNoCache([]string{"images", "--all"}) session2.WaitWithDefaultTimeout() Expect(session2.ExitCode()).To(Equal(0)) Expect(len(session2.OutputToStringArray())).To(Equal(6)) diff --git a/test/e2e/import_test.go b/test/e2e/import_test.go index e819d819c..84a91a783 100644 --- a/test/e2e/import_test.go +++ b/test/e2e/import_test.go @@ -25,7 +25,7 @@ var _ = Describe("Podman import", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() }) AfterEach(func() { @@ -62,14 +62,11 @@ var _ = Describe("Podman import", func() { export.WaitWithDefaultTimeout() Expect(export.ExitCode()).To(Equal(0)) - importImage := podmanTest.Podman([]string{"import", outfile}) + importImage := podmanTest.PodmanNoCache([]string{"import", outfile}) importImage.WaitWithDefaultTimeout() Expect(importImage.ExitCode()).To(Equal(0)) - results := podmanTest.Podman([]string{"images", "-q"}) - results.WaitWithDefaultTimeout() - Expect(results.ExitCode()).To(Equal(0)) - Expect(len(results.OutputToStringArray())).To(Equal(3)) + Expect(podmanTest.ImageExistsInMainStore(importImage.OutputToString())).To(BeTrue()) }) It("podman import with message flag", func() { diff --git a/test/e2e/init_test.go b/test/e2e/init_test.go index 5865930a5..919fe4abf 100644 --- a/test/e2e/init_test.go +++ b/test/e2e/init_test.go @@ -22,7 +22,7 @@ var _ = Describe("Podman init", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() }) AfterEach(func() { diff --git a/test/e2e/inspect_test.go b/test/e2e/inspect_test.go index 34328828f..ccd8602c4 100644 --- a/test/e2e/inspect_test.go +++ b/test/e2e/inspect_test.go @@ -23,7 +23,7 @@ var _ = Describe("Podman inspect", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() }) AfterEach(func() { diff --git a/test/e2e/kill_test.go b/test/e2e/kill_test.go index 3286180a4..017fe4a3f 100644 --- a/test/e2e/kill_test.go +++ b/test/e2e/kill_test.go @@ -22,7 +22,7 @@ var _ = Describe("Podman kill", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() }) AfterEach(func() { diff --git a/test/e2e/libpod_suite_remoteclient_test.go b/test/e2e/libpod_suite_remoteclient_test.go index b7fd8537d..c8210f7d1 100644 --- a/test/e2e/libpod_suite_remoteclient_test.go +++ b/test/e2e/libpod_suite_remoteclient_test.go @@ -30,7 +30,13 @@ func SkipIfRootless() { // Podman is the exec call to podman on the filesystem func (p *PodmanTestIntegration) Podman(args []string) *PodmanSessionIntegration { - podmanSession := p.PodmanBase(args) + podmanSession := p.PodmanBase(args, false) + return &PodmanSessionIntegration{podmanSession} +} + +// PodmanNoCache calls podman with out adding the imagecache +func (p *PodmanTestIntegration) PodmanNoCache(args []string) *PodmanSessionIntegration { + podmanSession := p.PodmanBase(args, true) return &PodmanSessionIntegration{podmanSession} } @@ -145,6 +151,21 @@ func getVarlinkOptions(p *PodmanTestIntegration, args []string) []string { return podmanOptions } +func (p *PodmanTestIntegration) RestoreArtifactToCache(image string) error { + fmt.Printf("Restoring %s...\n", image) + dest := strings.Split(image, "/") + destName := fmt.Sprintf("/tmp/%s.tar", strings.Replace(strings.Join(strings.Split(dest[len(dest)-1], "/"), ""), ":", "-", -1)) + p.CrioRoot = p.ImageCacheDir + restore := p.PodmanNoCache([]string{"load", "-q", "-i", destName}) + restore.WaitWithDefaultTimeout() + return nil +} + +// SeedImages restores all the artifacts into the main store for remote tests +func (p *PodmanTestIntegration) SeedImages() error { + return p.RestoreAllArtifacts() +} + // RestoreArtifact puts the cached image into our test store func (p *PodmanTestIntegration) RestoreArtifact(image string) error { fmt.Printf("Restoring %s...\n", image) @@ -169,3 +190,6 @@ func (p *PodmanTestIntegration) DelayForVarlink() { time.Sleep(1 * time.Second) } } + +func populateCache(podman *PodmanTestIntegration) {} +func removeCache() {} diff --git a/test/e2e/libpod_suite_test.go b/test/e2e/libpod_suite_test.go index 0a85c625d..8d993ee72 100644 --- a/test/e2e/libpod_suite_test.go +++ b/test/e2e/libpod_suite_test.go @@ -23,13 +23,19 @@ func SkipIfRootless() { // Podman is the exec call to podman on the filesystem func (p *PodmanTestIntegration) Podman(args []string) *PodmanSessionIntegration { - podmanSession := p.PodmanBase(args) + podmanSession := p.PodmanBase(args, false) + return &PodmanSessionIntegration{podmanSession} +} + +// PodmanNoCache calls the podman command with no configured imagecache +func (p *PodmanTestIntegration) PodmanNoCache(args []string) *PodmanSessionIntegration { + podmanSession := p.PodmanBase(args, true) return &PodmanSessionIntegration{podmanSession} } // PodmanAsUser is the exec call to podman on the filesystem with the specified uid/gid and environment func (p *PodmanTestIntegration) PodmanAsUser(args []string, uid, gid uint32, cwd string, env []string) *PodmanSessionIntegration { - podmanSession := p.PodmanAsUserBase(args, uid, gid, cwd, env) + podmanSession := p.PodmanAsUserBase(args, uid, gid, cwd, env, false) return &PodmanSessionIntegration{podmanSession} } @@ -75,9 +81,40 @@ func (p *PodmanTestIntegration) RestoreArtifact(image string) error { fmt.Printf("Restoring %s...\n", image) dest := strings.Split(image, "/") destName := fmt.Sprintf("/tmp/%s.tar", strings.Replace(strings.Join(strings.Split(dest[len(dest)-1], "/"), ""), ":", "-", -1)) - restore := p.Podman([]string{"load", "-q", "-i", destName}) + restore := p.PodmanNoCache([]string{"load", "-q", "-i", destName}) restore.Wait(90) return nil } + +// RestoreArtifactToCache populates the imagecache from tarballs that were cached earlier +func (p *PodmanTestIntegration) RestoreArtifactToCache(image string) error { + fmt.Printf("Restoring %s...\n", image) + dest := strings.Split(image, "/") + destName := fmt.Sprintf("/tmp/%s.tar", strings.Replace(strings.Join(strings.Split(dest[len(dest)-1], "/"), ""), ":", "-", -1)) + + p.CrioRoot = p.ImageCacheDir + restore := p.PodmanNoCache([]string{"load", "-q", "-i", destName}) + restore.WaitWithDefaultTimeout() + return nil +} + func (p *PodmanTestIntegration) StopVarlink() {} func (p *PodmanTestIntegration) DelayForVarlink() {} + +func populateCache(podman *PodmanTestIntegration) { + for _, image := range CACHE_IMAGES { + podman.RestoreArtifactToCache(image) + } +} + +func removeCache() { + // Remove cache dirs + if err := os.RemoveAll(ImageCacheDir); err != nil { + fmt.Printf("%q\n", err) + } +} + +// SeedImages is a no-op for localized testing +func (p *PodmanTestIntegration) SeedImages() error { + return nil +} diff --git a/test/e2e/load_test.go b/test/e2e/load_test.go index 0e193640e..9209e1770 100644 --- a/test/e2e/load_test.go +++ b/test/e2e/load_test.go @@ -3,6 +3,7 @@ package integration import ( + "fmt" "os" "path/filepath" @@ -38,7 +39,11 @@ var _ = Describe("Podman load", func() { It("podman load input flag", func() { outfile := filepath.Join(podmanTest.TempDir, "alpine.tar") - save := podmanTest.Podman([]string{"save", "-o", outfile, ALPINE}) + images := podmanTest.PodmanNoCache([]string{"images"}) + images.WaitWithDefaultTimeout() + fmt.Println(images.OutputToStringArray()) + + save := podmanTest.PodmanNoCache([]string{"save", "-o", outfile, ALPINE}) save.WaitWithDefaultTimeout() Expect(save.ExitCode()).To(Equal(0)) @@ -46,7 +51,7 @@ var _ = Describe("Podman load", func() { rmi.WaitWithDefaultTimeout() Expect(rmi.ExitCode()).To(Equal(0)) - result := podmanTest.Podman([]string{"load", "-i", outfile}) + result := podmanTest.PodmanNoCache([]string{"load", "-i", outfile}) result.WaitWithDefaultTimeout() Expect(result.ExitCode()).To(Equal(0)) }) @@ -54,7 +59,7 @@ var _ = Describe("Podman load", func() { It("podman load compressed tar file", func() { outfile := filepath.Join(podmanTest.TempDir, "alpine.tar") - save := podmanTest.Podman([]string{"save", "-o", outfile, ALPINE}) + save := podmanTest.PodmanNoCache([]string{"save", "-o", outfile, ALPINE}) save.WaitWithDefaultTimeout() Expect(save.ExitCode()).To(Equal(0)) @@ -62,11 +67,11 @@ var _ = Describe("Podman load", func() { Expect(compress.ExitCode()).To(Equal(0)) outfile = outfile + ".gz" - rmi := podmanTest.Podman([]string{"rmi", ALPINE}) + rmi := podmanTest.PodmanNoCache([]string{"rmi", ALPINE}) rmi.WaitWithDefaultTimeout() Expect(rmi.ExitCode()).To(Equal(0)) - result := podmanTest.Podman([]string{"load", "-i", outfile}) + result := podmanTest.PodmanNoCache([]string{"load", "-i", outfile}) result.WaitWithDefaultTimeout() Expect(result.ExitCode()).To(Equal(0)) }) @@ -74,15 +79,15 @@ var _ = Describe("Podman load", func() { It("podman load oci-archive image", func() { outfile := filepath.Join(podmanTest.TempDir, "alpine.tar") - save := podmanTest.Podman([]string{"save", "-o", outfile, "--format", "oci-archive", ALPINE}) + save := podmanTest.PodmanNoCache([]string{"save", "-o", outfile, "--format", "oci-archive", ALPINE}) save.WaitWithDefaultTimeout() Expect(save.ExitCode()).To(Equal(0)) - rmi := podmanTest.Podman([]string{"rmi", ALPINE}) + rmi := podmanTest.PodmanNoCache([]string{"rmi", ALPINE}) rmi.WaitWithDefaultTimeout() Expect(rmi.ExitCode()).To(Equal(0)) - result := podmanTest.Podman([]string{"load", "-i", outfile}) + result := podmanTest.PodmanNoCache([]string{"load", "-i", outfile}) result.WaitWithDefaultTimeout() Expect(result.ExitCode()).To(Equal(0)) }) @@ -90,15 +95,15 @@ var _ = Describe("Podman load", func() { It("podman load oci-archive with signature", func() { outfile := filepath.Join(podmanTest.TempDir, "alpine.tar") - save := podmanTest.Podman([]string{"save", "-o", outfile, "--format", "oci-archive", ALPINE}) + save := podmanTest.PodmanNoCache([]string{"save", "-o", outfile, "--format", "oci-archive", ALPINE}) save.WaitWithDefaultTimeout() Expect(save.ExitCode()).To(Equal(0)) - rmi := podmanTest.Podman([]string{"rmi", ALPINE}) + rmi := podmanTest.PodmanNoCache([]string{"rmi", ALPINE}) rmi.WaitWithDefaultTimeout() Expect(rmi.ExitCode()).To(Equal(0)) - result := podmanTest.Podman([]string{"load", "--signature-policy", "/etc/containers/policy.json", "-i", outfile}) + result := podmanTest.PodmanNoCache([]string{"load", "--signature-policy", "/etc/containers/policy.json", "-i", outfile}) result.WaitWithDefaultTimeout() Expect(result.ExitCode()).To(Equal(0)) }) @@ -106,15 +111,15 @@ var _ = Describe("Podman load", func() { It("podman load with quiet flag", func() { outfile := filepath.Join(podmanTest.TempDir, "alpine.tar") - save := podmanTest.Podman([]string{"save", "-o", outfile, ALPINE}) + save := podmanTest.PodmanNoCache([]string{"save", "-o", outfile, ALPINE}) save.WaitWithDefaultTimeout() Expect(save.ExitCode()).To(Equal(0)) - rmi := podmanTest.Podman([]string{"rmi", ALPINE}) + rmi := podmanTest.PodmanNoCache([]string{"rmi", ALPINE}) rmi.WaitWithDefaultTimeout() Expect(rmi.ExitCode()).To(Equal(0)) - result := podmanTest.Podman([]string{"load", "-q", "-i", outfile}) + result := podmanTest.PodmanNoCache([]string{"load", "-q", "-i", outfile}) result.WaitWithDefaultTimeout() Expect(result.ExitCode()).To(Equal(0)) }) @@ -122,7 +127,7 @@ var _ = Describe("Podman load", func() { It("podman load directory", func() { outdir := filepath.Join(podmanTest.TempDir, "alpine") - save := podmanTest.Podman([]string{"save", "--format", "oci-dir", "-o", outdir, ALPINE}) + save := podmanTest.PodmanNoCache([]string{"save", "--format", "oci-dir", "-o", outdir, ALPINE}) save.WaitWithDefaultTimeout() Expect(save.ExitCode()).To(Equal(0)) @@ -136,7 +141,7 @@ var _ = Describe("Podman load", func() { }) It("podman load bogus file", func() { - save := podmanTest.Podman([]string{"load", "-i", "foobar.tar"}) + save := podmanTest.PodmanNoCache([]string{"load", "-i", "foobar.tar"}) save.WaitWithDefaultTimeout() Expect(save.ExitCode()).ToNot(Equal(0)) }) @@ -148,75 +153,75 @@ var _ = Describe("Podman load", func() { outfile := filepath.Join(podmanTest.TempDir, "alpine.tar") alpVersion := "docker.io/library/alpine:3.2" - pull := podmanTest.Podman([]string{"pull", alpVersion}) + pull := podmanTest.PodmanNoCache([]string{"pull", alpVersion}) pull.WaitWithDefaultTimeout() Expect(pull.ExitCode()).To(Equal(0)) - save := podmanTest.Podman([]string{"save", "-o", outfile, ALPINE, alpVersion}) + save := podmanTest.PodmanNoCache([]string{"save", "-o", outfile, ALPINE, alpVersion}) save.WaitWithDefaultTimeout() Expect(save.ExitCode()).To(Equal(0)) - rmi := podmanTest.Podman([]string{"rmi", ALPINE, alpVersion}) + rmi := podmanTest.PodmanNoCache([]string{"rmi", ALPINE, alpVersion}) rmi.WaitWithDefaultTimeout() Expect(rmi.ExitCode()).To(Equal(0)) - result := podmanTest.Podman([]string{"load", "-i", outfile}) + result := podmanTest.PodmanNoCache([]string{"load", "-i", outfile}) result.WaitWithDefaultTimeout() Expect(result.ExitCode()).To(Equal(0)) - inspect := podmanTest.Podman([]string{"inspect", ALPINE}) + inspect := podmanTest.PodmanNoCache([]string{"inspect", ALPINE}) inspect.WaitWithDefaultTimeout() Expect(result.ExitCode()).To(Equal(0)) - inspect = podmanTest.Podman([]string{"inspect", alpVersion}) + inspect = podmanTest.PodmanNoCache([]string{"inspect", alpVersion}) inspect.WaitWithDefaultTimeout() Expect(result.ExitCode()).To(Equal(0)) }) It("podman load localhost registry from scratch", func() { outfile := filepath.Join(podmanTest.TempDir, "load_test.tar.gz") - setup := podmanTest.Podman([]string{"tag", ALPINE, "hello:world"}) + setup := podmanTest.PodmanNoCache([]string{"tag", ALPINE, "hello:world"}) setup.WaitWithDefaultTimeout() Expect(setup.ExitCode()).To(Equal(0)) - setup = podmanTest.Podman([]string{"save", "-o", outfile, "--format", "oci-archive", "hello:world"}) + setup = podmanTest.PodmanNoCache([]string{"save", "-o", outfile, "--format", "oci-archive", "hello:world"}) setup.WaitWithDefaultTimeout() Expect(setup.ExitCode()).To(Equal(0)) - setup = podmanTest.Podman([]string{"rmi", "hello:world"}) + setup = podmanTest.PodmanNoCache([]string{"rmi", "hello:world"}) setup.WaitWithDefaultTimeout() Expect(setup.ExitCode()).To(Equal(0)) - load := podmanTest.Podman([]string{"load", "-i", outfile}) + load := podmanTest.PodmanNoCache([]string{"load", "-i", outfile}) load.WaitWithDefaultTimeout() Expect(load.ExitCode()).To(Equal(0)) - result := podmanTest.Podman([]string{"images", "hello:world"}) + result := podmanTest.PodmanNoCache([]string{"images", "hello:world"}) result.WaitWithDefaultTimeout() Expect(result.LineInOutputContains("docker")).To(Not(BeTrue())) Expect(result.LineInOutputContains("localhost")).To(BeTrue()) }) It("podman load localhost registry from scratch and :latest", func() { + podmanTest.RestoreArtifact(fedoraMinimal) outfile := filepath.Join(podmanTest.TempDir, "load_test.tar.gz") - podmanTest.RestoreArtifact("fedora-minimal:latest") - setup := podmanTest.Podman([]string{"tag", "fedora-minimal", "hello"}) + setup := podmanTest.PodmanNoCache([]string{"tag", "fedora-minimal", "hello"}) setup.WaitWithDefaultTimeout() Expect(setup.ExitCode()).To(Equal(0)) - setup = podmanTest.Podman([]string{"save", "-o", outfile, "--format", "oci-archive", "hello"}) + setup = podmanTest.PodmanNoCache([]string{"save", "-o", outfile, "--format", "oci-archive", "hello"}) setup.WaitWithDefaultTimeout() Expect(setup.ExitCode()).To(Equal(0)) - setup = podmanTest.Podman([]string{"rmi", "hello"}) + setup = podmanTest.PodmanNoCache([]string{"rmi", "hello"}) setup.WaitWithDefaultTimeout() Expect(setup.ExitCode()).To(Equal(0)) - load := podmanTest.Podman([]string{"load", "-i", outfile}) + load := podmanTest.PodmanNoCache([]string{"load", "-i", outfile}) load.WaitWithDefaultTimeout() Expect(load.ExitCode()).To(Equal(0)) - result := podmanTest.Podman([]string{"images", "hello:latest"}) + result := podmanTest.PodmanNoCache([]string{"images", "hello:latest"}) result.WaitWithDefaultTimeout() Expect(result.LineInOutputContains("docker")).To(Not(BeTrue())) Expect(result.LineInOutputContains("localhost")).To(BeTrue()) @@ -225,23 +230,23 @@ var _ = Describe("Podman load", func() { It("podman load localhost registry from dir", func() { outfile := filepath.Join(podmanTest.TempDir, "load") - setup := podmanTest.Podman([]string{"tag", BB, "hello:world"}) + setup := podmanTest.PodmanNoCache([]string{"tag", BB, "hello:world"}) setup.WaitWithDefaultTimeout() Expect(setup.ExitCode()).To(Equal(0)) - setup = podmanTest.Podman([]string{"save", "-o", outfile, "--format", "oci-dir", "hello:world"}) + setup = podmanTest.PodmanNoCache([]string{"save", "-o", outfile, "--format", "oci-dir", "hello:world"}) setup.WaitWithDefaultTimeout() Expect(setup.ExitCode()).To(Equal(0)) - setup = podmanTest.Podman([]string{"rmi", "hello:world"}) + setup = podmanTest.PodmanNoCache([]string{"rmi", "hello:world"}) setup.WaitWithDefaultTimeout() Expect(setup.ExitCode()).To(Equal(0)) - load := podmanTest.Podman([]string{"load", "-i", outfile}) + load := podmanTest.PodmanNoCache([]string{"load", "-i", outfile}) load.WaitWithDefaultTimeout() Expect(load.ExitCode()).To(Equal(0)) - result := podmanTest.Podman([]string{"images", "load:latest"}) + result := podmanTest.PodmanNoCache([]string{"images", "load:latest"}) result.WaitWithDefaultTimeout() Expect(result.LineInOutputContains("docker")).To(Not(BeTrue())) Expect(result.LineInOutputContains("localhost")).To(BeTrue()) @@ -250,17 +255,17 @@ var _ = Describe("Podman load", func() { It("podman load xz compressed image", func() { outfile := filepath.Join(podmanTest.TempDir, "bb.tar") - save := podmanTest.Podman([]string{"save", "-o", outfile, BB}) + save := podmanTest.PodmanNoCache([]string{"save", "-o", outfile, BB}) save.WaitWithDefaultTimeout() Expect(save.ExitCode()).To(Equal(0)) session := SystemExec("xz", []string{outfile}) Expect(session.ExitCode()).To(Equal(0)) - rmi := podmanTest.Podman([]string{"rmi", BB}) + rmi := podmanTest.PodmanNoCache([]string{"rmi", BB}) rmi.WaitWithDefaultTimeout() Expect(rmi.ExitCode()).To(Equal(0)) - result := podmanTest.Podman([]string{"load", "-i", outfile + ".xz"}) + result := podmanTest.PodmanNoCache([]string{"load", "-i", outfile + ".xz"}) result.WaitWithDefaultTimeout() Expect(result.ExitCode()).To(Equal(0)) }) diff --git a/test/e2e/logs_test.go b/test/e2e/logs_test.go index 2c82182cf..cc50c4d95 100644 --- a/test/e2e/logs_test.go +++ b/test/e2e/logs_test.go @@ -23,7 +23,7 @@ var _ = Describe("Podman logs", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() }) AfterEach(func() { @@ -140,4 +140,82 @@ var _ = Describe("Podman logs", func() { results.WaitWithDefaultTimeout() Expect(results.ExitCode()).To(BeZero()) }) + + It("podman journald logs for container", func() { + Skip("need to verify images have correct packages for journald") + logc := podmanTest.Podman([]string{"run", "--log-driver", "journald", "-dt", ALPINE, "sh", "-c", "echo podman; echo podman; echo podman"}) + logc.WaitWithDefaultTimeout() + Expect(logc.ExitCode()).To(Equal(0)) + cid := logc.OutputToString() + + results := podmanTest.Podman([]string{"logs", cid}) + results.WaitWithDefaultTimeout() + Expect(results.ExitCode()).To(Equal(0)) + Expect(len(results.OutputToStringArray())).To(Equal(3)) + }) + + It("podman journald logs tail two lines", func() { + Skip("need to verify images have correct packages for journald") + logc := podmanTest.Podman([]string{"run", "--log-driver", "journald", "-dt", ALPINE, "sh", "-c", "echo podman; echo podman; echo podman"}) + logc.WaitWithDefaultTimeout() + Expect(logc.ExitCode()).To(Equal(0)) + cid := logc.OutputToString() + + results := podmanTest.Podman([]string{"logs", "--tail", "2", cid}) + results.WaitWithDefaultTimeout() + Expect(results.ExitCode()).To(Equal(0)) + Expect(len(results.OutputToStringArray())).To(Equal(2)) + }) + + It("podman journald logs tail 99 lines", func() { + Skip("need to verify images have correct packages for journald") + logc := podmanTest.Podman([]string{"run", "--log-driver", "journald", "-dt", ALPINE, "sh", "-c", "echo podman; echo podman; echo podman"}) + logc.WaitWithDefaultTimeout() + Expect(logc.ExitCode()).To(Equal(0)) + cid := logc.OutputToString() + + results := podmanTest.Podman([]string{"logs", "--tail", "99", cid}) + results.WaitWithDefaultTimeout() + Expect(results.ExitCode()).To(Equal(0)) + Expect(len(results.OutputToStringArray())).To(Equal(3)) + }) + + It("podman journald logs tail 2 lines with timestamps", func() { + Skip("need to verify images have correct packages for journald") + logc := podmanTest.Podman([]string{"run", "--log-driver", "journald", "-dt", ALPINE, "sh", "-c", "echo podman; echo podman; echo podman"}) + logc.WaitWithDefaultTimeout() + Expect(logc.ExitCode()).To(Equal(0)) + cid := logc.OutputToString() + + results := podmanTest.Podman([]string{"logs", "--tail", "2", "-t", cid}) + results.WaitWithDefaultTimeout() + Expect(results.ExitCode()).To(Equal(0)) + Expect(len(results.OutputToStringArray())).To(Equal(2)) + }) + + It("podman journald logs latest with since time", func() { + Skip("need to verify images have correct packages for journald") + logc := podmanTest.Podman([]string{"run", "--log-driver", "journald", "-dt", ALPINE, "sh", "-c", "echo podman; echo podman; echo podman"}) + logc.WaitWithDefaultTimeout() + Expect(logc.ExitCode()).To(Equal(0)) + cid := logc.OutputToString() + + results := podmanTest.Podman([]string{"logs", "--since", "2017-08-07T10:10:09.056611202-04:00", cid}) + results.WaitWithDefaultTimeout() + Expect(results.ExitCode()).To(Equal(0)) + Expect(len(results.OutputToStringArray())).To(Equal(3)) + }) + + It("podman journald logs latest with since duration", func() { + Skip("need to verify images have correct packages for journald") + logc := podmanTest.Podman([]string{"run", "--log-driver", "journald", "-dt", ALPINE, "sh", "-c", "echo podman; echo podman; echo podman"}) + logc.WaitWithDefaultTimeout() + Expect(logc.ExitCode()).To(Equal(0)) + cid := logc.OutputToString() + + results := podmanTest.Podman([]string{"logs", "--since", "10m", cid}) + results.WaitWithDefaultTimeout() + Expect(results.ExitCode()).To(Equal(0)) + Expect(len(results.OutputToStringArray())).To(Equal(3)) + }) }) diff --git a/test/e2e/mount_test.go b/test/e2e/mount_test.go index b361e0057..61abdf6fc 100644 --- a/test/e2e/mount_test.go +++ b/test/e2e/mount_test.go @@ -24,7 +24,7 @@ var _ = Describe("Podman mount", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() }) AfterEach(func() { diff --git a/test/e2e/namespace_test.go b/test/e2e/namespace_test.go index 28d050be3..88b48cb06 100644 --- a/test/e2e/namespace_test.go +++ b/test/e2e/namespace_test.go @@ -24,7 +24,7 @@ var _ = Describe("Podman namespaces", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() }) AfterEach(func() { diff --git a/test/e2e/pause_test.go b/test/e2e/pause_test.go index c47189a0e..01fb6d91b 100644 --- a/test/e2e/pause_test.go +++ b/test/e2e/pause_test.go @@ -27,7 +27,7 @@ var _ = Describe("Podman pause", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() }) AfterEach(func() { @@ -249,7 +249,6 @@ var _ = Describe("Podman pause", func() { }) It("Pause a bunch of running containers", func() { - podmanTest.RestoreArtifact(nginx) for i := 0; i < 3; i++ { name := fmt.Sprintf("test%d", i) run := podmanTest.Podman([]string{"run", "-dt", "--name", name, nginx}) @@ -277,7 +276,6 @@ var _ = Describe("Podman pause", func() { }) It("Unpause a bunch of running containers", func() { - podmanTest.RestoreArtifact(nginx) for i := 0; i < 3; i++ { name := fmt.Sprintf("test%d", i) run := podmanTest.Podman([]string{"run", "-dt", "--name", name, nginx}) diff --git a/test/e2e/pod_create_test.go b/test/e2e/pod_create_test.go index 84966f77b..2efa36141 100644 --- a/test/e2e/pod_create_test.go +++ b/test/e2e/pod_create_test.go @@ -22,7 +22,7 @@ var _ = Describe("Podman pod create", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() }) AfterEach(func() { diff --git a/test/e2e/pod_infra_container_test.go b/test/e2e/pod_infra_container_test.go index 82f35999c..c8763de9f 100644 --- a/test/e2e/pod_infra_container_test.go +++ b/test/e2e/pod_infra_container_test.go @@ -25,8 +25,7 @@ var _ = Describe("Podman pod create", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() - podmanTest.RestoreArtifact(infra) + podmanTest.SeedImages() }) AfterEach(func() { @@ -113,12 +112,10 @@ var _ = Describe("Podman pod create", func() { session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) - podmanTest.RestoreArtifact(nginx) session = podmanTest.Podman([]string{"run", "-d", "--pod", podID, nginx}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) - podmanTest.RestoreArtifact(fedoraMinimal) session = podmanTest.Podman([]string{"run", "--pod", podID, fedoraMinimal, "curl", "localhost:80"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) @@ -138,7 +135,6 @@ var _ = Describe("Podman pod create", func() { session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) - podmanTest.RestoreArtifact(fedoraMinimal) session = podmanTest.Podman([]string{"run", "--pod", podID, fedoraMinimal, "/bin/sh", "-c", "'touch /dev/shm/hi'"}) session.WaitWithDefaultTimeout() if session.ExitCode() != 0 { @@ -216,7 +212,6 @@ var _ = Describe("Podman pod create", func() { session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) - podmanTest.RestoreArtifact(nginx) session = podmanTest.Podman([]string{"run", "-d", "--pod", podID, nginx}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) diff --git a/test/e2e/pod_inspect_test.go b/test/e2e/pod_inspect_test.go index d1a023153..488dd1685 100644 --- a/test/e2e/pod_inspect_test.go +++ b/test/e2e/pod_inspect_test.go @@ -22,7 +22,7 @@ var _ = Describe("Podman pod inspect", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() }) AfterEach(func() { diff --git a/test/e2e/pod_kill_test.go b/test/e2e/pod_kill_test.go index 23a8ea97e..7cf67bbfc 100644 --- a/test/e2e/pod_kill_test.go +++ b/test/e2e/pod_kill_test.go @@ -23,7 +23,7 @@ var _ = Describe("Podman pod kill", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() }) AfterEach(func() { diff --git a/test/e2e/pod_pause_test.go b/test/e2e/pod_pause_test.go index ab828853b..619ee6f12 100644 --- a/test/e2e/pod_pause_test.go +++ b/test/e2e/pod_pause_test.go @@ -25,7 +25,7 @@ var _ = Describe("Podman pod pause", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() }) AfterEach(func() { diff --git a/test/e2e/pod_pod_namespaces.go b/test/e2e/pod_pod_namespaces.go index 9d6321c0e..83c877f5a 100644 --- a/test/e2e/pod_pod_namespaces.go +++ b/test/e2e/pod_pod_namespaces.go @@ -25,8 +25,7 @@ var _ = Describe("Podman pod create", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() - podmanTest.RestoreArtifact(infra) + podmanTest.SeedImages() }) AfterEach(func() { diff --git a/test/e2e/pod_prune_test.go b/test/e2e/pod_prune_test.go index 8a4ba2399..da0d425cb 100644 --- a/test/e2e/pod_prune_test.go +++ b/test/e2e/pod_prune_test.go @@ -22,7 +22,7 @@ var _ = Describe("Podman pod prune", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() }) AfterEach(func() { diff --git a/test/e2e/pod_ps_test.go b/test/e2e/pod_ps_test.go index 8513c6c2e..6d5873caa 100644 --- a/test/e2e/pod_ps_test.go +++ b/test/e2e/pod_ps_test.go @@ -24,7 +24,7 @@ var _ = Describe("Podman ps", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() }) AfterEach(func() { diff --git a/test/e2e/pod_restart_test.go b/test/e2e/pod_restart_test.go index 7b19ecc94..691fe5f0c 100644 --- a/test/e2e/pod_restart_test.go +++ b/test/e2e/pod_restart_test.go @@ -22,7 +22,7 @@ var _ = Describe("Podman pod restart", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() }) AfterEach(func() { diff --git a/test/e2e/pod_rm_test.go b/test/e2e/pod_rm_test.go index 7417a1298..0d3f47f30 100644 --- a/test/e2e/pod_rm_test.go +++ b/test/e2e/pod_rm_test.go @@ -23,7 +23,7 @@ var _ = Describe("Podman pod rm", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() }) AfterEach(func() { diff --git a/test/e2e/pod_start_test.go b/test/e2e/pod_start_test.go index 967fbc2da..2722cb5b3 100644 --- a/test/e2e/pod_start_test.go +++ b/test/e2e/pod_start_test.go @@ -22,7 +22,7 @@ var _ = Describe("Podman pod start", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() }) AfterEach(func() { diff --git a/test/e2e/pod_stats_test.go b/test/e2e/pod_stats_test.go index 6018b4494..01176f97c 100644 --- a/test/e2e/pod_stats_test.go +++ b/test/e2e/pod_stats_test.go @@ -25,7 +25,7 @@ var _ = Describe("Podman pod stats", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() }) AfterEach(func() { diff --git a/test/e2e/pod_stop_test.go b/test/e2e/pod_stop_test.go index 9fd9e3ef4..361a63a7f 100644 --- a/test/e2e/pod_stop_test.go +++ b/test/e2e/pod_stop_test.go @@ -22,7 +22,7 @@ var _ = Describe("Podman pod stop", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() }) AfterEach(func() { diff --git a/test/e2e/pod_top_test.go b/test/e2e/pod_top_test.go index 420e4aca9..c313b0675 100644 --- a/test/e2e/pod_top_test.go +++ b/test/e2e/pod_top_test.go @@ -26,7 +26,7 @@ var _ = Describe("Podman top", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() }) AfterEach(func() { diff --git a/test/e2e/port_test.go b/test/e2e/port_test.go index 7cf3e16bf..e45118361 100644 --- a/test/e2e/port_test.go +++ b/test/e2e/port_test.go @@ -26,7 +26,7 @@ var _ = Describe("Podman port", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() }) AfterEach(func() { @@ -49,7 +49,6 @@ var _ = Describe("Podman port", func() { }) It("podman port -l nginx", func() { - podmanTest.RestoreArtifact(nginx) session := podmanTest.Podman([]string{"run", "-dt", "-P", nginx}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) @@ -62,7 +61,6 @@ var _ = Describe("Podman port", func() { }) It("podman container port -l nginx", func() { - podmanTest.RestoreArtifact(nginx) session := podmanTest.Podman([]string{"container", "run", "-dt", "-P", nginx}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) @@ -75,7 +73,6 @@ var _ = Describe("Podman port", func() { }) It("podman port -l port nginx", func() { - podmanTest.RestoreArtifact(nginx) session := podmanTest.Podman([]string{"run", "-dt", "-P", nginx}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) @@ -88,7 +85,6 @@ var _ = Describe("Podman port", func() { }) It("podman port -a nginx", func() { - podmanTest.RestoreArtifact(nginx) session := podmanTest.Podman([]string{"run", "-dt", "-P", nginx}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) diff --git a/test/e2e/prune_test.go b/test/e2e/prune_test.go index 377c9f5e1..df0525a79 100644 --- a/test/e2e/prune_test.go +++ b/test/e2e/prune_test.go @@ -28,7 +28,7 @@ var _ = Describe("Podman prune", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() }) AfterEach(func() { @@ -77,11 +77,12 @@ var _ = Describe("Podman prune", func() { }) It("podman image prune unused images", func() { - prune := podmanTest.Podman([]string{"image", "prune", "-a"}) + podmanTest.RestoreAllArtifacts() + prune := podmanTest.PodmanNoCache([]string{"image", "prune", "-a"}) prune.WaitWithDefaultTimeout() Expect(prune.ExitCode()).To(Equal(0)) - images := podmanTest.Podman([]string{"images", "-aq"}) + images := podmanTest.PodmanNoCache([]string{"images", "-aq"}) images.WaitWithDefaultTimeout() // all images are unused, so they all should be deleted! Expect(len(images.OutputToStringArray())).To(Equal(0)) @@ -89,12 +90,13 @@ var _ = Describe("Podman prune", func() { It("podman system image prune unused images", func() { SkipIfRemote() + podmanTest.RestoreAllArtifacts() podmanTest.BuildImage(pruneImage, "alpine_bash:latest", "true") - prune := podmanTest.Podman([]string{"system", "prune", "-a", "--force"}) + prune := podmanTest.PodmanNoCache([]string{"system", "prune", "-a", "--force"}) prune.WaitWithDefaultTimeout() Expect(prune.ExitCode()).To(Equal(0)) - images := podmanTest.Podman([]string{"images", "-aq"}) + images := podmanTest.PodmanNoCache([]string{"images", "-aq"}) images.WaitWithDefaultTimeout() // all images are unused, so they all should be deleted! Expect(len(images.OutputToStringArray())).To(Equal(0)) diff --git a/test/e2e/ps_test.go b/test/e2e/ps_test.go index 7edb350f3..b0a28501a 100644 --- a/test/e2e/ps_test.go +++ b/test/e2e/ps_test.go @@ -27,7 +27,7 @@ var _ = Describe("Podman ps", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() }) AfterEach(func() { diff --git a/test/e2e/pull_test.go b/test/e2e/pull_test.go index 4e4e80d56..d6e7b44d1 100644 --- a/test/e2e/pull_test.go +++ b/test/e2e/pull_test.go @@ -28,7 +28,6 @@ var _ = Describe("Podman pull", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() }) AfterEach(func() { @@ -39,132 +38,135 @@ var _ = Describe("Podman pull", func() { }) It("podman pull from docker a not existing image", func() { - session := podmanTest.Podman([]string{"pull", "ibetthisdoesntexistthere:foo"}) + session := podmanTest.PodmanNoCache([]string{"pull", "ibetthisdoesntexistthere:foo"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Not(Equal(0))) }) It("podman pull from docker with tag", func() { - session := podmanTest.Podman([]string{"pull", "busybox:glibc"}) + session := podmanTest.PodmanNoCache([]string{"pull", "busybox:glibc"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) - session = podmanTest.Podman([]string{"rmi", "busybox:glibc"}) + session = podmanTest.PodmanNoCache([]string{"rmi", "busybox:glibc"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) }) It("podman pull from docker without tag", func() { - session := podmanTest.Podman([]string{"pull", "busybox"}) + session := podmanTest.PodmanNoCache([]string{"pull", "busybox"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) - session = podmanTest.Podman([]string{"rmi", "busybox"}) + session = podmanTest.PodmanNoCache([]string{"rmi", "busybox"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) }) It("podman pull from alternate registry with tag", func() { - session := podmanTest.Podman([]string{"pull", nginx}) + session := podmanTest.PodmanNoCache([]string{"pull", nginx}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) - session = podmanTest.Podman([]string{"rmi", nginx}) + session = podmanTest.PodmanNoCache([]string{"rmi", nginx}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) }) It("podman pull from alternate registry without tag", func() { - session := podmanTest.Podman([]string{"pull", "quay.io/libpod/alpine_nginx"}) + session := podmanTest.PodmanNoCache([]string{"pull", "quay.io/libpod/alpine_nginx"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) - session = podmanTest.Podman([]string{"rmi", "quay.io/libpod/alpine_nginx"}) + session = podmanTest.PodmanNoCache([]string{"rmi", "quay.io/libpod/alpine_nginx"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) }) It("podman pull by digest", func() { - session := podmanTest.Podman([]string{"pull", "alpine@sha256:1072e499f3f655a032e88542330cf75b02e7bdf673278f701d7ba61629ee3ebe"}) + session := podmanTest.PodmanNoCache([]string{"pull", "alpine@sha256:1072e499f3f655a032e88542330cf75b02e7bdf673278f701d7ba61629ee3ebe"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) - session = podmanTest.Podman([]string{"rmi", "alpine:none"}) + session = podmanTest.PodmanNoCache([]string{"rmi", "alpine:none"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) }) It("podman pull bogus image", func() { - session := podmanTest.Podman([]string{"pull", "umohnani/get-started"}) + session := podmanTest.PodmanNoCache([]string{"pull", "umohnani/get-started"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Not(Equal(0))) }) It("podman pull from docker-archive", func() { + podmanTest.RestoreArtifact(ALPINE) tarfn := filepath.Join(podmanTest.TempDir, "alp.tar") - session := podmanTest.Podman([]string{"save", "-o", tarfn, "alpine"}) + session := podmanTest.PodmanNoCache([]string{"save", "-o", tarfn, "alpine"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) - session = podmanTest.Podman([]string{"rmi", "alpine"}) + session = podmanTest.PodmanNoCache([]string{"rmi", "alpine"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) - session = podmanTest.Podman([]string{"pull", fmt.Sprintf("docker-archive:%s", tarfn)}) + session = podmanTest.PodmanNoCache([]string{"pull", fmt.Sprintf("docker-archive:%s", tarfn)}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) - session = podmanTest.Podman([]string{"rmi", "alpine"}) + session = podmanTest.PodmanNoCache([]string{"rmi", "alpine"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) }) It("podman pull from oci-archive", func() { + podmanTest.RestoreArtifact(ALPINE) tarfn := filepath.Join(podmanTest.TempDir, "oci-alp.tar") - session := podmanTest.Podman([]string{"save", "--format", "oci-archive", "-o", tarfn, "alpine"}) + session := podmanTest.PodmanNoCache([]string{"save", "--format", "oci-archive", "-o", tarfn, "alpine"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) - session = podmanTest.Podman([]string{"rmi", "alpine"}) + session = podmanTest.PodmanNoCache([]string{"rmi", "alpine"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) - session = podmanTest.Podman([]string{"pull", fmt.Sprintf("oci-archive:%s", tarfn)}) + session = podmanTest.PodmanNoCache([]string{"pull", fmt.Sprintf("oci-archive:%s", tarfn)}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) - session = podmanTest.Podman([]string{"rmi", "alpine"}) + session = podmanTest.PodmanNoCache([]string{"rmi", "alpine"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) }) It("podman pull from local directory", func() { + podmanTest.RestoreArtifact(ALPINE) dirpath := filepath.Join(podmanTest.TempDir, "alpine") os.MkdirAll(dirpath, os.ModePerm) imgPath := fmt.Sprintf("dir:%s", dirpath) - session := podmanTest.Podman([]string{"push", "alpine", imgPath}) + session := podmanTest.PodmanNoCache([]string{"push", "alpine", imgPath}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) - session = podmanTest.Podman([]string{"rmi", "alpine"}) + session = podmanTest.PodmanNoCache([]string{"rmi", "alpine"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) - session = podmanTest.Podman([]string{"pull", imgPath}) + session = podmanTest.PodmanNoCache([]string{"pull", imgPath}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) - session = podmanTest.Podman([]string{"rmi", "alpine"}) + session = podmanTest.PodmanNoCache([]string{"rmi", "alpine"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) }) It("podman pull check quiet", func() { podmanTest.RestoreArtifact(ALPINE) - setup := podmanTest.Podman([]string{"images", ALPINE, "-q", "--no-trunc"}) + setup := podmanTest.PodmanNoCache([]string{"images", ALPINE, "-q", "--no-trunc"}) setup.WaitWithDefaultTimeout() Expect(setup.ExitCode()).To(Equal(0)) shortImageId := strings.Split(setup.OutputToString(), ":")[1] - rmi := podmanTest.Podman([]string{"rmi", ALPINE}) + rmi := podmanTest.PodmanNoCache([]string{"rmi", ALPINE}) rmi.WaitWithDefaultTimeout() Expect(rmi.ExitCode()).To(Equal(0)) - pull := podmanTest.Podman([]string{"pull", "-q", ALPINE}) + pull := podmanTest.PodmanNoCache([]string{"pull", "-q", ALPINE}) pull.WaitWithDefaultTimeout() Expect(pull.ExitCode()).To(Equal(0)) @@ -172,17 +174,17 @@ var _ = Describe("Podman pull", func() { }) It("podman pull check all tags", func() { - session := podmanTest.Podman([]string{"pull", "--all-tags", "alpine"}) + session := podmanTest.PodmanNoCache([]string{"pull", "--all-tags", "alpine"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) Expect(session.LineInOuputStartsWith("Pulled Images:")).To(BeTrue()) - session = podmanTest.Podman([]string{"images"}) + session = podmanTest.PodmanNoCache([]string{"images"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) Expect(len(session.OutputToStringArray())).To(BeNumerically(">", 4)) - rmi := podmanTest.Podman([]string{"rmi", "-a", "-f"}) + rmi := podmanTest.PodmanNoCache([]string{"rmi", "-a", "-f"}) rmi.WaitWithDefaultTimeout() Expect(rmi.ExitCode()).To(Equal(0)) }) diff --git a/test/e2e/push_test.go b/test/e2e/push_test.go index 009067482..de2416868 100644 --- a/test/e2e/push_test.go +++ b/test/e2e/push_test.go @@ -38,22 +38,18 @@ var _ = Describe("Podman push", func() { }) It("podman push to containers/storage", func() { - session := podmanTest.Podman([]string{"push", ALPINE, "containers-storage:busybox:test"}) + session := podmanTest.PodmanNoCache([]string{"push", ALPINE, "containers-storage:busybox:test"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) - session = podmanTest.Podman([]string{"rmi", ALPINE}) - session.WaitWithDefaultTimeout() - Expect(session.ExitCode()).To(Equal(0)) - - session = podmanTest.Podman([]string{"rmi", "busybox:test"}) + session = podmanTest.PodmanNoCache([]string{"rmi", ALPINE}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) }) It("podman push to dir", func() { bbdir := filepath.Join(podmanTest.TempDir, "busybox") - session := podmanTest.Podman([]string{"push", "--remove-signatures", ALPINE, + session := podmanTest.PodmanNoCache([]string{"push", "--remove-signatures", ALPINE, fmt.Sprintf("dir:%s", bbdir)}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) @@ -65,8 +61,7 @@ var _ = Describe("Podman push", func() { } lock := GetPortLock("5000") defer lock.Unlock() - podmanTest.RestoreArtifact(registry) - session := podmanTest.Podman([]string{"run", "-d", "--name", "registry", "-p", "5000:5000", registry, "/entrypoint.sh", "/etc/docker/registry/config.yml"}) + session := podmanTest.PodmanNoCache([]string{"run", "-d", "--name", "registry", "-p", "5000:5000", registry, "/entrypoint.sh", "/etc/docker/registry/config.yml"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) @@ -74,7 +69,7 @@ var _ = Describe("Podman push", func() { Skip("Can not start docker registry.") } - push := podmanTest.Podman([]string{"push", "--tls-verify=false", "--remove-signatures", ALPINE, "localhost:5000/my-alpine"}) + push := podmanTest.PodmanNoCache([]string{"push", "--tls-verify=false", "--remove-signatures", ALPINE, "localhost:5000/my-alpine"}) push.WaitWithDefaultTimeout() Expect(push.ExitCode()).To(Equal(0)) }) @@ -106,8 +101,7 @@ var _ = Describe("Podman push", func() { } lock := GetPortLock("5000") defer lock.Unlock() - podmanTest.RestoreArtifact(registry) - session := podmanTest.Podman([]string{"run", "--entrypoint", "htpasswd", registry, "-Bbn", "podmantest", "test"}) + session := podmanTest.PodmanNoCache([]string{"run", "--entrypoint", "htpasswd", registry, "-Bbn", "podmantest", "test"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) @@ -117,7 +111,7 @@ var _ = Describe("Podman push", func() { f.WriteString(session.OutputToString()) f.Sync() - session = podmanTest.Podman([]string{"run", "-d", "-p", "5000:5000", "--name", "registry", "-v", + session = podmanTest.PodmanNoCache([]string{"run", "-d", "-p", "5000:5000", "--name", "registry", "-v", strings.Join([]string{authPath, "/auth"}, ":"), "-e", "REGISTRY_AUTH=htpasswd", "-e", "REGISTRY_AUTH_HTPASSWD_REALM=Registry Realm", "-e", "REGISTRY_AUTH_HTPASSWD_PATH=/auth/htpasswd", "-v", strings.Join([]string{certPath, "/certs"}, ":"), "-e", "REGISTRY_HTTP_TLS_CERTIFICATE=/certs/domain.crt", @@ -129,36 +123,36 @@ var _ = Describe("Podman push", func() { Skip("Can not start docker registry.") } - session = podmanTest.Podman([]string{"logs", "registry"}) + session = podmanTest.PodmanNoCache([]string{"logs", "registry"}) session.WaitWithDefaultTimeout() - push := podmanTest.Podman([]string{"push", "--creds=podmantest:test", ALPINE, "localhost:5000/tlstest"}) + push := podmanTest.PodmanNoCache([]string{"push", "--creds=podmantest:test", ALPINE, "localhost:5000/tlstest"}) push.WaitWithDefaultTimeout() Expect(push.ExitCode()).To(Not(Equal(0))) - push = podmanTest.Podman([]string{"push", "--creds=podmantest:test", "--tls-verify=false", ALPINE, "localhost:5000/tlstest"}) + push = podmanTest.PodmanNoCache([]string{"push", "--creds=podmantest:test", "--tls-verify=false", ALPINE, "localhost:5000/tlstest"}) push.WaitWithDefaultTimeout() Expect(push.ExitCode()).To(Equal(0)) setup := SystemExec("cp", []string{filepath.Join(certPath, "domain.crt"), "/etc/containers/certs.d/localhost:5000/ca.crt"}) Expect(setup.ExitCode()).To(Equal(0)) - push = podmanTest.Podman([]string{"push", "--creds=podmantest:wrongpasswd", ALPINE, "localhost:5000/credstest"}) + push = podmanTest.PodmanNoCache([]string{"push", "--creds=podmantest:wrongpasswd", ALPINE, "localhost:5000/credstest"}) push.WaitWithDefaultTimeout() Expect(push.ExitCode()).To(Not(Equal(0))) - push = podmanTest.Podman([]string{"push", "--creds=podmantest:test", "--cert-dir=fakedir", ALPINE, "localhost:5000/certdirtest"}) + push = podmanTest.PodmanNoCache([]string{"push", "--creds=podmantest:test", "--cert-dir=fakedir", ALPINE, "localhost:5000/certdirtest"}) push.WaitWithDefaultTimeout() Expect(push.ExitCode()).To(Not(Equal(0))) - push = podmanTest.Podman([]string{"push", "--creds=podmantest:test", ALPINE, "localhost:5000/defaultflags"}) + push = podmanTest.PodmanNoCache([]string{"push", "--creds=podmantest:test", ALPINE, "localhost:5000/defaultflags"}) push.WaitWithDefaultTimeout() Expect(push.ExitCode()).To(Equal(0)) }) It("podman push to docker-archive", func() { tarfn := filepath.Join(podmanTest.TempDir, "alp.tar") - session := podmanTest.Podman([]string{"push", ALPINE, + session := podmanTest.PodmanNoCache([]string{"push", ALPINE, fmt.Sprintf("docker-archive:%s:latest", tarfn)}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) @@ -177,7 +171,7 @@ var _ = Describe("Podman push", func() { Skip("Docker is not available") } - session := podmanTest.Podman([]string{"push", ALPINE, "docker-daemon:alpine:podmantest"}) + session := podmanTest.PodmanNoCache([]string{"push", ALPINE, "docker-daemon:alpine:podmantest"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) @@ -191,7 +185,7 @@ var _ = Describe("Podman push", func() { It("podman push to oci-archive", func() { tarfn := filepath.Join(podmanTest.TempDir, "alp.tar") - session := podmanTest.Podman([]string{"push", ALPINE, + session := podmanTest.PodmanNoCache([]string{"push", ALPINE, fmt.Sprintf("oci-archive:%s:latest", tarfn)}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) @@ -208,7 +202,7 @@ var _ = Describe("Podman push", func() { setup := SystemExec("ostree", []string{strings.Join([]string{"--repo=", ostreePath}, ""), "init"}) Expect(setup.ExitCode()).To(Equal(0)) - session := podmanTest.Podman([]string{"push", ALPINE, strings.Join([]string{"ostree:alp@", ostreePath}, "")}) + session := podmanTest.PodmanNoCache([]string{"push", ALPINE, strings.Join([]string{"ostree:alp@", ostreePath}, "")}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) @@ -216,7 +210,7 @@ var _ = Describe("Podman push", func() { It("podman push to docker-archive no reference", func() { tarfn := filepath.Join(podmanTest.TempDir, "alp.tar") - session := podmanTest.Podman([]string{"push", ALPINE, + session := podmanTest.PodmanNoCache([]string{"push", ALPINE, fmt.Sprintf("docker-archive:%s", tarfn)}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) @@ -224,7 +218,7 @@ var _ = Describe("Podman push", func() { It("podman push to oci-archive no reference", func() { ociarc := filepath.Join(podmanTest.TempDir, "alp-oci") - session := podmanTest.Podman([]string{"push", ALPINE, + session := podmanTest.PodmanNoCache([]string{"push", ALPINE, fmt.Sprintf("oci-archive:%s", ociarc)}) session.WaitWithDefaultTimeout() diff --git a/test/e2e/restart_test.go b/test/e2e/restart_test.go index 7a9a466d8..2b515f53b 100644 --- a/test/e2e/restart_test.go +++ b/test/e2e/restart_test.go @@ -23,7 +23,7 @@ var _ = Describe("Podman restart", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() }) AfterEach(func() { diff --git a/test/e2e/rm_test.go b/test/e2e/rm_test.go index 29150d67c..2dbabbf6a 100644 --- a/test/e2e/rm_test.go +++ b/test/e2e/rm_test.go @@ -22,7 +22,7 @@ var _ = Describe("Podman rm", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() }) AfterEach(func() { diff --git a/test/e2e/rmi_test.go b/test/e2e/rmi_test.go index e034f24cf..1687bf764 100644 --- a/test/e2e/rmi_test.go +++ b/test/e2e/rmi_test.go @@ -41,14 +41,14 @@ var _ = Describe("Podman rmi", func() { }) It("podman rmi with fq name", func() { - session := podmanTest.Podman([]string{"rmi", ALPINE}) + session := podmanTest.PodmanNoCache([]string{"rmi", ALPINE}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) }) It("podman rmi with short name", func() { - session := podmanTest.Podman([]string{"rmi", "alpine"}) + session := podmanTest.PodmanNoCache([]string{"rmi", "alpine"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) @@ -56,9 +56,9 @@ var _ = Describe("Podman rmi", func() { It("podman rmi all images", func() { podmanTest.PullImages([]string{nginx}) - session := podmanTest.Podman([]string{"rmi", "-a"}) + session := podmanTest.PodmanNoCache([]string{"rmi", "-a"}) session.WaitWithDefaultTimeout() - images := podmanTest.Podman([]string{"images"}) + images := podmanTest.PodmanNoCache([]string{"images"}) images.WaitWithDefaultTimeout() fmt.Println(images.OutputToStringArray()) Expect(session.ExitCode()).To(Equal(0)) @@ -67,22 +67,22 @@ var _ = Describe("Podman rmi", func() { It("podman rmi all images forcibly with short options", func() { podmanTest.PullImages([]string{nginx}) - session := podmanTest.Podman([]string{"rmi", "-fa"}) + session := podmanTest.PodmanNoCache([]string{"rmi", "-fa"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) }) It("podman rmi tagged image", func() { - setup := podmanTest.Podman([]string{"images", "-q", ALPINE}) + setup := podmanTest.PodmanNoCache([]string{"images", "-q", ALPINE}) setup.WaitWithDefaultTimeout() Expect(setup.ExitCode()).To(Equal(0)) - session := podmanTest.Podman([]string{"tag", "alpine", "foo:bar", "foo"}) + session := podmanTest.PodmanNoCache([]string{"tag", "alpine", "foo:bar", "foo"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) - result := podmanTest.Podman([]string{"images", "-q", "foo"}) + result := podmanTest.PodmanNoCache([]string{"images", "-q", "foo"}) result.WaitWithDefaultTimeout() Expect(result.ExitCode()).To(Equal(0)) @@ -90,95 +90,95 @@ var _ = Describe("Podman rmi", func() { }) It("podman rmi image with tags by ID cannot be done without force", func() { - setup := podmanTest.Podman([]string{"images", "-q", ALPINE}) + setup := podmanTest.PodmanNoCache([]string{"images", "-q", ALPINE}) setup.WaitWithDefaultTimeout() Expect(setup.ExitCode()).To(Equal(0)) alpineId := setup.OutputToString() - session := podmanTest.Podman([]string{"tag", "alpine", "foo:bar", "foo"}) + session := podmanTest.PodmanNoCache([]string{"tag", "alpine", "foo:bar", "foo"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) // Trying without --force should fail - result := podmanTest.Podman([]string{"rmi", alpineId}) + result := podmanTest.PodmanNoCache([]string{"rmi", alpineId}) result.WaitWithDefaultTimeout() Expect(result.ExitCode()).ToNot(Equal(0)) // With --force it should work - resultForce := podmanTest.Podman([]string{"rmi", "-f", alpineId}) + resultForce := podmanTest.PodmanNoCache([]string{"rmi", "-f", alpineId}) resultForce.WaitWithDefaultTimeout() Expect(resultForce.ExitCode()).To(Equal(0)) }) It("podman rmi image that is a parent of another image", func() { SkipIfRemote() - session := podmanTest.Podman([]string{"rmi", "-fa"}) + session := podmanTest.PodmanNoCache([]string{"rmi", "-fa"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) - session = podmanTest.Podman([]string{"run", "--name", "c_test", ALPINE, "true"}) + session = podmanTest.PodmanNoCache([]string{"run", "--name", "c_test", ALPINE, "true"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) - session = podmanTest.Podman([]string{"commit", "-q", "c_test", "test"}) + session = podmanTest.PodmanNoCache([]string{"commit", "-q", "c_test", "test"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) - session = podmanTest.Podman([]string{"rm", "c_test"}) + session = podmanTest.PodmanNoCache([]string{"rm", "c_test"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) - session = podmanTest.Podman([]string{"rmi", ALPINE}) + session = podmanTest.PodmanNoCache([]string{"rmi", ALPINE}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) - session = podmanTest.Podman([]string{"images", "-q"}) + session = podmanTest.PodmanNoCache([]string{"images", "-q"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) Expect(len(session.OutputToStringArray())).To(Equal(1)) - session = podmanTest.Podman([]string{"images", "-q", "-a"}) + session = podmanTest.PodmanNoCache([]string{"images", "-q", "-a"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) Expect(len(session.OutputToStringArray())).To(Equal(2)) untaggedImg := session.OutputToStringArray()[1] - session = podmanTest.Podman([]string{"rmi", "-f", untaggedImg}) + session = podmanTest.PodmanNoCache([]string{"rmi", "-f", untaggedImg}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Not(Equal(0))) }) It("podman rmi image that is created from another named imaged", func() { SkipIfRemote() - session := podmanTest.Podman([]string{"rmi", "-fa"}) + session := podmanTest.PodmanNoCache([]string{"rmi", "-fa"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) - session = podmanTest.Podman([]string{"create", "--name", "c_test1", ALPINE, "true"}) + session = podmanTest.PodmanNoCache([]string{"create", "--name", "c_test1", ALPINE, "true"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) - session = podmanTest.Podman([]string{"commit", "-q", "c_test1", "test1"}) + session = podmanTest.PodmanNoCache([]string{"commit", "-q", "c_test1", "test1"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) - session = podmanTest.Podman([]string{"create", "--name", "c_test2", "test1", "true"}) + session = podmanTest.PodmanNoCache([]string{"create", "--name", "c_test2", "test1", "true"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) - session = podmanTest.Podman([]string{"commit", "-q", "c_test2", "test2"}) + session = podmanTest.PodmanNoCache([]string{"commit", "-q", "c_test2", "test2"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) - session = podmanTest.Podman([]string{"rm", "-a"}) + session = podmanTest.PodmanNoCache([]string{"rm", "-a"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) - session = podmanTest.Podman([]string{"rmi", "test2"}) + session = podmanTest.PodmanNoCache([]string{"rmi", "test2"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) - session = podmanTest.Podman([]string{"images", "-q"}) + session = podmanTest.PodmanNoCache([]string{"images", "-q"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) Expect(len(session.OutputToStringArray())).To(Equal(2)) @@ -186,7 +186,7 @@ var _ = Describe("Podman rmi", func() { It("podman rmi with cached images", func() { SkipIfRemote() - session := podmanTest.Podman([]string{"rmi", "-fa"}) + session := podmanTest.PodmanNoCache([]string{"rmi", "-fa"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) @@ -205,51 +205,51 @@ var _ = Describe("Podman rmi", func() { ` podmanTest.BuildImage(dockerfile, "test2", "true") - session = podmanTest.Podman([]string{"images", "-q", "-a"}) + session = podmanTest.PodmanNoCache([]string{"images", "-q", "-a"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) numOfImages := len(session.OutputToStringArray()) - session = podmanTest.Podman([]string{"rmi", "test2"}) + session = podmanTest.PodmanNoCache([]string{"rmi", "test2"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) - session = podmanTest.Podman([]string{"images", "-q", "-a"}) + session = podmanTest.PodmanNoCache([]string{"images", "-q", "-a"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) Expect(numOfImages - len(session.OutputToStringArray())).To(Equal(2)) - session = podmanTest.Podman([]string{"rmi", "test"}) + session = podmanTest.PodmanNoCache([]string{"rmi", "test"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) - session = podmanTest.Podman([]string{"images", "-q", "-a"}) + session = podmanTest.PodmanNoCache([]string{"images", "-q", "-a"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) Expect(len(session.OutputToStringArray())).To(Equal(1)) podmanTest.BuildImage(dockerfile, "test3", "true") - session = podmanTest.Podman([]string{"rmi", ALPINE}) + session = podmanTest.PodmanNoCache([]string{"rmi", ALPINE}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) - session = podmanTest.Podman([]string{"rmi", "test3"}) + session = podmanTest.PodmanNoCache([]string{"rmi", "test3"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) - session = podmanTest.Podman([]string{"images", "-q", "-a"}) + session = podmanTest.PodmanNoCache([]string{"images", "-q", "-a"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) Expect(len(session.OutputToString())).To(Equal(0)) }) It("podman rmi -a with no images should be exit 0", func() { - session := podmanTest.Podman([]string{"rmi", "-fa"}) + session := podmanTest.PodmanNoCache([]string{"rmi", "-fa"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) - session2 := podmanTest.Podman([]string{"rmi", "-fa"}) + session2 := podmanTest.PodmanNoCache([]string{"rmi", "-fa"}) session2.WaitWithDefaultTimeout() Expect(session2.ExitCode()).To(Equal(0)) }) @@ -265,12 +265,12 @@ RUN find $LOCAL ` podmanTest.BuildImage(dockerfile, "test", "true") - session := podmanTest.Podman([]string{"rmi", "-a"}) + session := podmanTest.PodmanNoCache([]string{"rmi", "-a"}) session.WaitWithDefaultTimeout() fmt.Println(session.OutputToString()) Expect(session.ExitCode()).To(Equal(0)) - images := podmanTest.Podman([]string{"images", "-aq"}) + images := podmanTest.PodmanNoCache([]string{"images", "-aq"}) images.WaitWithDefaultTimeout() Expect(images.ExitCode()).To(Equal(0)) Expect(len(images.OutputToStringArray())).To(Equal(0)) @@ -279,7 +279,7 @@ RUN find $LOCAL // Don't rerun all tests; just assume that if we get that diagnostic, // we're getting rmi It("podman image rm is the same as rmi", func() { - session := podmanTest.Podman([]string{"image", "rm"}) + session := podmanTest.PodmanNoCache([]string{"image", "rm"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(125)) Expect(session.LineInOutputContains("image name or ID must be specified")) diff --git a/test/e2e/run_cgroup_parent_test.go b/test/e2e/run_cgroup_parent_test.go index 0d04c5f03..1fb9f6871 100644 --- a/test/e2e/run_cgroup_parent_test.go +++ b/test/e2e/run_cgroup_parent_test.go @@ -25,7 +25,7 @@ var _ = Describe("Podman run with --cgroup-parent", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreArtifact(fedoraMinimal) + podmanTest.SeedImages() }) AfterEach(func() { diff --git a/test/e2e/run_cleanup_test.go b/test/e2e/run_cleanup_test.go index b20e37794..73647b6bb 100644 --- a/test/e2e/run_cleanup_test.go +++ b/test/e2e/run_cleanup_test.go @@ -24,7 +24,7 @@ var _ = Describe("Podman run exit", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.RestoreArtifact(ALPINE) }) AfterEach(func() { diff --git a/test/e2e/run_cpu_test.go b/test/e2e/run_cpu_test.go index 42a66865c..87f89b1dd 100644 --- a/test/e2e/run_cpu_test.go +++ b/test/e2e/run_cpu_test.go @@ -24,7 +24,7 @@ var _ = Describe("Podman run cpu", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() }) AfterEach(func() { diff --git a/test/e2e/run_device_test.go b/test/e2e/run_device_test.go index fac09b78d..2e537a9f9 100644 --- a/test/e2e/run_device_test.go +++ b/test/e2e/run_device_test.go @@ -24,7 +24,7 @@ var _ = Describe("Podman run device", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() }) AfterEach(func() { diff --git a/test/e2e/run_dns_test.go b/test/e2e/run_dns_test.go index 0f4dd6742..f1196ff38 100644 --- a/test/e2e/run_dns_test.go +++ b/test/e2e/run_dns_test.go @@ -24,7 +24,7 @@ var _ = Describe("Podman run dns", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() }) AfterEach(func() { diff --git a/test/e2e/run_entrypoint_test.go b/test/e2e/run_entrypoint_test.go index ee9fd1263..b1344a371 100644 --- a/test/e2e/run_entrypoint_test.go +++ b/test/e2e/run_entrypoint_test.go @@ -24,7 +24,7 @@ var _ = Describe("Podman run entrypoint", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreArtifact(ALPINE) + podmanTest.SeedImages() }) AfterEach(func() { diff --git a/test/e2e/run_exit_test.go b/test/e2e/run_exit_test.go index da4cf7ee7..861d6b3b7 100644 --- a/test/e2e/run_exit_test.go +++ b/test/e2e/run_exit_test.go @@ -24,7 +24,7 @@ var _ = Describe("Podman run exit", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() }) AfterEach(func() { diff --git a/test/e2e/run_memory_test.go b/test/e2e/run_memory_test.go index 05d0b7a18..8fe90c8d8 100644 --- a/test/e2e/run_memory_test.go +++ b/test/e2e/run_memory_test.go @@ -25,7 +25,7 @@ var _ = Describe("Podman run memory", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() }) AfterEach(func() { diff --git a/test/e2e/run_networking_test.go b/test/e2e/run_networking_test.go index 93919925c..1497a651b 100644 --- a/test/e2e/run_networking_test.go +++ b/test/e2e/run_networking_test.go @@ -25,7 +25,7 @@ var _ = Describe("Podman run networking", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() }) AfterEach(func() { @@ -78,7 +78,6 @@ var _ = Describe("Podman run networking", func() { }) It("podman run network expose ports in image metadata", func() { - podmanTest.RestoreArtifact(nginx) session := podmanTest.Podman([]string{"create", "-dt", "-P", nginx}) session.Wait(90) Expect(session.ExitCode()).To(Equal(0)) diff --git a/test/e2e/run_ns_test.go b/test/e2e/run_ns_test.go index 51f921bce..6ba0d1aba 100644 --- a/test/e2e/run_ns_test.go +++ b/test/e2e/run_ns_test.go @@ -25,7 +25,7 @@ var _ = Describe("Podman run ns", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreArtifact(fedoraMinimal) + podmanTest.SeedImages() }) AfterEach(func() { diff --git a/test/e2e/run_passwd_test.go b/test/e2e/run_passwd_test.go index becbc5bfa..bd6a0e036 100644 --- a/test/e2e/run_passwd_test.go +++ b/test/e2e/run_passwd_test.go @@ -24,7 +24,7 @@ var _ = Describe("Podman run passwd", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() }) AfterEach(func() { diff --git a/test/e2e/run_privileged_test.go b/test/e2e/run_privileged_test.go index 16011b2fd..f2c424483 100644 --- a/test/e2e/run_privileged_test.go +++ b/test/e2e/run_privileged_test.go @@ -25,7 +25,7 @@ var _ = Describe("Podman privileged container tests", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() }) AfterEach(func() { diff --git a/test/e2e/run_restart_test.go b/test/e2e/run_restart_test.go index 9976b45e8..8bbdf2056 100644 --- a/test/e2e/run_restart_test.go +++ b/test/e2e/run_restart_test.go @@ -24,7 +24,7 @@ var _ = Describe("Podman run restart containers", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() }) AfterEach(func() { diff --git a/test/e2e/run_selinux_test.go b/test/e2e/run_selinux_test.go index 4d2bad49c..a2228411e 100644 --- a/test/e2e/run_selinux_test.go +++ b/test/e2e/run_selinux_test.go @@ -25,7 +25,7 @@ var _ = Describe("Podman run", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() if !selinux.GetEnabled() { Skip("SELinux not enabled") } diff --git a/test/e2e/run_signal_test.go b/test/e2e/run_signal_test.go index e482adb84..3a5ed483c 100644 --- a/test/e2e/run_signal_test.go +++ b/test/e2e/run_signal_test.go @@ -33,7 +33,7 @@ var _ = Describe("Podman run with --sig-proxy", func() { } podmanTest = PodmanTestCreate(tmpdir) podmanTest.Setup() - podmanTest.RestoreArtifact(fedoraMinimal) + podmanTest.SeedImages() }) AfterEach(func() { diff --git a/test/e2e/run_staticip_test.go b/test/e2e/run_staticip_test.go index 318a7a62d..9753cfc9c 100644 --- a/test/e2e/run_staticip_test.go +++ b/test/e2e/run_staticip_test.go @@ -25,7 +25,7 @@ var _ = Describe("Podman run with --ip flag", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() // Cleanup the CNI networks used by the tests os.RemoveAll("/var/lib/cni/networks/podman") }) diff --git a/test/e2e/run_test.go b/test/e2e/run_test.go index f908fe154..31720ea04 100644 --- a/test/e2e/run_test.go +++ b/test/e2e/run_test.go @@ -32,7 +32,7 @@ var _ = Describe("Podman run", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() }) AfterEach(func() { @@ -51,7 +51,6 @@ var _ = Describe("Podman run", func() { It("podman run a container based on a complex local image name", func() { SkipIfRootless() imageName := strings.TrimPrefix(nginx, "quay.io/") - podmanTest.RestoreArtifact(nginx) session := podmanTest.Podman([]string{"run", imageName, "ls"}) session.WaitWithDefaultTimeout() Expect(session.ErrorToString()).ToNot(ContainSubstring("Trying to pull")) @@ -59,7 +58,6 @@ var _ = Describe("Podman run", func() { }) It("podman run a container based on on a short name with localhost", func() { - podmanTest.RestoreArtifact(nginx) tag := podmanTest.Podman([]string{"tag", nginx, "localhost/libpod/alpine_nginx:latest"}) tag.WaitWithDefaultTimeout() @@ -73,7 +71,6 @@ var _ = Describe("Podman run", func() { }) It("podman container run a container based on on a short name with localhost", func() { - podmanTest.RestoreArtifact(nginx) tag := podmanTest.Podman([]string{"image", "tag", nginx, "localhost/libpod/alpine_nginx:latest"}) tag.WaitWithDefaultTimeout() @@ -229,7 +226,6 @@ var _ = Describe("Podman run", func() { It("podman run limits test", func() { SkipIfRootless() - podmanTest.RestoreArtifact(fedoraMinimal) session := podmanTest.Podman([]string{"run", "--rm", "--ulimit", "rtprio=99", "--cap-add=sys_nice", fedoraMinimal, "cat", "/proc/self/sched"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) @@ -349,11 +345,12 @@ var _ = Describe("Podman run", func() { }) It("podman run tagged image", func() { - tag := podmanTest.Podman([]string{"tag", "busybox", "bb"}) + podmanTest.RestoreArtifact(BB) + tag := podmanTest.PodmanNoCache([]string{"tag", "busybox", "bb"}) tag.WaitWithDefaultTimeout() Expect(tag.ExitCode()).To(Equal(0)) - session := podmanTest.Podman([]string{"run", "--rm", "bb", "ls"}) + session := podmanTest.PodmanNoCache([]string{"run", "--rm", "bb", "ls"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) }) @@ -536,15 +533,10 @@ var _ = Describe("Podman run", func() { }) It("podman run with built-in volume image", func() { - podmanTest.RestoreArtifact(redis) session := podmanTest.Podman([]string{"run", "--rm", redis, "ls"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) - session = podmanTest.Podman([]string{"rmi", redis}) - session.WaitWithDefaultTimeout() - Expect(session.ExitCode()).To(Equal(0)) - dockerfile := `FROM busybox RUN mkdir -p /myvol/data && chown -R mail.0 /myvol VOLUME ["/myvol/data"] @@ -555,10 +547,6 @@ USER mail` session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) Expect(session.OutputToString()).To(ContainSubstring("mail root")) - - session = podmanTest.Podman([]string{"rmi", "test"}) - session.WaitWithDefaultTimeout() - Expect(session.ExitCode()).To(Equal(0)) }) It("podman run --volumes-from flag", func() { @@ -571,7 +559,6 @@ USER mail` err = ioutil.WriteFile(volFile, []byte(data), 0755) Expect(err).To(BeNil()) - podmanTest.RestoreArtifact(redis) session := podmanTest.Podman([]string{"create", "--volume", vol + ":/myvol", redis, "sh"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) @@ -587,7 +574,6 @@ USER mail` }) It("podman run --volumes-from flag with built-in volumes", func() { - podmanTest.RestoreArtifact(redis) session := podmanTest.Podman([]string{"create", redis, "sh"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) @@ -641,7 +627,6 @@ USER mail` }) It("podman run findmnt nothing shared", func() { - podmanTest.RestoreArtifact(fedoraMinimal) vol1 := filepath.Join(podmanTest.TempDir, "vol-test1") err := os.MkdirAll(vol1, 0755) Expect(err).To(BeNil()) @@ -657,7 +642,6 @@ USER mail` }) It("podman run findmnt shared", func() { - podmanTest.RestoreArtifact(fedoraMinimal) vol1 := filepath.Join(podmanTest.TempDir, "vol-test1") err := os.MkdirAll(vol1, 0755) Expect(err).To(BeNil()) @@ -765,10 +749,9 @@ USER mail` }) It("podman run with restart-policy always restarts containers", func() { - podmanTest.RestoreArtifact(fedoraMinimal) testDir := filepath.Join(podmanTest.RunRoot, "restart-test") - err := os.Mkdir(testDir, 0755) + err := os.MkdirAll(testDir, 0755) Expect(err).To(BeNil()) aliveFile := filepath.Join(testDir, "running") diff --git a/test/e2e/run_userns_test.go b/test/e2e/run_userns_test.go index ce6971cd1..be3f7df49 100644 --- a/test/e2e/run_userns_test.go +++ b/test/e2e/run_userns_test.go @@ -31,7 +31,7 @@ var _ = Describe("Podman UserNS support", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() }) AfterEach(func() { diff --git a/test/e2e/run_volume_test.go b/test/e2e/run_volume_test.go index d031ca143..d89c80909 100644 --- a/test/e2e/run_volume_test.go +++ b/test/e2e/run_volume_test.go @@ -26,7 +26,7 @@ var _ = Describe("Podman run with volumes", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() }) AfterEach(func() { diff --git a/test/e2e/runlabel_test.go b/test/e2e/runlabel_test.go index f52a2b8fc..5ef68603e 100644 --- a/test/e2e/runlabel_test.go +++ b/test/e2e/runlabel_test.go @@ -37,7 +37,7 @@ var _ = Describe("podman container runlabel", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() }) AfterEach(func() { diff --git a/test/e2e/save_test.go b/test/e2e/save_test.go index ffb5182d6..be1ede962 100644 --- a/test/e2e/save_test.go +++ b/test/e2e/save_test.go @@ -37,7 +37,7 @@ var _ = Describe("Podman save", func() { It("podman save output flag", func() { outfile := filepath.Join(podmanTest.TempDir, "alpine.tar") - save := podmanTest.Podman([]string{"save", "-o", outfile, ALPINE}) + save := podmanTest.PodmanNoCache([]string{"save", "-o", outfile, ALPINE}) save.WaitWithDefaultTimeout() Expect(save.ExitCode()).To(Equal(0)) }) @@ -45,7 +45,7 @@ var _ = Describe("Podman save", func() { It("podman save oci flag", func() { outfile := filepath.Join(podmanTest.TempDir, "alpine.tar") - save := podmanTest.Podman([]string{"save", "-o", outfile, "--format", "oci-archive", ALPINE}) + save := podmanTest.PodmanNoCache([]string{"save", "-o", outfile, "--format", "oci-archive", ALPINE}) save.WaitWithDefaultTimeout() Expect(save.ExitCode()).To(Equal(0)) }) @@ -54,7 +54,7 @@ var _ = Describe("Podman save", func() { Skip("Pipe redirection in ginkgo probably wont work") outfile := filepath.Join(podmanTest.TempDir, "alpine.tar") - save := podmanTest.Podman([]string{"save", ALPINE, ">", outfile}) + save := podmanTest.PodmanNoCache([]string{"save", ALPINE, ">", outfile}) save.WaitWithDefaultTimeout() Expect(save.ExitCode()).To(Equal(0)) }) @@ -62,7 +62,7 @@ var _ = Describe("Podman save", func() { It("podman save quiet flag", func() { outfile := filepath.Join(podmanTest.TempDir, "alpine.tar") - save := podmanTest.Podman([]string{"save", "-q", "-o", outfile, ALPINE}) + save := podmanTest.PodmanNoCache([]string{"save", "-q", "-o", outfile, ALPINE}) save.WaitWithDefaultTimeout() Expect(save.ExitCode()).To(Equal(0)) }) @@ -70,7 +70,7 @@ var _ = Describe("Podman save", func() { It("podman save bogus image", func() { outfile := filepath.Join(podmanTest.TempDir, "alpine.tar") - save := podmanTest.Podman([]string{"save", "-o", outfile, "FOOBAR"}) + save := podmanTest.PodmanNoCache([]string{"save", "-o", outfile, "FOOBAR"}) save.WaitWithDefaultTimeout() Expect(save.ExitCode()).To(Not(Equal(0))) }) @@ -81,7 +81,7 @@ var _ = Describe("Podman save", func() { } outdir := filepath.Join(podmanTest.TempDir, "save") - save := podmanTest.Podman([]string{"save", "--format", "oci-dir", "-o", outdir, ALPINE}) + save := podmanTest.PodmanNoCache([]string{"save", "--format", "oci-dir", "-o", outdir, ALPINE}) save.WaitWithDefaultTimeout() Expect(save.ExitCode()).To(Equal(0)) }) @@ -92,7 +92,7 @@ var _ = Describe("Podman save", func() { } outdir := filepath.Join(podmanTest.TempDir, "save") - save := podmanTest.Podman([]string{"save", "--format", "docker-dir", "-o", outdir, ALPINE}) + save := podmanTest.PodmanNoCache([]string{"save", "--format", "docker-dir", "-o", outdir, ALPINE}) save.WaitWithDefaultTimeout() Expect(save.ExitCode()).To(Equal(0)) }) @@ -103,7 +103,7 @@ var _ = Describe("Podman save", func() { } outdir := filepath.Join(podmanTest.TempDir, "save") - save := podmanTest.Podman([]string{"save", "--compress", "--format", "docker-dir", "-o", outdir, ALPINE}) + save := podmanTest.PodmanNoCache([]string{"save", "--compress", "--format", "docker-dir", "-o", outdir, ALPINE}) save.WaitWithDefaultTimeout() Expect(save.ExitCode()).To(Equal(0)) }) @@ -111,7 +111,7 @@ var _ = Describe("Podman save", func() { It("podman save bad filename", func() { outdir := filepath.Join(podmanTest.TempDir, "save:colon") - save := podmanTest.Podman([]string{"save", "--compress", "--format", "docker-dir", "-o", outdir, ALPINE}) + save := podmanTest.PodmanNoCache([]string{"save", "--compress", "--format", "docker-dir", "-o", outdir, ALPINE}) save.WaitWithDefaultTimeout() Expect(save.ExitCode()).To(Not(Equal(0))) }) diff --git a/test/e2e/search_test.go b/test/e2e/search_test.go index 72b083de8..3b1da859c 100644 --- a/test/e2e/search_test.go +++ b/test/e2e/search_test.go @@ -75,8 +75,8 @@ registries = ['{{.Host}}:{{.Port}}']` podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() + podmanTest.SeedImages() - podmanTest.RestoreAllArtifacts() }) AfterEach(func() { @@ -168,7 +168,6 @@ registries = ['{{.Host}}:{{.Port}}']` lock := GetPortLock(registryEndpoints[0].Port) defer lock.Unlock() - podmanTest.RestoreArtifact(registry) fakereg := podmanTest.Podman([]string{"run", "-d", "--name", "registry", "-p", fmt.Sprintf("%s:5000", registryEndpoints[0].Port), registry, "/entrypoint.sh", "/etc/docker/registry/config.yml"}) @@ -196,7 +195,6 @@ registries = ['{{.Host}}:{{.Port}}']` } lock := GetPortLock(registryEndpoints[3].Port) defer lock.Unlock() - podmanTest.RestoreArtifact(registry) registry := podmanTest.Podman([]string{"run", "-d", "--name", "registry3", "-p", fmt.Sprintf("%s:5000", registryEndpoints[3].Port), registry, "/entrypoint.sh", "/etc/docker/registry/config.yml"}) @@ -207,11 +205,12 @@ registries = ['{{.Host}}:{{.Port}}']` Skip("Can not start docker registry.") } + podmanTest.RestoreArtifact(ALPINE) image := fmt.Sprintf("%s/my-alpine", registryEndpoints[3].Address()) - push := podmanTest.Podman([]string{"push", "--tls-verify=false", "--remove-signatures", ALPINE, image}) + push := podmanTest.PodmanNoCache([]string{"push", "--tls-verify=false", "--remove-signatures", ALPINE, image}) push.WaitWithDefaultTimeout() Expect(push.ExitCode()).To(Equal(0)) - search := podmanTest.Podman([]string{"search", image, "--tls-verify=false"}) + search := podmanTest.PodmanNoCache([]string{"search", image, "--tls-verify=false"}) search.WaitWithDefaultTimeout() Expect(search.ExitCode()).To(Equal(0)) @@ -225,7 +224,6 @@ registries = ['{{.Host}}:{{.Port}}']` lock := GetPortLock(registryEndpoints[4].Port) defer lock.Unlock() - podmanTest.RestoreArtifact(registry) registry := podmanTest.Podman([]string{"run", "-d", "-p", fmt.Sprintf("%s:5000", registryEndpoints[4].Port), "--name", "registry4", registry, "/entrypoint.sh", "/etc/docker/registry/config.yml"}) registry.WaitWithDefaultTimeout() @@ -235,8 +233,9 @@ registries = ['{{.Host}}:{{.Port}}']` Skip("Can not start docker registry.") } + podmanTest.RestoreArtifact(ALPINE) image := fmt.Sprintf("%s/my-alpine", registryEndpoints[4].Address()) - push := podmanTest.Podman([]string{"push", "--tls-verify=false", "--remove-signatures", ALPINE, image}) + push := podmanTest.PodmanNoCache([]string{"push", "--tls-verify=false", "--remove-signatures", ALPINE, image}) push.WaitWithDefaultTimeout() Expect(push.ExitCode()).To(Equal(0)) @@ -246,7 +245,7 @@ registries = ['{{.Host}}:{{.Port}}']` podmanTest.setRegistriesConfigEnv(buffer.Bytes()) ioutil.WriteFile(fmt.Sprintf("%s/registry4.conf", tempdir), buffer.Bytes(), 0644) - search := podmanTest.Podman([]string{"search", image}) + search := podmanTest.PodmanNoCache([]string{"search", image}) search.WaitWithDefaultTimeout() Expect(search.ExitCode()).To(Equal(0)) @@ -264,7 +263,6 @@ registries = ['{{.Host}}:{{.Port}}']` } lock := GetPortLock(registryEndpoints[5].Port) defer lock.Unlock() - podmanTest.RestoreArtifact(registry) registry := podmanTest.Podman([]string{"run", "-d", "-p", fmt.Sprintf("%s:5000", registryEndpoints[5].Port), "--name", "registry5", registry}) registry.WaitWithDefaultTimeout() @@ -274,8 +272,9 @@ registries = ['{{.Host}}:{{.Port}}']` Skip("Can not start docker registry.") } + podmanTest.RestoreArtifact(ALPINE) image := fmt.Sprintf("%s/my-alpine", registryEndpoints[5].Address()) - push := podmanTest.Podman([]string{"push", "--tls-verify=false", "--remove-signatures", ALPINE, image}) + push := podmanTest.PodmanNoCache([]string{"push", "--tls-verify=false", "--remove-signatures", ALPINE, image}) push.WaitWithDefaultTimeout() Expect(push.ExitCode()).To(Equal(0)) @@ -284,7 +283,7 @@ registries = ['{{.Host}}:{{.Port}}']` podmanTest.setRegistriesConfigEnv(buffer.Bytes()) ioutil.WriteFile(fmt.Sprintf("%s/registry5.conf", tempdir), buffer.Bytes(), 0644) - search := podmanTest.Podman([]string{"search", image, "--tls-verify=true"}) + search := podmanTest.PodmanNoCache([]string{"search", image, "--tls-verify=true"}) search.WaitWithDefaultTimeout() Expect(search.ExitCode()).To(Equal(0)) @@ -302,7 +301,6 @@ registries = ['{{.Host}}:{{.Port}}']` } lock := GetPortLock(registryEndpoints[6].Port) defer lock.Unlock() - podmanTest.RestoreArtifact(registry) registry := podmanTest.Podman([]string{"run", "-d", "-p", fmt.Sprintf("%s:5000", registryEndpoints[6].Port), "--name", "registry6", registry}) registry.WaitWithDefaultTimeout() @@ -312,8 +310,9 @@ registries = ['{{.Host}}:{{.Port}}']` Skip("Can not start docker registry.") } + podmanTest.RestoreArtifact(ALPINE) image := fmt.Sprintf("%s/my-alpine", registryEndpoints[6].Address()) - push := podmanTest.Podman([]string{"push", "--tls-verify=false", "--remove-signatures", ALPINE, image}) + push := podmanTest.PodmanNoCache([]string{"push", "--tls-verify=false", "--remove-signatures", ALPINE, image}) push.WaitWithDefaultTimeout() Expect(push.ExitCode()).To(Equal(0)) @@ -322,7 +321,7 @@ registries = ['{{.Host}}:{{.Port}}']` podmanTest.setRegistriesConfigEnv(buffer.Bytes()) ioutil.WriteFile(fmt.Sprintf("%s/registry6.conf", tempdir), buffer.Bytes(), 0644) - search := podmanTest.Podman([]string{"search", image}) + search := podmanTest.PodmanNoCache([]string{"search", image}) search.WaitWithDefaultTimeout() Expect(search.ExitCode()).To(Equal(0)) @@ -343,7 +342,6 @@ registries = ['{{.Host}}:{{.Port}}']` lock8 := GetPortLock("6000") defer lock8.Unlock() - podmanTest.RestoreArtifact(registry) registryLocal := podmanTest.Podman([]string{"run", "-d", "--net=host", "-p", fmt.Sprintf("%s:5000", registryEndpoints[7].Port), "--name", "registry7", registry}) registryLocal.WaitWithDefaultTimeout() @@ -361,7 +359,8 @@ registries = ['{{.Host}}:{{.Port}}']` Skip("Can not start docker registry.") } - push := podmanTest.Podman([]string{"push", "--tls-verify=false", "--remove-signatures", ALPINE, "localhost:6000/my-alpine"}) + podmanTest.RestoreArtifact(ALPINE) + push := podmanTest.PodmanNoCache([]string{"push", "--tls-verify=false", "--remove-signatures", ALPINE, "localhost:6000/my-alpine"}) push.WaitWithDefaultTimeout() Expect(push.ExitCode()).To(Equal(0)) @@ -371,7 +370,7 @@ registries = ['{{.Host}}:{{.Port}}']` podmanTest.setRegistriesConfigEnv(buffer.Bytes()) ioutil.WriteFile(fmt.Sprintf("%s/registry8.conf", tempdir), buffer.Bytes(), 0644) - search := podmanTest.Podman([]string{"search", "my-alpine"}) + search := podmanTest.PodmanNoCache([]string{"search", "my-alpine"}) search.WaitWithDefaultTimeout() Expect(search.ExitCode()).To(Equal(0)) diff --git a/test/e2e/start_test.go b/test/e2e/start_test.go index c92da9777..fc1203ed1 100644 --- a/test/e2e/start_test.go +++ b/test/e2e/start_test.go @@ -22,7 +22,7 @@ var _ = Describe("Podman start", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() }) AfterEach(func() { diff --git a/test/e2e/stats_test.go b/test/e2e/stats_test.go index 05f24539f..45511edb8 100644 --- a/test/e2e/stats_test.go +++ b/test/e2e/stats_test.go @@ -26,7 +26,7 @@ var _ = Describe("Podman stats", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() }) AfterEach(func() { diff --git a/test/e2e/stop_test.go b/test/e2e/stop_test.go index e201204df..77ab6dbb0 100644 --- a/test/e2e/stop_test.go +++ b/test/e2e/stop_test.go @@ -23,7 +23,7 @@ var _ = Describe("Podman stop", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() }) AfterEach(func() { diff --git a/test/e2e/system_df_test.go b/test/e2e/system_df_test.go index 92787f17c..bbbdf30b0 100644 --- a/test/e2e/system_df_test.go +++ b/test/e2e/system_df_test.go @@ -25,7 +25,8 @@ var _ = Describe("podman system df", func() { os.Exit(1) } podmanTest = PodmanTestCreate(tempdir) - podmanTest.RestoreAllArtifacts() + podmanTest.Setup() + podmanTest.SeedImages() }) AfterEach(func() { @@ -55,7 +56,7 @@ var _ = Describe("podman system df", func() { images := strings.Fields(session.OutputToStringArray()[1]) containers := strings.Fields(session.OutputToStringArray()[2]) volumes := strings.Fields(session.OutputToStringArray()[3]) - Expect(images[1]).To(Equal("2")) + Expect(images[1]).To(Equal("9")) Expect(containers[1]).To(Equal("2")) Expect(volumes[2]).To(Equal("1")) }) diff --git a/test/e2e/systemd_test.go b/test/e2e/systemd_test.go index 52efc9fca..91604867d 100644 --- a/test/e2e/systemd_test.go +++ b/test/e2e/systemd_test.go @@ -27,7 +27,7 @@ var _ = Describe("Podman systemd", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() systemd_unit_file = `[Unit] Description=redis container [Service] diff --git a/test/e2e/tag_test.go b/test/e2e/tag_test.go index 26d6dfa75..2b513015b 100644 --- a/test/e2e/tag_test.go +++ b/test/e2e/tag_test.go @@ -33,11 +33,11 @@ var _ = Describe("Podman tag", func() { }) It("podman tag shortname:latest", func() { - session := podmanTest.Podman([]string{"tag", ALPINE, "foobar:latest"}) + session := podmanTest.PodmanNoCache([]string{"tag", ALPINE, "foobar:latest"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) - results := podmanTest.Podman([]string{"inspect", "foobar:latest"}) + results := podmanTest.PodmanNoCache([]string{"inspect", "foobar:latest"}) results.WaitWithDefaultTimeout() Expect(results.ExitCode()).To(Equal(0)) inspectData := results.InspectImageJSON() @@ -46,11 +46,11 @@ var _ = Describe("Podman tag", func() { }) It("podman tag shortname", func() { - session := podmanTest.Podman([]string{"tag", ALPINE, "foobar"}) + session := podmanTest.PodmanNoCache([]string{"tag", ALPINE, "foobar"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) - results := podmanTest.Podman([]string{"inspect", "foobar:latest"}) + results := podmanTest.PodmanNoCache([]string{"inspect", "foobar:latest"}) results.WaitWithDefaultTimeout() Expect(results.ExitCode()).To(Equal(0)) inspectData := results.InspectImageJSON() @@ -59,11 +59,11 @@ var _ = Describe("Podman tag", func() { }) It("podman tag shortname:tag", func() { - session := podmanTest.Podman([]string{"tag", ALPINE, "foobar:new"}) + session := podmanTest.PodmanNoCache([]string{"tag", ALPINE, "foobar:new"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) - results := podmanTest.Podman([]string{"inspect", "foobar:new"}) + results := podmanTest.PodmanNoCache([]string{"inspect", "foobar:new"}) results.WaitWithDefaultTimeout() Expect(results.ExitCode()).To(Equal(0)) inspectData := results.InspectImageJSON() @@ -72,15 +72,15 @@ var _ = Describe("Podman tag", func() { }) It("podman tag shortname image no tag", func() { - session := podmanTest.Podman([]string{"tag", ALPINE, "foobar"}) + session := podmanTest.PodmanNoCache([]string{"tag", ALPINE, "foobar"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) - results := podmanTest.Podman([]string{"tag", "foobar", "barfoo"}) + results := podmanTest.PodmanNoCache([]string{"tag", "foobar", "barfoo"}) results.WaitWithDefaultTimeout() Expect(results.ExitCode()).To(Equal(0)) - verify := podmanTest.Podman([]string{"inspect", "barfoo"}) + verify := podmanTest.PodmanNoCache([]string{"inspect", "barfoo"}) verify.WaitWithDefaultTimeout() Expect(verify.ExitCode()).To(Equal(0)) }) diff --git a/test/e2e/top_test.go b/test/e2e/top_test.go index 4c2cdb7b5..b62d242f1 100644 --- a/test/e2e/top_test.go +++ b/test/e2e/top_test.go @@ -24,7 +24,7 @@ var _ = Describe("Podman top", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() }) AfterEach(func() { diff --git a/test/e2e/tree_test.go b/test/e2e/tree_test.go index 9740adada..2db7aeb5e 100644 --- a/test/e2e/tree_test.go +++ b/test/e2e/tree_test.go @@ -22,7 +22,8 @@ var _ = Describe("Podman image tree", func() { os.Exit(1) } podmanTest = PodmanTestCreate(tempdir) - podmanTest.RestoreAllArtifacts() + podmanTest.Setup() + podmanTest.RestoreArtifact(BB) }) AfterEach(func() { @@ -36,7 +37,7 @@ var _ = Describe("Podman image tree", func() { if podmanTest.RemoteTest { Skip("Does not work on remote client") } - session := podmanTest.Podman([]string{"pull", "docker.io/library/busybox:latest"}) + session := podmanTest.PodmanNoCache([]string{"pull", "docker.io/library/busybox:latest"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) @@ -47,17 +48,17 @@ ENV foo=bar ` podmanTest.BuildImage(dockerfile, "test:latest", "true") - session = podmanTest.Podman([]string{"image", "tree", "test:latest"}) + session = podmanTest.PodmanNoCache([]string{"image", "tree", "test:latest"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) - session = podmanTest.Podman([]string{"image", "tree", "--whatrequires", "docker.io/library/busybox:latest"}) + session = podmanTest.PodmanNoCache([]string{"image", "tree", "--whatrequires", "docker.io/library/busybox:latest"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) - session = podmanTest.Podman([]string{"rmi", "test:latest"}) + session = podmanTest.PodmanNoCache([]string{"rmi", "test:latest"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) - session = podmanTest.Podman([]string{"rmi", "docker.io/library/busybox:latest"}) + session = podmanTest.PodmanNoCache([]string{"rmi", "docker.io/library/busybox:latest"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) }) diff --git a/test/e2e/trust_test.go b/test/e2e/trust_test.go index 493c4a7d5..8c97e6b28 100644 --- a/test/e2e/trust_test.go +++ b/test/e2e/trust_test.go @@ -27,7 +27,7 @@ var _ = Describe("Podman trust", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() }) AfterEach(func() { diff --git a/test/e2e/unshare_test.go b/test/e2e/unshare_test.go index 1e3f06a62..5f342ffc3 100644 --- a/test/e2e/unshare_test.go +++ b/test/e2e/unshare_test.go @@ -32,7 +32,7 @@ var _ = Describe("Podman unshare", func() { podmanTest.CgroupManager = "cgroupfs" podmanTest.StorageOptions = ROOTLESS_STORAGE_OPTIONS podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() }) AfterEach(func() { diff --git a/test/e2e/version_test.go b/test/e2e/version_test.go index 35ee21e49..2c8b3068c 100644 --- a/test/e2e/version_test.go +++ b/test/e2e/version_test.go @@ -27,6 +27,7 @@ var _ = Describe("Podman version", func() { podmanTest.Cleanup() f := CurrentGinkgoTestDescription() processTestResult(f) + podmanTest.SeedImages() }) diff --git a/test/e2e/volume_create_test.go b/test/e2e/volume_create_test.go index dccecd457..041a9e6f0 100644 --- a/test/e2e/volume_create_test.go +++ b/test/e2e/volume_create_test.go @@ -22,7 +22,7 @@ var _ = Describe("Podman volume create", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() }) AfterEach(func() { diff --git a/test/e2e/volume_inspect_test.go b/test/e2e/volume_inspect_test.go index e7f20ce7b..0683c6bbf 100644 --- a/test/e2e/volume_inspect_test.go +++ b/test/e2e/volume_inspect_test.go @@ -22,7 +22,7 @@ var _ = Describe("Podman volume inspect", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() }) AfterEach(func() { diff --git a/test/e2e/volume_ls_test.go b/test/e2e/volume_ls_test.go index 1f0177def..da2d7ae77 100644 --- a/test/e2e/volume_ls_test.go +++ b/test/e2e/volume_ls_test.go @@ -22,7 +22,7 @@ var _ = Describe("Podman volume ls", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() }) AfterEach(func() { diff --git a/test/e2e/volume_prune_test.go b/test/e2e/volume_prune_test.go index 55a95c8c9..ba249278b 100644 --- a/test/e2e/volume_prune_test.go +++ b/test/e2e/volume_prune_test.go @@ -24,7 +24,7 @@ var _ = Describe("Podman volume prune", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() }) AfterEach(func() { diff --git a/test/e2e/volume_rm_test.go b/test/e2e/volume_rm_test.go index 39628d56f..5dcf51ccd 100644 --- a/test/e2e/volume_rm_test.go +++ b/test/e2e/volume_rm_test.go @@ -22,7 +22,7 @@ var _ = Describe("Podman volume rm", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() }) AfterEach(func() { diff --git a/test/e2e/wait_test.go b/test/e2e/wait_test.go index 28a4c1e40..c03116b01 100644 --- a/test/e2e/wait_test.go +++ b/test/e2e/wait_test.go @@ -22,7 +22,7 @@ var _ = Describe("Podman wait", func() { } podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() - podmanTest.RestoreAllArtifacts() + podmanTest.SeedImages() }) AfterEach(func() { diff --git a/test/utils/podmantest_test.go b/test/utils/podmantest_test.go index 28f294a94..cb31d5548 100644 --- a/test/utils/podmantest_test.go +++ b/test/utils/podmantest_test.go @@ -23,7 +23,7 @@ var _ = Describe("PodmanTest test", func() { FakeOutputs["check"] = []string{"check"} os.Setenv("HOOK_OPTION", "hook_option") env := os.Environ() - session := podmanTest.PodmanAsUserBase([]string{"check"}, 1000, 1000, "", env) + session := podmanTest.PodmanAsUserBase([]string{"check"}, 1000, 1000, "", env, true) os.Unsetenv("HOOK_OPTION") session.WaitWithDefaultTimeout() Expect(session.Command.Process).ShouldNot(BeNil()) diff --git a/test/utils/utils.go b/test/utils/utils.go index beadab549..98031385d 100644 --- a/test/utils/utils.go +++ b/test/utils/utils.go @@ -42,6 +42,8 @@ type PodmanTest struct { VarlinkSession *os.Process VarlinkEndpoint string VarlinkCommand *exec.Cmd + ImageCacheDir string + ImageCacheFS string } // PodmanSession wraps the gexec.session so we can extend it @@ -63,7 +65,7 @@ func (p *PodmanTest) MakeOptions(args []string) []string { // PodmanAsUserBase exec podman as user. uid and gid is set for credentials useage. env is used // to record the env for debugging -func (p *PodmanTest) PodmanAsUserBase(args []string, uid, gid uint32, cwd string, env []string) *PodmanSession { +func (p *PodmanTest) PodmanAsUserBase(args []string, uid, gid uint32, cwd string, env []string, nocache bool) *PodmanSession { var command *exec.Cmd podmanOptions := p.MakeOptions(args) podmanBinary := p.PodmanBinary @@ -71,6 +73,10 @@ func (p *PodmanTest) PodmanAsUserBase(args []string, uid, gid uint32, cwd string podmanBinary = p.RemotePodmanBinary env = append(env, fmt.Sprintf("PODMAN_VARLINK_ADDRESS=%s", p.VarlinkEndpoint)) } + if !nocache && !p.RemoteTest { + cacheOptions := []string{"--storage-opt", fmt.Sprintf("%s.imagestore=%s", p.ImageCacheFS, p.ImageCacheDir)} + podmanOptions = append(cacheOptions, podmanOptions...) + } if env == nil { fmt.Printf("Running: %s %s\n", podmanBinary, strings.Join(podmanOptions, " ")) @@ -99,8 +105,8 @@ func (p *PodmanTest) PodmanAsUserBase(args []string, uid, gid uint32, cwd string } // PodmanBase exec podman with default env. -func (p *PodmanTest) PodmanBase(args []string) *PodmanSession { - return p.PodmanAsUserBase(args, 0, 0, "", nil) +func (p *PodmanTest) PodmanBase(args []string, nocache bool) *PodmanSession { + return p.PodmanAsUserBase(args, 0, 0, "", nil, nocache) } // WaitForContainer waits on a started container @@ -118,7 +124,7 @@ func (p *PodmanTest) WaitForContainer() bool { // containers are currently running. func (p *PodmanTest) NumberOfContainersRunning() int { var containers []string - ps := p.PodmanBase([]string{"ps", "-q"}) + ps := p.PodmanBase([]string{"ps", "-q"}, true) ps.WaitWithDefaultTimeout() Expect(ps.ExitCode()).To(Equal(0)) for _, i := range ps.OutputToStringArray() { @@ -133,7 +139,7 @@ func (p *PodmanTest) NumberOfContainersRunning() int { // containers are currently defined. func (p *PodmanTest) NumberOfContainers() int { var containers []string - ps := p.PodmanBase([]string{"ps", "-aq"}) + ps := p.PodmanBase([]string{"ps", "-aq"}, true) ps.WaitWithDefaultTimeout() Expect(ps.ExitCode()).To(Equal(0)) for _, i := range ps.OutputToStringArray() { @@ -148,7 +154,7 @@ func (p *PodmanTest) NumberOfContainers() int { // pods are currently defined. func (p *PodmanTest) NumberOfPods() int { var pods []string - ps := p.PodmanBase([]string{"pod", "ps", "-q"}) + ps := p.PodmanBase([]string{"pod", "ps", "-q"}, true) ps.WaitWithDefaultTimeout() Expect(ps.ExitCode()).To(Equal(0)) for _, i := range ps.OutputToStringArray() { @@ -164,7 +170,7 @@ func (p *PodmanTest) NumberOfPods() int { func (p *PodmanTest) GetContainerStatus() string { var podmanArgs = []string{"ps"} podmanArgs = append(podmanArgs, "--all", "--format={{.Status}}") - session := p.PodmanBase(podmanArgs) + session := p.PodmanBase(podmanArgs, true) session.WaitWithDefaultTimeout() return session.OutputToString() } @@ -172,7 +178,7 @@ func (p *PodmanTest) GetContainerStatus() string { // WaitContainerReady waits process or service inside container start, and ready to be used. func (p *PodmanTest) WaitContainerReady(id string, expStr string, timeout int, step int) bool { startTime := time.Now() - s := p.PodmanBase([]string{"logs", id}) + s := p.PodmanBase([]string{"logs", id}, true) s.WaitWithDefaultTimeout() for { @@ -185,7 +191,7 @@ func (p *PodmanTest) WaitContainerReady(id string, expStr string, timeout int, s return true } time.Sleep(time.Duration(step) * time.Second) - s = p.PodmanBase([]string{"logs", id}) + s = p.PodmanBase([]string{"logs", id}, true) s.WaitWithDefaultTimeout() } } diff --git a/transfer.md b/transfer.md index 79b6d3461..7cfd5a85d 100644 --- a/transfer.md +++ b/transfer.md @@ -91,10 +91,11 @@ Those Docker commands currently do not have equivalents in `podman`: | Missing command | Description| | :--- | :--- | +| `docker container update` | podman does not support altering running containers. We recommend recreating containers with the correct arguments.| +| `docker container rename` | podman does not support `container rename` - or the `rename` shorthand. We recommend using `podman rm` and `podman create` to create a container with a specific name.| | `docker network` || | `docker node` || | `docker plugin` | podman does not support plugins. We recommend you use alternative OCI Runtimes or OCI Runtime Hooks to alter behavior of podman.| -| `docker rename` | podman does not support rename, you need to use `podman rm` and `podman create` to rename a container.| | `docker secret` || | `docker service` || | `docker stack` || diff --git a/vendor.conf b/vendor.conf index 5c41d6908..a33a52786 100644 --- a/vendor.conf +++ b/vendor.conf @@ -3,7 +3,7 @@ # # TODO: no release, can we find an alternative? github.com/Azure/go-ansiterm d6e3b3328b783f23731bc4d058875b0371ff8109 -github.com/BurntSushi/toml v0.2.0 +github.com/BurntSushi/toml v0.3.1 github.com/Microsoft/go-winio v0.4.11 github.com/Microsoft/hcsshim v0.8.3 github.com/blang/semver v3.5.0 @@ -19,9 +19,9 @@ github.com/containers/image 2c0349c99af7d90694b3faa0e9bde404d407b145 github.com/vbauerster/mpb v3.3.4 github.com/mattn/go-isatty v0.0.4 github.com/VividCortex/ewma v1.1.1 -github.com/containers/storage v1.12.7 +github.com/containers/storage 9b10041d7b2ef767ce9c42b5862b6c51eeb82214 github.com/containers/psgo v1.3.0 -github.com/coreos/go-systemd v14 +github.com/coreos/go-systemd v17 github.com/coreos/pkg v4 github.com/cri-o/ocicni 0c180f981b27ef6036fa5be29bcb4dd666e406eb github.com/cyphar/filepath-securejoin v0.2.1 @@ -93,7 +93,7 @@ k8s.io/api kubernetes-1.10.13-beta.0 https://github.com/kubernetes/api k8s.io/apimachinery kubernetes-1.10.13-beta.0 https://github.com/kubernetes/apimachinery k8s.io/client-go kubernetes-1.10.13-beta.0 https://github.com/kubernetes/client-go github.com/mrunalp/fileutils 7d4729fb36185a7c1719923406c9d40e54fb93c7 -github.com/containers/buildah bcc5e51a948c1847fab1c932f1a343696a96cafd +github.com/containers/buildah v1.8.3 github.com/varlink/go 0f1d566d194b9d6d48e0d47c5e4d822628919066 # TODO: Gotty has not been updated since 2012. Can we find replacement? github.com/Nvveen/Gotty cd527374f1e5bff4938207604a14f2e38a9cf512 diff --git a/vendor/github.com/BurntSushi/toml/COPYING b/vendor/github.com/BurntSushi/toml/COPYING index 5a8e33254..01b574320 100644 --- a/vendor/github.com/BurntSushi/toml/COPYING +++ b/vendor/github.com/BurntSushi/toml/COPYING @@ -1,14 +1,21 @@ - DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE - Version 2, December 2004 +The MIT License (MIT) - Copyright (C) 2004 Sam Hocevar <sam@hocevar.net> +Copyright (c) 2013 TOML authors - Everyone is permitted to copy and distribute verbatim or modified - copies of this license document, and changing it is allowed as long - as the name is changed. +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: - DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. You just DO WHAT THE FUCK YOU WANT TO. +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/vendor/github.com/BurntSushi/toml/README.md b/vendor/github.com/BurntSushi/toml/README.md index 5a5df6370..7c1b37ecc 100644 --- a/vendor/github.com/BurntSushi/toml/README.md +++ b/vendor/github.com/BurntSushi/toml/README.md @@ -1,17 +1,17 @@ ## TOML parser and encoder for Go with reflection TOML stands for Tom's Obvious, Minimal Language. This Go package provides a -reflection interface similar to Go's standard library `json` and `xml` +reflection interface similar to Go's standard library `json` and `xml` packages. This package also supports the `encoding.TextUnmarshaler` and -`encoding.TextMarshaler` interfaces so that you can define custom data +`encoding.TextMarshaler` interfaces so that you can define custom data representations. (There is an example of this below.) -Spec: https://github.com/mojombo/toml +Spec: https://github.com/toml-lang/toml Compatible with TOML version -[v0.2.0](https://github.com/toml-lang/toml/blob/master/versions/en/toml-v0.2.0.md) +[v0.4.0](https://github.com/toml-lang/toml/blob/master/versions/en/toml-v0.4.0.md) -Documentation: http://godoc.org/github.com/BurntSushi/toml +Documentation: https://godoc.org/github.com/BurntSushi/toml Installation: @@ -26,8 +26,7 @@ go get github.com/BurntSushi/toml/cmd/tomlv tomlv some-toml-file.toml ``` -[](https://travis-ci.org/BurntSushi/toml) - +[](https://travis-ci.org/BurntSushi/toml) [](https://godoc.org/github.com/BurntSushi/toml) ### Testing @@ -87,7 +86,7 @@ type TOML struct { ### Using the `encoding.TextUnmarshaler` interface -Here's an example that automatically parses duration strings into +Here's an example that automatically parses duration strings into `time.Duration` values: ```toml @@ -120,7 +119,7 @@ for _, s := range favorites.Song { } ``` -And you'll also need a `duration` type that satisfies the +And you'll also need a `duration` type that satisfies the `encoding.TextUnmarshaler` interface: ```go @@ -217,4 +216,3 @@ Note that a case insensitive match will be tried if an exact match can't be found. A working example of the above can be found in `_examples/example.{go,toml}`. - diff --git a/vendor/github.com/BurntSushi/toml/decode.go b/vendor/github.com/BurntSushi/toml/decode.go index c26b00c01..b0fd51d5b 100644 --- a/vendor/github.com/BurntSushi/toml/decode.go +++ b/vendor/github.com/BurntSushi/toml/decode.go @@ -10,7 +10,9 @@ import ( "time" ) -var e = fmt.Errorf +func e(format string, args ...interface{}) error { + return fmt.Errorf("toml: "+format, args...) +} // Unmarshaler is the interface implemented by objects that can unmarshal a // TOML description of themselves. @@ -103,6 +105,13 @@ func (md *MetaData) PrimitiveDecode(primValue Primitive, v interface{}) error { // This decoder will not handle cyclic types. If a cyclic type is passed, // `Decode` will not terminate. func Decode(data string, v interface{}) (MetaData, error) { + rv := reflect.ValueOf(v) + if rv.Kind() != reflect.Ptr { + return MetaData{}, e("Decode of non-pointer %s", reflect.TypeOf(v)) + } + if rv.IsNil() { + return MetaData{}, e("Decode of nil %s", reflect.TypeOf(v)) + } p, err := parse(data) if err != nil { return MetaData{}, err @@ -111,7 +120,7 @@ func Decode(data string, v interface{}) (MetaData, error) { p.mapping, p.types, p.ordered, make(map[string]bool, len(p.ordered)), nil, } - return md, md.unify(p.mapping, rvalue(v)) + return md, md.unify(p.mapping, indirect(rv)) } // DecodeFile is just like Decode, except it will automatically read the @@ -211,7 +220,7 @@ func (md *MetaData) unify(data interface{}, rv reflect.Value) error { case reflect.Interface: // we only support empty interfaces. if rv.NumMethod() > 0 { - return e("Unsupported type '%s'.", rv.Kind()) + return e("unsupported type %s", rv.Type()) } return md.unifyAnything(data, rv) case reflect.Float32: @@ -219,7 +228,7 @@ func (md *MetaData) unify(data interface{}, rv reflect.Value) error { case reflect.Float64: return md.unifyFloat64(data, rv) } - return e("Unsupported type '%s'.", rv.Kind()) + return e("unsupported type %s", rv.Kind()) } func (md *MetaData) unifyStruct(mapping interface{}, rv reflect.Value) error { @@ -228,7 +237,8 @@ func (md *MetaData) unifyStruct(mapping interface{}, rv reflect.Value) error { if mapping == nil { return nil } - return mismatch(rv, "map", mapping) + return e("type mismatch for %s: expected table but found %T", + rv.Type().String(), mapping) } for key, datum := range tmap { @@ -253,14 +263,13 @@ func (md *MetaData) unifyStruct(mapping interface{}, rv reflect.Value) error { md.decoded[md.context.add(key).String()] = true md.context = append(md.context, key) if err := md.unify(datum, subv); err != nil { - return e("Type mismatch for '%s.%s': %s", - rv.Type().String(), f.name, err) + return err } md.context = md.context[0 : len(md.context)-1] } else if f.name != "" { // Bad user! No soup for you! - return e("Field '%s.%s' is unexported, and therefore cannot "+ - "be loaded with reflection.", rv.Type().String(), f.name) + return e("cannot write unexported field %s.%s", + rv.Type().String(), f.name) } } } @@ -378,15 +387,15 @@ func (md *MetaData) unifyInt(data interface{}, rv reflect.Value) error { // No bounds checking necessary. case reflect.Int8: if num < math.MinInt8 || num > math.MaxInt8 { - return e("Value '%d' is out of range for int8.", num) + return e("value %d is out of range for int8", num) } case reflect.Int16: if num < math.MinInt16 || num > math.MaxInt16 { - return e("Value '%d' is out of range for int16.", num) + return e("value %d is out of range for int16", num) } case reflect.Int32: if num < math.MinInt32 || num > math.MaxInt32 { - return e("Value '%d' is out of range for int32.", num) + return e("value %d is out of range for int32", num) } } rv.SetInt(num) @@ -397,15 +406,15 @@ func (md *MetaData) unifyInt(data interface{}, rv reflect.Value) error { // No bounds checking necessary. case reflect.Uint8: if num < 0 || unum > math.MaxUint8 { - return e("Value '%d' is out of range for uint8.", num) + return e("value %d is out of range for uint8", num) } case reflect.Uint16: if num < 0 || unum > math.MaxUint16 { - return e("Value '%d' is out of range for uint16.", num) + return e("value %d is out of range for uint16", num) } case reflect.Uint32: if num < 0 || unum > math.MaxUint32 { - return e("Value '%d' is out of range for uint32.", num) + return e("value %d is out of range for uint32", num) } } rv.SetUint(unum) @@ -471,7 +480,7 @@ func rvalue(v interface{}) reflect.Value { // interest to us (like encoding.TextUnmarshaler). func indirect(v reflect.Value) reflect.Value { if v.Kind() != reflect.Ptr { - if v.CanAddr() { + if v.CanSet() { pv := v.Addr() if _, ok := pv.Interface().(TextUnmarshaler); ok { return pv @@ -496,10 +505,5 @@ func isUnifiable(rv reflect.Value) bool { } func badtype(expected string, data interface{}) error { - return e("Expected %s but found '%T'.", expected, data) -} - -func mismatch(user reflect.Value, expected string, data interface{}) error { - return e("Type mismatch for %s. Expected %s but found '%T'.", - user.Type().String(), expected, data) + return e("cannot load TOML value of type %T into a Go %s", data, expected) } diff --git a/vendor/github.com/BurntSushi/toml/decode_meta.go b/vendor/github.com/BurntSushi/toml/decode_meta.go index ef6f545fa..b9914a679 100644 --- a/vendor/github.com/BurntSushi/toml/decode_meta.go +++ b/vendor/github.com/BurntSushi/toml/decode_meta.go @@ -77,9 +77,8 @@ func (k Key) maybeQuoted(i int) string { } if quote { return "\"" + strings.Replace(k[i], "\"", "\\\"", -1) + "\"" - } else { - return k[i] } + return k[i] } func (k Key) add(piece string) Key { diff --git a/vendor/github.com/BurntSushi/toml/doc.go b/vendor/github.com/BurntSushi/toml/doc.go index fe2680004..b371f396e 100644 --- a/vendor/github.com/BurntSushi/toml/doc.go +++ b/vendor/github.com/BurntSushi/toml/doc.go @@ -4,7 +4,7 @@ files via reflection. There is also support for delaying decoding with the Primitive type, and querying the set of keys in a TOML document with the MetaData type. -The specification implemented: https://github.com/mojombo/toml +The specification implemented: https://github.com/toml-lang/toml The sub-command github.com/BurntSushi/toml/cmd/tomlv can be used to verify whether a file is a valid TOML document. It can also be used to print the diff --git a/vendor/github.com/BurntSushi/toml/encode.go b/vendor/github.com/BurntSushi/toml/encode.go index 4e4c97aed..d905c21a2 100644 --- a/vendor/github.com/BurntSushi/toml/encode.go +++ b/vendor/github.com/BurntSushi/toml/encode.go @@ -16,17 +16,17 @@ type tomlEncodeError struct{ error } var ( errArrayMixedElementTypes = errors.New( - "can't encode array with mixed element types") + "toml: cannot encode array with mixed element types") errArrayNilElement = errors.New( - "can't encode array with nil element") + "toml: cannot encode array with nil element") errNonString = errors.New( - "can't encode a map with non-string key type") + "toml: cannot encode a map with non-string key type") errAnonNonStruct = errors.New( - "can't encode an anonymous field that is not a struct") + "toml: cannot encode an anonymous field that is not a struct") errArrayNoTable = errors.New( - "TOML array element can't contain a table") + "toml: TOML array element cannot contain a table") errNoKey = errors.New( - "top-level values must be a Go map or struct") + "toml: top-level values must be Go maps or structs") errAnything = errors.New("") // used in testing ) @@ -148,7 +148,7 @@ func (enc *Encoder) encode(key Key, rv reflect.Value) { case reflect.Struct: enc.eTable(key, rv) default: - panic(e("Unsupported type for key '%s': %s", key, k)) + panic(e("unsupported type for key '%s': %s", key, k)) } } @@ -160,7 +160,7 @@ func (enc *Encoder) eElement(rv reflect.Value) { // Special case time.Time as a primitive. Has to come before // TextMarshaler below because time.Time implements // encoding.TextMarshaler, but we need to always use UTC. - enc.wf(v.In(time.FixedZone("UTC", 0)).Format("2006-01-02T15:04:05Z")) + enc.wf(v.UTC().Format("2006-01-02T15:04:05Z")) return case TextMarshaler: // Special case. Use text marshaler if it's available for this value. @@ -191,7 +191,7 @@ func (enc *Encoder) eElement(rv reflect.Value) { case reflect.String: enc.writeQuoted(rv.String()) default: - panic(e("Unexpected primitive type: %s", rv.Kind())) + panic(e("unexpected primitive type: %s", rv.Kind())) } } @@ -241,7 +241,7 @@ func (enc *Encoder) eArrayOfTables(key Key, rv reflect.Value) { func (enc *Encoder) eTable(key Key, rv reflect.Value) { panicIfInvalidKey(key) if len(key) == 1 { - // Output an extra new line between top-level tables. + // Output an extra newline between top-level tables. // (The newline isn't written if nothing else has been written though.) enc.newline() } @@ -315,10 +315,16 @@ func (enc *Encoder) eStruct(key Key, rv reflect.Value) { t := f.Type switch t.Kind() { case reflect.Struct: - addFields(t, frv, f.Index) - continue + // Treat anonymous struct fields with + // tag names as though they are not + // anonymous, like encoding/json does. + if getOptions(f.Tag).name == "" { + addFields(t, frv, f.Index) + continue + } case reflect.Ptr: - if t.Elem().Kind() == reflect.Struct { + if t.Elem().Kind() == reflect.Struct && + getOptions(f.Tag).name == "" { if !frv.IsNil() { addFields(t.Elem(), frv.Elem(), f.Index) } @@ -347,17 +353,18 @@ func (enc *Encoder) eStruct(key Key, rv reflect.Value) { continue } - tag := sft.Tag.Get("toml") - if tag == "-" { + opts := getOptions(sft.Tag) + if opts.skip { continue } - keyName, opts := getOptions(tag) - if keyName == "" { - keyName = sft.Name + keyName := sft.Name + if opts.name != "" { + keyName = opts.name } - if _, ok := opts["omitempty"]; ok && isEmpty(sf) { + if opts.omitempty && isEmpty(sf) { continue - } else if _, ok := opts["omitzero"]; ok && isZero(sf) { + } + if opts.omitzero && isZero(sf) { continue } @@ -392,9 +399,8 @@ func tomlTypeOfGo(rv reflect.Value) tomlType { case reflect.Array, reflect.Slice: if typeEqual(tomlHash, tomlArrayType(rv)) { return tomlArrayHash - } else { - return tomlArray } + return tomlArray case reflect.Ptr, reflect.Interface: return tomlTypeOfGo(rv.Elem()) case reflect.String: @@ -451,17 +457,30 @@ func tomlArrayType(rv reflect.Value) tomlType { return firstType } -func getOptions(keyName string) (string, map[string]struct{}) { - opts := make(map[string]struct{}) - ss := strings.Split(keyName, ",") - name := ss[0] - if len(ss) > 1 { - for _, opt := range ss { - opts[opt] = struct{}{} +type tagOptions struct { + skip bool // "-" + name string + omitempty bool + omitzero bool +} + +func getOptions(tag reflect.StructTag) tagOptions { + t := tag.Get("toml") + if t == "-" { + return tagOptions{skip: true} + } + var opts tagOptions + parts := strings.Split(t, ",") + opts.name = parts[0] + for _, s := range parts[1:] { + switch s { + case "omitempty": + opts.omitempty = true + case "omitzero": + opts.omitzero = true } } - - return name, opts + return opts } func isZero(rv reflect.Value) bool { diff --git a/vendor/github.com/BurntSushi/toml/lex.go b/vendor/github.com/BurntSushi/toml/lex.go index 9b20b3a81..e0a742a88 100644 --- a/vendor/github.com/BurntSushi/toml/lex.go +++ b/vendor/github.com/BurntSushi/toml/lex.go @@ -3,6 +3,7 @@ package toml import ( "fmt" "strings" + "unicode" "unicode/utf8" ) @@ -29,24 +30,28 @@ const ( itemArrayTableEnd itemKeyStart itemCommentStart + itemInlineTableStart + itemInlineTableEnd ) const ( - eof = 0 - tableStart = '[' - tableEnd = ']' - arrayTableStart = '[' - arrayTableEnd = ']' - tableSep = '.' - keySep = '=' - arrayStart = '[' - arrayEnd = ']' - arrayValTerm = ',' - commentStart = '#' - stringStart = '"' - stringEnd = '"' - rawStringStart = '\'' - rawStringEnd = '\'' + eof = 0 + comma = ',' + tableStart = '[' + tableEnd = ']' + arrayTableStart = '[' + arrayTableEnd = ']' + tableSep = '.' + keySep = '=' + arrayStart = '[' + arrayEnd = ']' + commentStart = '#' + stringStart = '"' + stringEnd = '"' + rawStringStart = '\'' + rawStringEnd = '\'' + inlineTableStart = '{' + inlineTableEnd = '}' ) type stateFn func(lx *lexer) stateFn @@ -55,11 +60,18 @@ type lexer struct { input string start int pos int - width int line int state stateFn items chan item + // Allow for backing up up to three runes. + // This is necessary because TOML contains 3-rune tokens (""" and '''). + prevWidths [3]int + nprev int // how many of prevWidths are in use + // If we emit an eof, we can still back up, but it is not OK to call + // next again. + atEOF bool + // A stack of state functions used to maintain context. // The idea is to reuse parts of the state machine in various places. // For example, values can appear at the top level or within arbitrarily @@ -87,7 +99,7 @@ func (lx *lexer) nextItem() item { func lex(input string) *lexer { lx := &lexer{ - input: input + "\n", + input: input, state: lexTop, line: 1, items: make(chan item, 10), @@ -102,7 +114,7 @@ func (lx *lexer) push(state stateFn) { func (lx *lexer) pop() stateFn { if len(lx.stack) == 0 { - return lx.errorf("BUG in lexer: no states to pop.") + return lx.errorf("BUG in lexer: no states to pop") } last := lx.stack[len(lx.stack)-1] lx.stack = lx.stack[0 : len(lx.stack)-1] @@ -124,16 +136,25 @@ func (lx *lexer) emitTrim(typ itemType) { } func (lx *lexer) next() (r rune) { + if lx.atEOF { + panic("next called after EOF") + } if lx.pos >= len(lx.input) { - lx.width = 0 + lx.atEOF = true return eof } if lx.input[lx.pos] == '\n' { lx.line++ } - r, lx.width = utf8.DecodeRuneInString(lx.input[lx.pos:]) - lx.pos += lx.width + lx.prevWidths[2] = lx.prevWidths[1] + lx.prevWidths[1] = lx.prevWidths[0] + if lx.nprev < 3 { + lx.nprev++ + } + r, w := utf8.DecodeRuneInString(lx.input[lx.pos:]) + lx.prevWidths[0] = w + lx.pos += w return r } @@ -142,9 +163,20 @@ func (lx *lexer) ignore() { lx.start = lx.pos } -// backup steps back one rune. Can be called only once per call of next. +// backup steps back one rune. Can be called only twice between calls to next. func (lx *lexer) backup() { - lx.pos -= lx.width + if lx.atEOF { + lx.atEOF = false + return + } + if lx.nprev < 1 { + panic("backed up too far") + } + w := lx.prevWidths[0] + lx.prevWidths[0] = lx.prevWidths[1] + lx.prevWidths[1] = lx.prevWidths[2] + lx.nprev-- + lx.pos -= w if lx.pos < len(lx.input) && lx.input[lx.pos] == '\n' { lx.line-- } @@ -166,9 +198,22 @@ func (lx *lexer) peek() rune { return r } +// skip ignores all input that matches the given predicate. +func (lx *lexer) skip(pred func(rune) bool) { + for { + r := lx.next() + if pred(r) { + continue + } + lx.backup() + lx.ignore() + return + } +} + // errorf stops all lexing by emitting an error and returning `nil`. // Note that any value that is a character is escaped if it's a special -// character (new lines, tabs, etc.). +// character (newlines, tabs, etc.). func (lx *lexer) errorf(format string, values ...interface{}) stateFn { lx.items <- item{ itemError, @@ -184,7 +229,6 @@ func lexTop(lx *lexer) stateFn { if isWhitespace(r) || isNL(r) { return lexSkip(lx, lexTop) } - switch r { case commentStart: lx.push(lexTop) @@ -193,7 +237,7 @@ func lexTop(lx *lexer) stateFn { return lexTableStart case eof: if lx.pos > lx.start { - return lx.errorf("Unexpected EOF.") + return lx.errorf("unexpected EOF") } lx.emit(itemEOF) return nil @@ -208,12 +252,12 @@ func lexTop(lx *lexer) stateFn { // lexTopEnd is entered whenever a top-level item has been consumed. (A value // or a table.) It must see only whitespace, and will turn back to lexTop -// upon a new line. If it sees EOF, it will quit the lexer successfully. +// upon a newline. If it sees EOF, it will quit the lexer successfully. func lexTopEnd(lx *lexer) stateFn { r := lx.next() switch { case r == commentStart: - // a comment will read to a new line for us. + // a comment will read to a newline for us. lx.push(lexTop) return lexCommentStart case isWhitespace(r): @@ -222,11 +266,11 @@ func lexTopEnd(lx *lexer) stateFn { lx.ignore() return lexTop case r == eof: - lx.ignore() - return lexTop + lx.emit(itemEOF) + return nil } - return lx.errorf("Expected a top-level item to end with a new line, "+ - "comment or EOF, but got %q instead.", r) + return lx.errorf("expected a top-level item to end with a newline, "+ + "comment, or EOF, but got %q instead", r) } // lexTable lexes the beginning of a table. Namely, it makes sure that @@ -253,21 +297,22 @@ func lexTableEnd(lx *lexer) stateFn { func lexArrayTableEnd(lx *lexer) stateFn { if r := lx.next(); r != arrayTableEnd { - return lx.errorf("Expected end of table array name delimiter %q, "+ - "but got %q instead.", arrayTableEnd, r) + return lx.errorf("expected end of table array name delimiter %q, "+ + "but got %q instead", arrayTableEnd, r) } lx.emit(itemArrayTableEnd) return lexTopEnd } func lexTableNameStart(lx *lexer) stateFn { + lx.skip(isWhitespace) switch r := lx.peek(); { case r == tableEnd || r == eof: - return lx.errorf("Unexpected end of table name. (Table names cannot " + - "be empty.)") + return lx.errorf("unexpected end of table name " + + "(table names cannot be empty)") case r == tableSep: - return lx.errorf("Unexpected table separator. (Table names cannot " + - "be empty.)") + return lx.errorf("unexpected table separator " + + "(table names cannot be empty)") case r == stringStart || r == rawStringStart: lx.ignore() lx.push(lexTableNameEnd) @@ -277,24 +322,22 @@ func lexTableNameStart(lx *lexer) stateFn { } } -// lexTableName lexes the name of a table. It assumes that at least one +// lexBareTableName lexes the name of a table. It assumes that at least one // valid character for the table has already been read. func lexBareTableName(lx *lexer) stateFn { - switch r := lx.next(); { - case isBareKeyChar(r): + r := lx.next() + if isBareKeyChar(r) { return lexBareTableName - case r == tableSep || r == tableEnd: - lx.backup() - lx.emitTrim(itemText) - return lexTableNameEnd - default: - return lx.errorf("Bare keys cannot contain %q.", r) } + lx.backup() + lx.emit(itemText) + return lexTableNameEnd } // lexTableNameEnd reads the end of a piece of a table name, optionally // consuming whitespace. func lexTableNameEnd(lx *lexer) stateFn { + lx.skip(isWhitespace) switch r := lx.next(); { case isWhitespace(r): return lexTableNameEnd @@ -304,8 +347,8 @@ func lexTableNameEnd(lx *lexer) stateFn { case r == tableEnd: return lx.pop() default: - return lx.errorf("Expected '.' or ']' to end table name, but got %q "+ - "instead.", r) + return lx.errorf("expected '.' or ']' to end table name, "+ + "but got %q instead", r) } } @@ -315,7 +358,7 @@ func lexKeyStart(lx *lexer) stateFn { r := lx.peek() switch { case r == keySep: - return lx.errorf("Unexpected key separator %q.", keySep) + return lx.errorf("unexpected key separator %q", keySep) case isWhitespace(r) || isNL(r): lx.next() return lexSkip(lx, lexKeyStart) @@ -338,14 +381,15 @@ func lexBareKey(lx *lexer) stateFn { case isBareKeyChar(r): return lexBareKey case isWhitespace(r): - lx.emitTrim(itemText) + lx.backup() + lx.emit(itemText) return lexKeyEnd case r == keySep: lx.backup() - lx.emitTrim(itemText) + lx.emit(itemText) return lexKeyEnd default: - return lx.errorf("Bare keys cannot contain %q.", r) + return lx.errorf("bare keys cannot contain %q", r) } } @@ -358,7 +402,7 @@ func lexKeyEnd(lx *lexer) stateFn { case isWhitespace(r): return lexSkip(lx, lexKeyEnd) default: - return lx.errorf("Expected key separator %q, but got %q instead.", + return lx.errorf("expected key separator %q, but got %q instead", keySep, r) } } @@ -367,20 +411,26 @@ func lexKeyEnd(lx *lexer) stateFn { // lexValue will ignore whitespace. // After a value is lexed, the last state on the next is popped and returned. func lexValue(lx *lexer) stateFn { - // We allow whitespace to precede a value, but NOT new lines. - // In array syntax, the array states are responsible for ignoring new - // lines. + // We allow whitespace to precede a value, but NOT newlines. + // In array syntax, the array states are responsible for ignoring newlines. r := lx.next() - if isWhitespace(r) { + switch { + case isWhitespace(r): return lexSkip(lx, lexValue) + case isDigit(r): + lx.backup() // avoid an extra state and use the same as above + return lexNumberOrDateStart } - - switch { - case r == arrayStart: + switch r { + case arrayStart: lx.ignore() lx.emit(itemArray) return lexArrayValue - case r == stringStart: + case inlineTableStart: + lx.ignore() + lx.emit(itemInlineTableStart) + return lexInlineTableValue + case stringStart: if lx.accept(stringStart) { if lx.accept(stringStart) { lx.ignore() // Ignore """ @@ -390,7 +440,7 @@ func lexValue(lx *lexer) stateFn { } lx.ignore() // ignore the '"' return lexString - case r == rawStringStart: + case rawStringStart: if lx.accept(rawStringStart) { if lx.accept(rawStringStart) { lx.ignore() // Ignore """ @@ -400,23 +450,24 @@ func lexValue(lx *lexer) stateFn { } lx.ignore() // ignore the "'" return lexRawString - case r == 't': - return lexTrue - case r == 'f': - return lexFalse - case r == '-': + case '+', '-': return lexNumberStart - case isDigit(r): - lx.backup() // avoid an extra state and use the same as above - return lexNumberOrDateStart - case r == '.': // special error case, be kind to users - return lx.errorf("Floats must start with a digit, not '.'.") + case '.': // special error case, be kind to users + return lx.errorf("floats must start with a digit, not '.'") + } + if unicode.IsLetter(r) { + // Be permissive here; lexBool will give a nice error if the + // user wrote something like + // x = foo + // (i.e. not 'true' or 'false' but is something else word-like.) + lx.backup() + return lexBool } - return lx.errorf("Expected value but found %q instead.", r) + return lx.errorf("expected value but found %q instead", r) } // lexArrayValue consumes one value in an array. It assumes that '[' or ',' -// have already been consumed. All whitespace and new lines are ignored. +// have already been consumed. All whitespace and newlines are ignored. func lexArrayValue(lx *lexer) stateFn { r := lx.next() switch { @@ -425,10 +476,11 @@ func lexArrayValue(lx *lexer) stateFn { case r == commentStart: lx.push(lexArrayValue) return lexCommentStart - case r == arrayValTerm: - return lx.errorf("Unexpected array value terminator %q.", - arrayValTerm) + case r == comma: + return lx.errorf("unexpected comma") case r == arrayEnd: + // NOTE(caleb): The spec isn't clear about whether you can have + // a trailing comma or not, so we'll allow it. return lexArrayEnd } @@ -437,8 +489,9 @@ func lexArrayValue(lx *lexer) stateFn { return lexValue } -// lexArrayValueEnd consumes the cruft between values of an array. Namely, -// it ignores whitespace and expects either a ',' or a ']'. +// lexArrayValueEnd consumes everything between the end of an array value and +// the next value (or the end of the array): it ignores whitespace and newlines +// and expects either a ',' or a ']'. func lexArrayValueEnd(lx *lexer) stateFn { r := lx.next() switch { @@ -447,31 +500,88 @@ func lexArrayValueEnd(lx *lexer) stateFn { case r == commentStart: lx.push(lexArrayValueEnd) return lexCommentStart - case r == arrayValTerm: + case r == comma: lx.ignore() return lexArrayValue // move on to the next value case r == arrayEnd: return lexArrayEnd } - return lx.errorf("Expected an array value terminator %q or an array "+ - "terminator %q, but got %q instead.", arrayValTerm, arrayEnd, r) + return lx.errorf( + "expected a comma or array terminator %q, but got %q instead", + arrayEnd, r, + ) } -// lexArrayEnd finishes the lexing of an array. It assumes that a ']' has -// just been consumed. +// lexArrayEnd finishes the lexing of an array. +// It assumes that a ']' has just been consumed. func lexArrayEnd(lx *lexer) stateFn { lx.ignore() lx.emit(itemArrayEnd) return lx.pop() } +// lexInlineTableValue consumes one key/value pair in an inline table. +// It assumes that '{' or ',' have already been consumed. Whitespace is ignored. +func lexInlineTableValue(lx *lexer) stateFn { + r := lx.next() + switch { + case isWhitespace(r): + return lexSkip(lx, lexInlineTableValue) + case isNL(r): + return lx.errorf("newlines not allowed within inline tables") + case r == commentStart: + lx.push(lexInlineTableValue) + return lexCommentStart + case r == comma: + return lx.errorf("unexpected comma") + case r == inlineTableEnd: + return lexInlineTableEnd + } + lx.backup() + lx.push(lexInlineTableValueEnd) + return lexKeyStart +} + +// lexInlineTableValueEnd consumes everything between the end of an inline table +// key/value pair and the next pair (or the end of the table): +// it ignores whitespace and expects either a ',' or a '}'. +func lexInlineTableValueEnd(lx *lexer) stateFn { + r := lx.next() + switch { + case isWhitespace(r): + return lexSkip(lx, lexInlineTableValueEnd) + case isNL(r): + return lx.errorf("newlines not allowed within inline tables") + case r == commentStart: + lx.push(lexInlineTableValueEnd) + return lexCommentStart + case r == comma: + lx.ignore() + return lexInlineTableValue + case r == inlineTableEnd: + return lexInlineTableEnd + } + return lx.errorf("expected a comma or an inline table terminator %q, "+ + "but got %q instead", inlineTableEnd, r) +} + +// lexInlineTableEnd finishes the lexing of an inline table. +// It assumes that a '}' has just been consumed. +func lexInlineTableEnd(lx *lexer) stateFn { + lx.ignore() + lx.emit(itemInlineTableEnd) + return lx.pop() +} + // lexString consumes the inner contents of a string. It assumes that the // beginning '"' has already been consumed and ignored. func lexString(lx *lexer) stateFn { r := lx.next() switch { + case r == eof: + return lx.errorf("unexpected EOF") case isNL(r): - return lx.errorf("Strings cannot contain new lines.") + return lx.errorf("strings cannot contain newlines") case r == '\\': lx.push(lexString) return lexStringEscape @@ -488,11 +598,12 @@ func lexString(lx *lexer) stateFn { // lexMultilineString consumes the inner contents of a string. It assumes that // the beginning '"""' has already been consumed and ignored. func lexMultilineString(lx *lexer) stateFn { - r := lx.next() - switch { - case r == '\\': + switch lx.next() { + case eof: + return lx.errorf("unexpected EOF") + case '\\': return lexMultilineStringEscape - case r == stringEnd: + case stringEnd: if lx.accept(stringEnd) { if lx.accept(stringEnd) { lx.backup() @@ -516,8 +627,10 @@ func lexMultilineString(lx *lexer) stateFn { func lexRawString(lx *lexer) stateFn { r := lx.next() switch { + case r == eof: + return lx.errorf("unexpected EOF") case isNL(r): - return lx.errorf("Strings cannot contain new lines.") + return lx.errorf("strings cannot contain newlines") case r == rawStringEnd: lx.backup() lx.emit(itemRawString) @@ -529,12 +642,13 @@ func lexRawString(lx *lexer) stateFn { } // lexMultilineRawString consumes a raw string. Nothing can be escaped in such -// a string. It assumes that the beginning "'" has already been consumed and +// a string. It assumes that the beginning "'''" has already been consumed and // ignored. func lexMultilineRawString(lx *lexer) stateFn { - r := lx.next() - switch { - case r == rawStringEnd: + switch lx.next() { + case eof: + return lx.errorf("unexpected EOF") + case rawStringEnd: if lx.accept(rawStringEnd) { if lx.accept(rawStringEnd) { lx.backup() @@ -559,11 +673,10 @@ func lexMultilineStringEscape(lx *lexer) stateFn { // Handle the special case first: if isNL(lx.next()) { return lexMultilineString - } else { - lx.backup() - lx.push(lexMultilineString) - return lexStringEscape(lx) } + lx.backup() + lx.push(lexMultilineString) + return lexStringEscape(lx) } func lexStringEscape(lx *lexer) stateFn { @@ -588,10 +701,9 @@ func lexStringEscape(lx *lexer) stateFn { case 'U': return lexLongUnicodeEscape } - return lx.errorf("Invalid escape character %q. Only the following "+ + return lx.errorf("invalid escape character %q; only the following "+ "escape characters are allowed: "+ - "\\b, \\t, \\n, \\f, \\r, \\\", \\/, \\\\, "+ - "\\uXXXX and \\UXXXXXXXX.", r) + `\b, \t, \n, \f, \r, \", \\, \uXXXX, and \UXXXXXXXX`, r) } func lexShortUnicodeEscape(lx *lexer) stateFn { @@ -599,8 +711,8 @@ func lexShortUnicodeEscape(lx *lexer) stateFn { for i := 0; i < 4; i++ { r = lx.next() if !isHexadecimal(r) { - return lx.errorf("Expected four hexadecimal digits after '\\u', "+ - "but got '%s' instead.", lx.current()) + return lx.errorf(`expected four hexadecimal digits after '\u', `+ + "but got %q instead", lx.current()) } } return lx.pop() @@ -611,40 +723,43 @@ func lexLongUnicodeEscape(lx *lexer) stateFn { for i := 0; i < 8; i++ { r = lx.next() if !isHexadecimal(r) { - return lx.errorf("Expected eight hexadecimal digits after '\\U', "+ - "but got '%s' instead.", lx.current()) + return lx.errorf(`expected eight hexadecimal digits after '\U', `+ + "but got %q instead", lx.current()) } } return lx.pop() } -// lexNumberOrDateStart consumes either a (positive) integer, float or -// datetime. It assumes that NO negative sign has been consumed. +// lexNumberOrDateStart consumes either an integer, a float, or datetime. func lexNumberOrDateStart(lx *lexer) stateFn { r := lx.next() - if !isDigit(r) { - if r == '.' { - return lx.errorf("Floats must start with a digit, not '.'.") - } else { - return lx.errorf("Expected a digit but got %q.", r) - } + if isDigit(r) { + return lexNumberOrDate } - return lexNumberOrDate + switch r { + case '_': + return lexNumber + case 'e', 'E': + return lexFloat + case '.': + return lx.errorf("floats must start with a digit, not '.'") + } + return lx.errorf("expected a digit but got %q", r) } -// lexNumberOrDate consumes either a (positive) integer, float or datetime. +// lexNumberOrDate consumes either an integer, float or datetime. func lexNumberOrDate(lx *lexer) stateFn { r := lx.next() - switch { - case r == '-': - if lx.pos-lx.start != 5 { - return lx.errorf("All ISO8601 dates must be in full Zulu form.") - } - return lexDateAfterYear - case isDigit(r): + if isDigit(r) { return lexNumberOrDate - case r == '.': - return lexFloatStart + } + switch r { + case '-': + return lexDatetime + case '_': + return lexNumber + case '.', 'e', 'E': + return lexFloat } lx.backup() @@ -652,46 +767,34 @@ func lexNumberOrDate(lx *lexer) stateFn { return lx.pop() } -// lexDateAfterYear consumes a full Zulu Datetime in ISO8601 format. -// It assumes that "YYYY-" has already been consumed. -func lexDateAfterYear(lx *lexer) stateFn { - formats := []rune{ - // digits are '0'. - // everything else is direct equality. - '0', '0', '-', '0', '0', - 'T', - '0', '0', ':', '0', '0', ':', '0', '0', - 'Z', +// lexDatetime consumes a Datetime, to a first approximation. +// The parser validates that it matches one of the accepted formats. +func lexDatetime(lx *lexer) stateFn { + r := lx.next() + if isDigit(r) { + return lexDatetime } - for _, f := range formats { - r := lx.next() - if f == '0' { - if !isDigit(r) { - return lx.errorf("Expected digit in ISO8601 datetime, "+ - "but found %q instead.", r) - } - } else if f != r { - return lx.errorf("Expected %q in ISO8601 datetime, "+ - "but found %q instead.", f, r) - } + switch r { + case '-', 'T', ':', '.', 'Z', '+': + return lexDatetime } + + lx.backup() lx.emit(itemDatetime) return lx.pop() } -// lexNumberStart consumes either an integer or a float. It assumes that -// a negative sign has already been read, but that *no* digits have been -// consumed. lexNumberStart will move to the appropriate integer or float -// states. +// lexNumberStart consumes either an integer or a float. It assumes that a sign +// has already been read, but that *no* digits have been consumed. +// lexNumberStart will move to the appropriate integer or float states. func lexNumberStart(lx *lexer) stateFn { - // we MUST see a digit. Even floats have to start with a digit. + // We MUST see a digit. Even floats have to start with a digit. r := lx.next() if !isDigit(r) { if r == '.' { - return lx.errorf("Floats must start with a digit, not '.'.") - } else { - return lx.errorf("Expected a digit but got %q.", r) + return lx.errorf("floats must start with a digit, not '.'") } + return lx.errorf("expected a digit but got %q", r) } return lexNumber } @@ -699,11 +802,14 @@ func lexNumberStart(lx *lexer) stateFn { // lexNumber consumes an integer or a float after seeing the first digit. func lexNumber(lx *lexer) stateFn { r := lx.next() - switch { - case isDigit(r): + if isDigit(r) { return lexNumber - case r == '.': - return lexFloatStart + } + switch r { + case '_': + return lexNumber + case '.', 'e', 'E': + return lexFloat } lx.backup() @@ -711,60 +817,42 @@ func lexNumber(lx *lexer) stateFn { return lx.pop() } -// lexFloatStart starts the consumption of digits of a float after a '.'. -// Namely, at least one digit is required. -func lexFloatStart(lx *lexer) stateFn { - r := lx.next() - if !isDigit(r) { - return lx.errorf("Floats must have a digit after the '.', but got "+ - "%q instead.", r) - } - return lexFloat -} - -// lexFloat consumes the digits of a float after a '.'. -// Assumes that one digit has been consumed after a '.' already. +// lexFloat consumes the elements of a float. It allows any sequence of +// float-like characters, so floats emitted by the lexer are only a first +// approximation and must be validated by the parser. func lexFloat(lx *lexer) stateFn { r := lx.next() if isDigit(r) { return lexFloat } + switch r { + case '_', '.', '-', '+', 'e', 'E': + return lexFloat + } lx.backup() lx.emit(itemFloat) return lx.pop() } -// lexConst consumes the s[1:] in s. It assumes that s[0] has already been -// consumed. -func lexConst(lx *lexer, s string) stateFn { - for i := range s[1:] { - if r := lx.next(); r != rune(s[i+1]) { - return lx.errorf("Expected %q, but found %q instead.", s[:i+1], - s[:i]+string(r)) +// lexBool consumes a bool string: 'true' or 'false. +func lexBool(lx *lexer) stateFn { + var rs []rune + for { + r := lx.next() + if !unicode.IsLetter(r) { + lx.backup() + break } + rs = append(rs, r) } - return nil -} - -// lexTrue consumes the "rue" in "true". It assumes that 't' has already -// been consumed. -func lexTrue(lx *lexer) stateFn { - if fn := lexConst(lx, "true"); fn != nil { - return fn - } - lx.emit(itemBool) - return lx.pop() -} - -// lexFalse consumes the "alse" in "false". It assumes that 'f' has already -// been consumed. -func lexFalse(lx *lexer) stateFn { - if fn := lexConst(lx, "false"); fn != nil { - return fn + s := string(rs) + switch s { + case "true", "false": + lx.emit(itemBool) + return lx.pop() } - lx.emit(itemBool) - return lx.pop() + return lx.errorf("expected value but found %q instead", s) } // lexCommentStart begins the lexing of a comment. It will emit @@ -776,7 +864,7 @@ func lexCommentStart(lx *lexer) stateFn { } // lexComment lexes an entire comment. It assumes that '#' has been consumed. -// It will consume *up to* the first new line character, and pass control +// It will consume *up to* the first newline character, and pass control // back to the last state on the stack. func lexComment(lx *lexer) stateFn { r := lx.peek() @@ -834,13 +922,7 @@ func (itype itemType) String() string { return "EOF" case itemText: return "Text" - case itemString: - return "String" - case itemRawString: - return "String" - case itemMultilineString: - return "String" - case itemRawMultilineString: + case itemString, itemRawString, itemMultilineString, itemRawMultilineString: return "String" case itemBool: return "Bool" diff --git a/vendor/github.com/BurntSushi/toml/parse.go b/vendor/github.com/BurntSushi/toml/parse.go index 6a82e84f6..50869ef92 100644 --- a/vendor/github.com/BurntSushi/toml/parse.go +++ b/vendor/github.com/BurntSushi/toml/parse.go @@ -2,7 +2,6 @@ package toml import ( "fmt" - "log" "strconv" "strings" "time" @@ -81,7 +80,7 @@ func (p *parser) next() item { } func (p *parser) bug(format string, v ...interface{}) { - log.Panicf("BUG: %s\n\n", fmt.Sprintf(format, v...)) + panic(fmt.Sprintf("BUG: "+format+"\n\n", v...)) } func (p *parser) expect(typ itemType) item { @@ -179,10 +178,18 @@ func (p *parser) value(it item) (interface{}, tomlType) { } p.bug("Expected boolean value, but got '%s'.", it.val) case itemInteger: - num, err := strconv.ParseInt(it.val, 10, 64) + if !numUnderscoresOK(it.val) { + p.panicf("Invalid integer %q: underscores must be surrounded by digits", + it.val) + } + val := strings.Replace(it.val, "_", "", -1) + num, err := strconv.ParseInt(val, 10, 64) if err != nil { - // See comment below for floats describing why we make a - // distinction between a bug and a user error. + // Distinguish integer values. Normally, it'd be a bug if the lexer + // provides an invalid integer, but it's possible that the number is + // out of range of valid values (which the lexer cannot determine). + // So mark the former as a bug but the latter as a legitimate user + // error. if e, ok := err.(*strconv.NumError); ok && e.Err == strconv.ErrRange { @@ -194,29 +201,57 @@ func (p *parser) value(it item) (interface{}, tomlType) { } return num, p.typeOfPrimitive(it) case itemFloat: - num, err := strconv.ParseFloat(it.val, 64) + parts := strings.FieldsFunc(it.val, func(r rune) bool { + switch r { + case '.', 'e', 'E': + return true + } + return false + }) + for _, part := range parts { + if !numUnderscoresOK(part) { + p.panicf("Invalid float %q: underscores must be "+ + "surrounded by digits", it.val) + } + } + if !numPeriodsOK(it.val) { + // As a special case, numbers like '123.' or '1.e2', + // which are valid as far as Go/strconv are concerned, + // must be rejected because TOML says that a fractional + // part consists of '.' followed by 1+ digits. + p.panicf("Invalid float %q: '.' must be followed "+ + "by one or more digits", it.val) + } + val := strings.Replace(it.val, "_", "", -1) + num, err := strconv.ParseFloat(val, 64) if err != nil { - // Distinguish float values. Normally, it'd be a bug if the lexer - // provides an invalid float, but it's possible that the float is - // out of range of valid values (which the lexer cannot determine). - // So mark the former as a bug but the latter as a legitimate user - // error. - // - // This is also true for integers. if e, ok := err.(*strconv.NumError); ok && e.Err == strconv.ErrRange { p.panicf("Float '%s' is out of the range of 64-bit "+ "IEEE-754 floating-point numbers.", it.val) } else { - p.bug("Expected float value, but got '%s'.", it.val) + p.panicf("Invalid float value: %q", it.val) } } return num, p.typeOfPrimitive(it) case itemDatetime: - t, err := time.Parse("2006-01-02T15:04:05Z", it.val) - if err != nil { - p.panicf("Invalid RFC3339 Zulu DateTime: '%s'.", it.val) + var t time.Time + var ok bool + var err error + for _, format := range []string{ + "2006-01-02T15:04:05Z07:00", + "2006-01-02T15:04:05", + "2006-01-02", + } { + t, err = time.ParseInLocation(format, it.val, time.Local) + if err == nil { + ok = true + break + } + } + if !ok { + p.panicf("Invalid TOML Datetime: %q.", it.val) } return t, p.typeOfPrimitive(it) case itemArray: @@ -234,11 +269,75 @@ func (p *parser) value(it item) (interface{}, tomlType) { types = append(types, typ) } return array, p.typeOfArray(types) + case itemInlineTableStart: + var ( + hash = make(map[string]interface{}) + outerContext = p.context + outerKey = p.currentKey + ) + + p.context = append(p.context, p.currentKey) + p.currentKey = "" + for it := p.next(); it.typ != itemInlineTableEnd; it = p.next() { + if it.typ != itemKeyStart { + p.bug("Expected key start but instead found %q, around line %d", + it.val, p.approxLine) + } + if it.typ == itemCommentStart { + p.expect(itemText) + continue + } + + // retrieve key + k := p.next() + p.approxLine = k.line + kname := p.keyString(k) + + // retrieve value + p.currentKey = kname + val, typ := p.value(p.next()) + // make sure we keep metadata up to date + p.setType(kname, typ) + p.ordered = append(p.ordered, p.context.add(p.currentKey)) + hash[kname] = val + } + p.context = outerContext + p.currentKey = outerKey + return hash, tomlHash } p.bug("Unexpected value type: %s", it.typ) panic("unreachable") } +// numUnderscoresOK checks whether each underscore in s is surrounded by +// characters that are not underscores. +func numUnderscoresOK(s string) bool { + accept := false + for _, r := range s { + if r == '_' { + if !accept { + return false + } + accept = false + continue + } + accept = true + } + return accept +} + +// numPeriodsOK checks whether every period in s is followed by a digit. +func numPeriodsOK(s string) bool { + period := false + for _, r := range s { + if period && !isDigit(r) { + return false + } + period = r == '.' + } + return !period +} + // establishContext sets the current context of the parser, // where the context is either a hash or an array of hashes. Which one is // set depends on the value of the `array` parameter. diff --git a/vendor/github.com/BurntSushi/toml/type_fields.go b/vendor/github.com/BurntSushi/toml/type_fields.go index 6da608af4..608997c22 100644 --- a/vendor/github.com/BurntSushi/toml/type_fields.go +++ b/vendor/github.com/BurntSushi/toml/type_fields.go @@ -95,8 +95,8 @@ func typeFields(t reflect.Type) []field { if sf.PkgPath != "" && !sf.Anonymous { // unexported continue } - name, _ := getOptions(sf.Tag.Get("toml")) - if name == "-" { + opts := getOptions(sf.Tag) + if opts.skip { continue } index := make([]int, len(f.index)+1) @@ -110,8 +110,9 @@ func typeFields(t reflect.Type) []field { } // Record found field and index sequence. - if name != "" || !sf.Anonymous || ft.Kind() != reflect.Struct { - tagged := name != "" + if opts.name != "" || !sf.Anonymous || ft.Kind() != reflect.Struct { + tagged := opts.name != "" + name := opts.name if name == "" { name = sf.Name } diff --git a/vendor/github.com/containers/buildah/add.go b/vendor/github.com/containers/buildah/add.go index 589e090a8..11bfb6a12 100644 --- a/vendor/github.com/containers/buildah/add.go +++ b/vendor/github.com/containers/buildah/add.go @@ -14,6 +14,7 @@ import ( "github.com/containers/buildah/pkg/chrootuser" "github.com/containers/buildah/util" "github.com/containers/storage/pkg/archive" + "github.com/containers/storage/pkg/fileutils" "github.com/containers/storage/pkg/idtools" "github.com/containers/storage/pkg/system" "github.com/opencontainers/runtime-spec/specs-go" @@ -89,7 +90,10 @@ func addURL(destination, srcurl string, owner idtools.IDPair, hasher io.Writer) // filesystem, optionally extracting contents of local files that look like // non-empty archives. func (b *Builder) Add(destination string, extract bool, options AddAndCopyOptions, source ...string) error { - excludes := dockerIgnoreHelper(options.Excludes, options.ContextDir) + excludes, err := dockerIgnoreMatcher(options.Excludes, options.ContextDir) + if err != nil { + return err + } mountPoint, err := b.Mount(b.MountLabel) if err != nil { return err @@ -100,7 +104,7 @@ func (b *Builder) Add(destination string, extract bool, options AddAndCopyOption } }() // Find out which user (and group) the destination should belong to. - user, err := b.user(mountPoint, options.Chown) + user, _, err := b.user(mountPoint, options.Chown) if err != nil { return err } @@ -153,12 +157,12 @@ func (b *Builder) Add(destination string, extract bool, options AddAndCopyOption } // user returns the user (and group) information which the destination should belong to. -func (b *Builder) user(mountPoint string, userspec string) (specs.User, error) { +func (b *Builder) user(mountPoint string, userspec string) (specs.User, string, error) { if userspec == "" { userspec = b.User() } - uid, gid, err := chrootuser.GetUser(mountPoint, userspec) + uid, gid, homeDir, err := chrootuser.GetUser(mountPoint, userspec) u := specs.User{ UID: uid, GID: gid, @@ -175,45 +179,48 @@ func (b *Builder) user(mountPoint string, userspec string) (specs.User, error) { } } - return u, err + return u, homeDir, err } -// dockerIgnore struct keep info from .dockerignore -type dockerIgnore struct { - ExcludePath string - IsExcluded bool -} - -// dockerIgnoreHelper returns the lines from .dockerignore file without the comments -// and reverses the order -func dockerIgnoreHelper(lines []string, contextDir string) []dockerIgnore { - var excludes []dockerIgnore - // the last match of a file in the .dockerignmatches determines whether it is included or excluded - // reverse the order - for i := len(lines) - 1; i >= 0; i-- { - exclude := lines[i] - // ignore the comment in .dockerignore - if strings.HasPrefix(exclude, "#") || len(exclude) == 0 { +// dockerIgnoreMatcher returns a matcher based on the contents of the .dockerignore file under contextDir +func dockerIgnoreMatcher(lines []string, contextDir string) (*fileutils.PatternMatcher, error) { + // if there's no context dir, there's no .dockerignore file to consult + if contextDir == "" { + return nil, nil + } + patterns := []string{".dockerignore"} + for _, ignoreSpec := range lines { + ignoreSpec = strings.TrimSpace(ignoreSpec) + // ignore comments passed back from .dockerignore + if ignoreSpec == "" || ignoreSpec[0] == '#' { continue } - excludeFlag := true - if strings.HasPrefix(exclude, "!") { - exclude = strings.TrimPrefix(exclude, "!") - excludeFlag = false + // if the spec starts with '!' it means the pattern + // should be included. make a note so that we can move + // it to the front of the updated pattern + includeFlag := "" + if strings.HasPrefix(ignoreSpec, "!") { + includeFlag = "!" + ignoreSpec = ignoreSpec[1:] } - excludes = append(excludes, dockerIgnore{ExcludePath: filepath.Join(contextDir, exclude), IsExcluded: excludeFlag}) + if ignoreSpec == "" { + continue + } + patterns = append(patterns, includeFlag+filepath.Join(contextDir, ignoreSpec)) } - if len(excludes) != 0 { - excludes = append(excludes, dockerIgnore{ExcludePath: filepath.Join(contextDir, ".dockerignore"), IsExcluded: true}) + // if there are no patterns, save time by not constructing the object + if len(patterns) == 0 { + return nil, nil } - return excludes -} - -func addHelper(excludes []dockerIgnore, extract bool, dest string, destfi os.FileInfo, hostOwner idtools.IDPair, options AddAndCopyOptions, copyFileWithTar, copyWithTar, untarPath func(src, dest string) error, source ...string) error { - dirsInDockerignore, err := getDirsInDockerignore(options.ContextDir, excludes) + // return a matcher object + matcher, err := fileutils.NewPatternMatcher(patterns) if err != nil { - return errors.Wrapf(err, "error checking directories in .dockerignore") + return nil, errors.Wrapf(err, "error creating file matcher using patterns %v", patterns) } + return matcher, nil +} + +func addHelper(excludes *fileutils.PatternMatcher, extract bool, dest string, destfi os.FileInfo, hostOwner idtools.IDPair, options AddAndCopyOptions, copyFileWithTar, copyWithTar, untarPath func(src, dest string) error, source ...string) error { for _, src := range source { if strings.HasPrefix(src, "http://") || strings.HasPrefix(src, "https://") { // We assume that source is a file, and we're copying @@ -242,7 +249,7 @@ func addHelper(excludes []dockerIgnore, extract bool, dest string, destfi os.Fil if len(glob) == 0 { return errors.Wrapf(syscall.ENOENT, "no files found matching %q", src) } - outer: + for _, gsrc := range glob { esrc, err := filepath.EvalSymlinks(gsrc) if err != nil { @@ -261,7 +268,7 @@ func addHelper(excludes []dockerIgnore, extract bool, dest string, destfi os.Fil return errors.Wrapf(err, "error creating directory %q", dest) } logrus.Debugf("copying %q to %q", esrc+string(os.PathSeparator)+"*", dest+string(os.PathSeparator)+"*") - if len(excludes) == 0 { + if excludes == nil { if err = copyWithTar(esrc, dest); err != nil { return errors.Wrapf(err, "error copying %q to %q", esrc, dest) } @@ -271,23 +278,12 @@ func addHelper(excludes []dockerIgnore, extract bool, dest string, destfi os.Fil if err != nil { return err } - for _, exclude := range excludes { - match, err := filepath.Match(filepath.Clean(exclude.ExcludePath), filepath.Clean(path)) - if err != nil { - return err - } - prefix, exist := dirsInDockerignore[exclude.ExcludePath] - hasPrefix := false - if exist { - hasPrefix = filepath.HasPrefix(path, prefix) - } - if !(match || hasPrefix) { - continue - } - if (hasPrefix && exclude.IsExcluded) || (match && exclude.IsExcluded) { - return nil - } - break + skip, err := excludes.Matches(path) + if err != nil { + return errors.Wrapf(err, "error checking if %s is an excluded path", path) + } + if skip { + return nil } // combine the filename with the dest directory fpath, err := filepath.Rel(esrc, path) @@ -297,8 +293,8 @@ func addHelper(excludes []dockerIgnore, extract bool, dest string, destfi os.Fil mtime := info.ModTime() atime := mtime times := []syscall.Timespec{ - {Sec: atime.Unix(), Nsec: atime.UnixNano() % 1000000000}, - {Sec: mtime.Unix(), Nsec: mtime.UnixNano() % 1000000000}, + syscall.NsecToTimespec(atime.Unix()), + syscall.NsecToTimespec(mtime.Unix()), } if info.IsDir() { return addHelperDirectory(esrc, path, filepath.Join(dest, fpath), info, hostOwner, times) @@ -320,20 +316,6 @@ func addHelper(excludes []dockerIgnore, extract bool, dest string, destfi os.Fil continue } - for _, exclude := range excludes { - match, err := filepath.Match(filepath.Clean(exclude.ExcludePath), esrc) - if err != nil { - return err - } - if !match { - continue - } - if exclude.IsExcluded { - continue outer - } - break - } - if !extract || !archive.IsArchivePath(esrc) { // This source is a file, and either it's not an // archive, or we don't care whether or not it's an @@ -349,6 +331,7 @@ func addHelper(excludes []dockerIgnore, extract bool, dest string, destfi os.Fil } continue } + // We're extracting an archive into the destination directory. logrus.Debugf("extracting contents of %q into %q", esrc, dest) if err = untarPath(esrc, dest); err != nil { @@ -381,7 +364,15 @@ func addHelperSymlink(src, dest string, info os.FileInfo, hostOwner idtools.IDPa return errors.Wrapf(err, "error reading contents of symbolic link at %q", src) } if err = os.Symlink(linkContents, dest); err != nil { - return errors.Wrapf(err, "error creating symbolic link to %q at %q", linkContents, dest) + if !os.IsExist(err) { + return errors.Wrapf(err, "error creating symbolic link to %q at %q", linkContents, dest) + } + if err = os.RemoveAll(dest); err != nil { + return errors.Wrapf(err, "error clearing symbolic link target %q", dest) + } + if err = os.Symlink(linkContents, dest); err != nil { + return errors.Wrapf(err, "error creating symbolic link to %q at %q (second try)", linkContents, dest) + } } if err = idtools.SafeLchown(dest, hostOwner.UID, hostOwner.GID); err != nil { return errors.Wrapf(err, "error setting owner of symbolic link %q to %d:%d", dest, hostOwner.UID, hostOwner.GID) @@ -392,35 +383,3 @@ func addHelperSymlink(src, dest string, info os.FileInfo, hostOwner idtools.IDPa logrus.Debugf("Symlink(%s, %s)", linkContents, dest) return nil } - -func getDirsInDockerignore(srcAbsPath string, excludes []dockerIgnore) (map[string]string, error) { - visitedDir := make(map[string]string) - if len(excludes) == 0 { - return visitedDir, nil - } - err := filepath.Walk(srcAbsPath, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if info.IsDir() { - for _, exclude := range excludes { - match, err := filepath.Match(filepath.Clean(exclude.ExcludePath), filepath.Clean(path)) - if err != nil { - return err - } - if !match { - continue - } - if _, exist := visitedDir[exclude.ExcludePath]; exist { - continue - } - visitedDir[exclude.ExcludePath] = path - } - } - return nil - }) - if err != nil { - return visitedDir, err - } - return visitedDir, nil -} diff --git a/vendor/github.com/containers/buildah/buildah.go b/vendor/github.com/containers/buildah/buildah.go index 33b7afccd..56c8f088f 100644 --- a/vendor/github.com/containers/buildah/buildah.go +++ b/vendor/github.com/containers/buildah/buildah.go @@ -26,7 +26,7 @@ const ( Package = "buildah" // Version for the Package. Bump version in contrib/rpm/buildah.spec // too. - Version = "1.9.0-dev" + Version = "1.8.3" // The value we use to identify what type of information, currently a // serialized Builder structure, we are using as per-container state. // This should only be changed when we make incompatible changes to diff --git a/vendor/github.com/containers/buildah/chroot/run.go b/vendor/github.com/containers/buildah/chroot/run.go index c65926c8e..d6e5a61ea 100644 --- a/vendor/github.com/containers/buildah/chroot/run.go +++ b/vendor/github.com/containers/buildah/chroot/run.go @@ -84,9 +84,18 @@ type runUsingChrootExecSubprocOptions struct { // RunUsingChroot runs a chrooted process, using some of the settings from the // passed-in spec, and using the specified bundlePath to hold temporary files, // directories, and mountpoints. -func RunUsingChroot(spec *specs.Spec, bundlePath string, stdin io.Reader, stdout, stderr io.Writer) (err error) { +func RunUsingChroot(spec *specs.Spec, bundlePath, homeDir string, stdin io.Reader, stdout, stderr io.Writer) (err error) { var confwg sync.WaitGroup - + var homeFound bool + for _, env := range spec.Process.Env { + if strings.HasPrefix(env, "HOME=") { + homeFound = true + break + } + } + if !homeFound { + spec.Process.Env = append(spec.Process.Env, fmt.Sprintf("HOME=%s", homeDir)) + } runtime.LockOSThread() defer runtime.UnlockOSThread() diff --git a/vendor/github.com/containers/buildah/config.go b/vendor/github.com/containers/buildah/config.go index 05b0abb23..234f93259 100644 --- a/vendor/github.com/containers/buildah/config.go +++ b/vendor/github.com/containers/buildah/config.go @@ -3,7 +3,6 @@ package buildah import ( "context" "encoding/json" - "os" "runtime" "strings" "time" @@ -269,21 +268,11 @@ func (b *Builder) Env() []string { // built using an image built from this container. func (b *Builder) SetEnv(k string, v string) { reset := func(s *[]string) { - getenv := func(name string) string { - for i := range *s { - val := strings.SplitN((*s)[i], "=", 2) - if len(val) == 2 && val[0] == name { - return val[1] - } - } - return name - } n := []string{} for i := range *s { if !strings.HasPrefix((*s)[i], k+"=") { n = append(n, (*s)[i]) } - v = os.Expand(v, getenv) } n = append(n, k+"="+v) *s = n diff --git a/vendor/github.com/containers/buildah/imagebuildah/build.go b/vendor/github.com/containers/buildah/imagebuildah/build.go index b8b9db0f3..3665251cd 100644 --- a/vendor/github.com/containers/buildah/imagebuildah/build.go +++ b/vendor/github.com/containers/buildah/imagebuildah/build.go @@ -487,6 +487,7 @@ func (s *StageExecutor) Copy(excludes []string, copies ...imagebuilder.Copy) err // Check the file and see if part of it is a symlink. // Convert it to the target if so. To be ultrasafe // do the same for the mountpoint. + hadFinalPathSeparator := len(copy.Dest) > 0 && copy.Dest[len(copy.Dest)-1] == os.PathSeparator secureMountPoint, err := securejoin.SecureJoin("", s.mountPoint) finalPath, err := securejoin.SecureJoin(secureMountPoint, copy.Dest) if err != nil { @@ -496,6 +497,11 @@ func (s *StageExecutor) Copy(excludes []string, copies ...imagebuilder.Copy) err return errors.Wrapf(err, "error resolving copy destination %s", copy.Dest) } copy.Dest = strings.TrimPrefix(finalPath, secureMountPoint) + if len(copy.Dest) == 0 || copy.Dest[len(copy.Dest)-1] != os.PathSeparator { + if hadFinalPathSeparator { + copy.Dest += string(os.PathSeparator) + } + } if copy.Download { logrus.Debugf("ADD %#v, %#v", excludes, copy) @@ -507,29 +513,32 @@ func (s *StageExecutor) Copy(excludes []string, copies ...imagebuilder.Copy) err } sources := []string{} for _, src := range copy.Src { + contextDir := s.executor.contextDir + copyExcludes := excludes if strings.HasPrefix(src, "http://") || strings.HasPrefix(src, "https://") { sources = append(sources, src) } else if len(copy.From) > 0 { if other, ok := s.executor.stages[copy.From]; ok && other.index < s.index { sources = append(sources, filepath.Join(other.mountPoint, src)) + contextDir = other.mountPoint } else if builder, ok := s.executor.containerMap[copy.From]; ok { sources = append(sources, filepath.Join(builder.MountPoint, src)) + contextDir = builder.MountPoint } else { return errors.Errorf("the stage %q has not been built", copy.From) } } else { sources = append(sources, filepath.Join(s.executor.contextDir, src)) + copyExcludes = append(s.executor.excludes, excludes...) + } + options := buildah.AddAndCopyOptions{ + Chown: copy.Chown, + ContextDir: contextDir, + Excludes: copyExcludes, + } + if err := s.builder.Add(copy.Dest, copy.Download, options, sources...); err != nil { + return err } - } - - options := buildah.AddAndCopyOptions{ - Chown: copy.Chown, - ContextDir: s.executor.contextDir, - Excludes: s.executor.excludes, - } - - if err := s.builder.Add(copy.Dest, copy.Download, options, sources...); err != nil { - return err } } return nil @@ -590,7 +599,11 @@ func (s *StageExecutor) Run(run imagebuilder.Run, config docker.Config) error { args := run.Args if run.Shell { - args = append([]string{"/bin/sh", "-c"}, args...) + if len(config.Shell) > 0 && s.builder.Format == buildah.Dockerv2ImageManifest { + args = append(config.Shell, args...) + } else { + args = append([]string{"/bin/sh", "-c"}, args...) + } } if err := s.volumeCacheSave(); err != nil { return err diff --git a/vendor/github.com/containers/buildah/imagebuildah/util.go b/vendor/github.com/containers/buildah/imagebuildah/util.go index f982fcebf..3962d1a9d 100644 --- a/vendor/github.com/containers/buildah/imagebuildah/util.go +++ b/vendor/github.com/containers/buildah/imagebuildah/util.go @@ -17,7 +17,7 @@ import ( ) func cloneToDirectory(url, dir string) error { - if !strings.HasPrefix(url, "git://") { + if !strings.HasPrefix(url, "git://") && !strings.HasSuffix(url, ".git") { url = "git://" + url } logrus.Debugf("cloning %q to %q", url, dir) @@ -72,7 +72,7 @@ func TempDirForURL(dir, prefix, url string) (name string, subdir string, err err if err != nil { return "", "", errors.Wrapf(err, "error creating temporary directory for %q", url) } - if strings.HasPrefix(url, "git://") { + if strings.HasPrefix(url, "git://") || strings.HasSuffix(url, ".git") { err = cloneToDirectory(url, name) if err != nil { if err2 := os.Remove(name); err2 != nil { diff --git a/vendor/github.com/containers/buildah/pkg/chrootuser/user.go b/vendor/github.com/containers/buildah/pkg/chrootuser/user.go index c83dcc230..26a67c35a 100644 --- a/vendor/github.com/containers/buildah/pkg/chrootuser/user.go +++ b/vendor/github.com/containers/buildah/pkg/chrootuser/user.go @@ -18,7 +18,7 @@ var ( // it will use the /etc/passwd and /etc/group files inside of the rootdir // to return this information. // userspec format [user | user:group | uid | uid:gid | user:gid | uid:group ] -func GetUser(rootdir, userspec string) (uint32, uint32, error) { +func GetUser(rootdir, userspec string) (uint32, uint32, string, error) { var gid64 uint64 var gerr error = user.UnknownGroupError("error looking up group") @@ -26,7 +26,7 @@ func GetUser(rootdir, userspec string) (uint32, uint32, error) { userspec = spec[0] groupspec := "" if userspec == "" { - return 0, 0, nil + return 0, 0, "/", nil } if len(spec) > 1 { groupspec = spec[1] @@ -65,15 +65,21 @@ func GetUser(rootdir, userspec string) (uint32, uint32, error) { } } + homedir, err := lookupHomedirInContainer(rootdir, uid64) + if err != nil { + homedir = "/" + } + if uerr == nil && gerr == nil { - return uint32(uid64), uint32(gid64), nil + return uint32(uid64), uint32(gid64), homedir, nil } - err := errors.Wrapf(uerr, "error determining run uid") + err = errors.Wrapf(uerr, "error determining run uid") if uerr == nil { err = errors.Wrapf(gerr, "error determining run gid") } - return 0, 0, err + + return 0, 0, homedir, err } // GetGroup returns the gid by looking it up in the /etc/group file diff --git a/vendor/github.com/containers/buildah/pkg/chrootuser/user_basic.go b/vendor/github.com/containers/buildah/pkg/chrootuser/user_basic.go index 79b0b24b5..6c997c4c9 100644 --- a/vendor/github.com/containers/buildah/pkg/chrootuser/user_basic.go +++ b/vendor/github.com/containers/buildah/pkg/chrootuser/user_basic.go @@ -25,3 +25,7 @@ func lookupAdditionalGroupsForUIDInContainer(rootdir string, userid uint64) (gid func lookupUIDInContainer(rootdir string, uid uint64) (string, uint64, error) { return "", 0, errors.New("UID lookup not supported") } + +func lookupHomedirInContainer(rootdir string, uid uint64) (string, error) { + return "", errors.New("Home directory lookup not supported") +} diff --git a/vendor/github.com/containers/buildah/pkg/chrootuser/user_linux.go b/vendor/github.com/containers/buildah/pkg/chrootuser/user_linux.go index 583eca569..ea20fca80 100644 --- a/vendor/github.com/containers/buildah/pkg/chrootuser/user_linux.go +++ b/vendor/github.com/containers/buildah/pkg/chrootuser/user_linux.go @@ -84,6 +84,7 @@ type lookupPasswdEntry struct { name string uid uint64 gid uint64 + home string } type lookupGroupEntry struct { name string @@ -135,6 +136,7 @@ func parseNextPasswd(rc *bufio.Reader) *lookupPasswdEntry { name: fields[0], uid: uid, gid: gid, + home: fields[5], } } @@ -291,3 +293,29 @@ func lookupUIDInContainer(rootdir string, uid uint64) (string, uint64, error) { return "", 0, user.UnknownUserError(fmt.Sprintf("error looking up uid %q", uid)) } + +func lookupHomedirInContainer(rootdir string, uid uint64) (string, error) { + cmd, f, err := openChrootedFile(rootdir, "/etc/passwd") + if err != nil { + return "", err + } + defer func() { + _ = cmd.Wait() + }() + rc := bufio.NewReader(f) + defer f.Close() + + lookupUser.Lock() + defer lookupUser.Unlock() + + pwd := parseNextPasswd(rc) + for pwd != nil { + if pwd.uid != uid { + pwd = parseNextPasswd(rc) + continue + } + return pwd.home, nil + } + + return "", user.UnknownUserError(fmt.Sprintf("error looking up uid %q for homedir", uid)) +} diff --git a/vendor/github.com/containers/buildah/pkg/secrets/secrets.go b/vendor/github.com/containers/buildah/pkg/secrets/secrets.go index 97b681125..70bd6a4b7 100644 --- a/vendor/github.com/containers/buildah/pkg/secrets/secrets.go +++ b/vendor/github.com/containers/buildah/pkg/secrets/secrets.go @@ -117,7 +117,12 @@ func getMounts(filePath string) []string { } var mounts []string for scanner.Scan() { - mounts = append(mounts, scanner.Text()) + if strings.HasPrefix(strings.TrimSpace(scanner.Text()), "/") { + mounts = append(mounts, scanner.Text()) + } else { + logrus.Debugf("skipping unrecognized mount in %v: %q", + filePath, scanner.Text()) + } } return mounts } @@ -190,58 +195,79 @@ func addSecretsFromMountsFile(filePath, mountLabel, containerWorkingDir, mountPr var mounts []rspec.Mount defaultMountsPaths := getMounts(filePath) for _, path := range defaultMountsPaths { - hostDir, ctrDir, err := getMountsMap(path) + hostDirOrFile, ctrDirOrFile, err := getMountsMap(path) if err != nil { return nil, err } - // skip if the hostDir path doesn't exist - if _, err = os.Stat(hostDir); err != nil { + // skip if the hostDirOrFile path doesn't exist + fileInfo, err := os.Stat(hostDirOrFile) + if err != nil { if os.IsNotExist(err) { - logrus.Warnf("Path %q from %q doesn't exist, skipping", hostDir, filePath) + logrus.Warnf("Path %q from %q doesn't exist, skipping", hostDirOrFile, filePath) continue } - return nil, errors.Wrapf(err, "failed to stat %q", hostDir) + return nil, errors.Wrapf(err, "failed to stat %q", hostDirOrFile) } - ctrDirOnHost := filepath.Join(containerWorkingDir, ctrDir) + ctrDirOrFileOnHost := filepath.Join(containerWorkingDir, ctrDirOrFile) - // In the event of a restart, don't want to copy secrets over again as they already would exist in ctrDirOnHost - _, err = os.Stat(ctrDirOnHost) + // In the event of a restart, don't want to copy secrets over again as they already would exist in ctrDirOrFileOnHost + _, err = os.Stat(ctrDirOrFileOnHost) if os.IsNotExist(err) { - if err = os.MkdirAll(ctrDirOnHost, 0755); err != nil { - return nil, errors.Wrapf(err, "making container directory %q failed", ctrDirOnHost) - } - hostDir, err = resolveSymbolicLink(hostDir) + + hostDirOrFile, err = resolveSymbolicLink(hostDirOrFile) if err != nil { return nil, err } - data, err := getHostSecretData(hostDir) - if err != nil { - return nil, errors.Wrapf(err, "getting host secret data failed") - } - for _, s := range data { - if err := s.saveTo(ctrDirOnHost); err != nil { - return nil, errors.Wrapf(err, "error saving data to container filesystem on host %q", ctrDirOnHost) + switch mode := fileInfo.Mode(); { + case mode.IsDir(): + if err = os.MkdirAll(ctrDirOrFileOnHost, 0755); err != nil { + return nil, errors.Wrapf(err, "making container directory %q failed", ctrDirOrFileOnHost) + } + data, err := getHostSecretData(hostDirOrFile) + if err != nil { + return nil, errors.Wrapf(err, "getting host secret data failed") + } + for _, s := range data { + if err := s.saveTo(ctrDirOrFileOnHost); err != nil { + return nil, errors.Wrapf(err, "error saving data to container filesystem on host %q", ctrDirOrFileOnHost) + } + } + case mode.IsRegular(): + data, err := readFile("", hostDirOrFile) + if err != nil { + return nil, errors.Wrapf(err, "error reading file %q", hostDirOrFile) + + } + for _, s := range data { + if err := os.MkdirAll(filepath.Dir(ctrDirOrFileOnHost), 0700); err != nil { + return nil, err + } + if err := ioutil.WriteFile(ctrDirOrFileOnHost, s.data, 0700); err != nil { + return nil, errors.Wrapf(err, "error saving data to container filesystem on host %q", ctrDirOrFileOnHost) + } } + default: + return nil, errors.Errorf("unsupported file type for: %q", hostDirOrFile) } - err = label.Relabel(ctrDirOnHost, mountLabel, false) + err = label.Relabel(ctrDirOrFileOnHost, mountLabel, false) if err != nil { return nil, errors.Wrap(err, "error applying correct labels") } if uid != 0 || gid != 0 { - if err := rchown(ctrDirOnHost, uid, gid); err != nil { + if err := rchown(ctrDirOrFileOnHost, uid, gid); err != nil { return nil, err } } } else if err != nil { - return nil, errors.Wrapf(err, "error getting status of %q", ctrDirOnHost) + return nil, errors.Wrapf(err, "error getting status of %q", ctrDirOrFileOnHost) } m := rspec.Mount{ - Source: filepath.Join(mountPrefix, ctrDir), - Destination: ctrDir, + Source: filepath.Join(mountPrefix, ctrDirOrFile), + Destination: ctrDirOrFile, Type: "bind", Options: []string{"bind", "rprivate"}, } diff --git a/vendor/github.com/containers/buildah/pkg/unshare/unshare.go b/vendor/github.com/containers/buildah/pkg/unshare/unshare.go index 33232740e..5a68d6b8d 100644 --- a/vendor/github.com/containers/buildah/pkg/unshare/unshare.go +++ b/vendor/github.com/containers/buildah/pkg/unshare/unshare.go @@ -64,6 +64,7 @@ func (c *Cmd) Start() error { if os.Geteuid() != 0 { c.Env = append(c.Env, "_CONTAINERS_USERNS_CONFIGURED=done") c.Env = append(c.Env, fmt.Sprintf("_CONTAINERS_ROOTLESS_UID=%d", os.Geteuid())) + c.Env = append(c.Env, fmt.Sprintf("_CONTAINERS_ROOTLESS_GID=%d", os.Getegid())) } // Create the pipe for reading the child's PID. @@ -183,6 +184,7 @@ func (c *Cmd) Start() error { for _, m := range c.GidMappings { fmt.Fprintf(g, "%d %d %d\n", m.ContainerID, m.HostID, m.Size) } + gidmapSet := false // Set the GID map. if c.UseNewgidmap { cmd := exec.Command("newgidmap", append([]string{pidString}, strings.Fields(strings.Replace(g.String(), "\n", " ", -1))...)...) @@ -190,11 +192,16 @@ func (c *Cmd) Start() error { cmd.Stdout = g cmd.Stderr = g err := cmd.Run() - if err != nil { + if err == nil { + gidmapSet = true + } else { fmt.Fprintf(continueWrite, "error running newgidmap: %v: %s", err, g.String()) - return errors.Wrapf(err, "error running newgidmap: %s", g.String()) + fmt.Fprintf(continueWrite, "falling back to single mapping\n") + g.Reset() + g.Write([]byte(fmt.Sprintf("0 %d 1\n", os.Getegid()))) } - } else { + } + if !gidmapSet { gidmap, err := os.OpenFile(fmt.Sprintf("/proc/%s/gid_map", pidString), os.O_TRUNC|os.O_WRONLY, 0) if err != nil { fmt.Fprintf(continueWrite, "error opening /proc/%s/gid_map: %v", pidString, err) @@ -214,6 +221,7 @@ func (c *Cmd) Start() error { for _, m := range c.UidMappings { fmt.Fprintf(u, "%d %d %d\n", m.ContainerID, m.HostID, m.Size) } + uidmapSet := false // Set the GID map. if c.UseNewuidmap { cmd := exec.Command("newuidmap", append([]string{pidString}, strings.Fields(strings.Replace(u.String(), "\n", " ", -1))...)...) @@ -221,11 +229,16 @@ func (c *Cmd) Start() error { cmd.Stdout = u cmd.Stderr = u err := cmd.Run() - if err != nil { + if err == nil { + uidmapSet = true + } else { fmt.Fprintf(continueWrite, "error running newuidmap: %v: %s", err, u.String()) - return errors.Wrapf(err, "error running newuidmap: %s", u.String()) + fmt.Fprintf(continueWrite, "falling back to single mapping\n") + u.Reset() + u.Write([]byte(fmt.Sprintf("0 %d 1\n", os.Geteuid()))) } - } else { + } + if !uidmapSet { uidmap, err := os.OpenFile(fmt.Sprintf("/proc/%s/uid_map", pidString), os.O_TRUNC|os.O_WRONLY, 0) if err != nil { fmt.Fprintf(continueWrite, "error opening /proc/%s/uid_map: %v", pidString, err) @@ -354,7 +367,9 @@ func MaybeReexecUsingUserNamespace(evenForRoot bool) { // range in /etc/subuid and /etc/subgid file is a starting host // ID and a range size. uidmap, gidmap, err = GetSubIDMappings(me.Username, me.Username) - bailOnError(err, "error reading allowed ID mappings") + if err != nil { + logrus.Warnf("error reading allowed ID mappings: %v", err) + } if len(uidmap) == 0 { logrus.Warnf("Found no UID ranges set aside for user %q in /etc/subuid.", me.Username) } diff --git a/vendor/github.com/containers/buildah/run_linux.go b/vendor/github.com/containers/buildah/run_linux.go index 81ce2b944..16c0550aa 100644 --- a/vendor/github.com/containers/buildah/run_linux.go +++ b/vendor/github.com/containers/buildah/run_linux.go @@ -131,7 +131,8 @@ func (b *Builder) Run(command []string, options RunOptions) error { return err } - if err := b.configureUIDGID(g, mountPoint, options); err != nil { + homeDir, err := b.configureUIDGID(g, mountPoint, options) + if err != nil { return err } @@ -210,7 +211,7 @@ func (b *Builder) Run(command []string, options RunOptions) error { } err = b.runUsingRuntimeSubproc(isolation, options, configureNetwork, configureNetworks, moreCreateArgs, spec, mountPoint, path, Package+"-"+filepath.Base(path)) case IsolationChroot: - err = chroot.RunUsingChroot(spec, path, options.Stdin, options.Stdout, options.Stderr) + err = chroot.RunUsingChroot(spec, path, homeDir, options.Stdin, options.Stdout, options.Stderr) case IsolationOCIRootless: moreCreateArgs := []string{"--no-new-keyring"} if options.NoPivot { @@ -1454,7 +1455,18 @@ func setupNamespaces(g *generate.Generator, namespaceOptions NamespaceOptions, i } if configureNetwork { for name, val := range util.DefaultNetworkSysctl { - g.AddLinuxSysctl(name, val) + // Check that the sysctl we are adding is actually supported + // by the kernel + p := filepath.Join("/proc/sys", strings.Replace(name, ".", "/", -1)) + _, err := os.Stat(p) + if err != nil && !os.IsNotExist(err) { + return false, nil, false, errors.Wrapf(err, "cannot stat %s", p) + } + if err == nil { + g.AddLinuxSysctl(name, val) + } else { + logrus.Warnf("ignoring sysctl %s since %s doesn't exist", name, p) + } } } return configureNetwork, configureNetworks, configureUTS, nil @@ -1775,14 +1787,14 @@ func getDNSIP(dnsServers []string) (dns []net.IP, err error) { return dns, nil } -func (b *Builder) configureUIDGID(g *generate.Generator, mountPoint string, options RunOptions) error { +func (b *Builder) configureUIDGID(g *generate.Generator, mountPoint string, options RunOptions) (string, error) { // Set the user UID/GID/supplemental group list/capabilities lists. - user, err := b.user(mountPoint, options.User) + user, homeDir, err := b.user(mountPoint, options.User) if err != nil { - return err + return "", err } if err := setupCapabilities(g, b.AddCapabilities, b.DropCapabilities, options.AddCapabilities, options.DropCapabilities); err != nil { - return err + return "", err } g.SetProcessUID(user.UID) g.SetProcessGID(user.GID) @@ -1797,7 +1809,7 @@ func (b *Builder) configureUIDGID(g *generate.Generator, mountPoint string, opti g.Config.Process.Capabilities.Bounding = bounding } - return nil + return homeDir, nil } func (b *Builder) configureEnvironment(g *generate.Generator, options RunOptions) { diff --git a/vendor/github.com/containers/storage/containers.go b/vendor/github.com/containers/storage/containers.go index bbac78b60..e69552361 100644 --- a/vendor/github.com/containers/storage/containers.go +++ b/vendor/github.com/containers/storage/containers.go @@ -572,6 +572,10 @@ func (r *containerStore) Lock() { r.lockfile.Lock() } +func (r *containerStore) RecursiveLock() { + r.lockfile.RecursiveLock() +} + func (r *containerStore) RLock() { r.lockfile.RLock() } diff --git a/vendor/github.com/containers/storage/drivers/aufs/aufs.go b/vendor/github.com/containers/storage/drivers/aufs/aufs.go index e821bc0c5..353d1707a 100644 --- a/vendor/github.com/containers/storage/drivers/aufs/aufs.go +++ b/vendor/github.com/containers/storage/drivers/aufs/aufs.go @@ -255,6 +255,9 @@ func (a *Driver) AdditionalImageStores() []string { // CreateFromTemplate creates a layer with the same contents and parent as another layer. func (a *Driver) CreateFromTemplate(id, template string, templateIDMappings *idtools.IDMappings, parent string, parentIDMappings *idtools.IDMappings, opts *graphdriver.CreateOpts, readWrite bool) error { + if opts == nil { + opts = &graphdriver.CreateOpts{} + } return graphdriver.NaiveCreateFromTemplate(a, id, template, templateIDMappings, parent, parentIDMappings, opts, readWrite) } diff --git a/vendor/github.com/containers/storage/drivers/chown.go b/vendor/github.com/containers/storage/drivers/chown.go index 4d4011ee0..f2f1ec386 100644 --- a/vendor/github.com/containers/storage/drivers/chown.go +++ b/vendor/github.com/containers/storage/drivers/chown.go @@ -55,6 +55,9 @@ func chownByMapsMain() { if err != nil { return fmt.Errorf("error walking to %q: %v", path, err) } + if path == "." { + return nil + } return platformLChown(path, info, toHost, toContainer) } if err := filepath.Walk(".", chown); err != nil { diff --git a/vendor/github.com/containers/storage/images.go b/vendor/github.com/containers/storage/images.go index 38b5a3ef3..6f487504a 100644 --- a/vendor/github.com/containers/storage/images.go +++ b/vendor/github.com/containers/storage/images.go @@ -82,6 +82,9 @@ type Image struct { // is set before using it. Created time.Time `json:"created,omitempty"` + // ReadOnly is true if this image resides in a read-only layer store. + ReadOnly bool `json:"-"` + Flags map[string]interface{} `json:"flags,omitempty"` } @@ -159,6 +162,7 @@ func copyImage(i *Image) *Image { BigDataSizes: copyStringInt64Map(i.BigDataSizes), BigDataDigests: copyStringDigestMap(i.BigDataDigests), Created: i.Created, + ReadOnly: i.ReadOnly, Flags: copyStringInterfaceMap(i.Flags), } } @@ -269,6 +273,7 @@ func (r *imageStore) Load() error { list := digests[digest] digests[digest] = append(list, image) } + image.ReadOnly = !r.IsReadWrite() } } if shouldSave && (!r.IsReadWrite() || !r.Locked()) { @@ -739,6 +744,10 @@ func (r *imageStore) Lock() { r.lockfile.Lock() } +func (r *imageStore) RecursiveLock() { + r.lockfile.RecursiveLock() +} + func (r *imageStore) RLock() { r.lockfile.RLock() } diff --git a/vendor/github.com/containers/storage/layers.go b/vendor/github.com/containers/storage/layers.go index a35dd476b..fb79238cd 100644 --- a/vendor/github.com/containers/storage/layers.go +++ b/vendor/github.com/containers/storage/layers.go @@ -103,6 +103,9 @@ type Layer struct { // for use inside of a user namespace where UID mapping is being used. UIDMap []idtools.IDMap `json:"uidmap,omitempty"` GIDMap []idtools.IDMap `json:"gidmap,omitempty"` + + // ReadOnly is true if this layer resides in a read-only layer store. + ReadOnly bool `json:"-"` } type layerMountPoint struct { @@ -259,6 +262,7 @@ func copyLayer(l *Layer) *Layer { UncompressedDigest: l.UncompressedDigest, UncompressedSize: l.UncompressedSize, CompressionType: l.CompressionType, + ReadOnly: l.ReadOnly, Flags: copyStringInterfaceMap(l.Flags), UIDMap: copyIDMap(l.UIDMap), GIDMap: copyIDMap(l.GIDMap), @@ -318,6 +322,7 @@ func (r *layerStore) Load() error { if layer.MountLabel != "" { label.ReserveLabel(layer.MountLabel) } + layer.ReadOnly = !r.IsReadWrite() } err = nil } @@ -1304,6 +1309,10 @@ func (r *layerStore) Lock() { r.lockfile.Lock() } +func (r *layerStore) RecursiveLock() { + r.lockfile.RecursiveLock() +} + func (r *layerStore) RLock() { r.lockfile.RLock() } diff --git a/vendor/github.com/containers/storage/layers_ffjson.go b/vendor/github.com/containers/storage/layers_ffjson.go index 09b5d0f33..125b5d8c9 100644 --- a/vendor/github.com/containers/storage/layers_ffjson.go +++ b/vendor/github.com/containers/storage/layers_ffjson.go @@ -1,5 +1,5 @@ // Code generated by ffjson <https://github.com/pquerna/ffjson>. DO NOT EDIT. -// source: ./layers.go +// source: layers.go package storage diff --git a/vendor/github.com/containers/storage/lockfile.go b/vendor/github.com/containers/storage/lockfile.go index ed8753337..c4f1b5549 100644 --- a/vendor/github.com/containers/storage/lockfile.go +++ b/vendor/github.com/containers/storage/lockfile.go @@ -15,6 +15,10 @@ type Locker interface { // Acquire a writer lock. Lock() + // Acquire a writer lock recursively, allowing for recursive acquisitions + // within the same process space. + RecursiveLock() + // Unlock the lock. Unlock() diff --git a/vendor/github.com/containers/storage/lockfile_linux.go b/vendor/github.com/containers/storage/lockfile_linux.go deleted file mode 100644 index 903387c66..000000000 --- a/vendor/github.com/containers/storage/lockfile_linux.go +++ /dev/null @@ -1,20 +0,0 @@ -// +build linux solaris - -package storage - -import ( - "time" - - "golang.org/x/sys/unix" -) - -// TouchedSince indicates if the lock file has been touched since the specified time -func (l *lockfile) TouchedSince(when time.Time) bool { - st := unix.Stat_t{} - err := unix.Fstat(int(l.fd), &st) - if err != nil { - return true - } - touched := time.Unix(st.Mtim.Unix()) - return when.Before(touched) -} diff --git a/vendor/github.com/containers/storage/lockfile_otherunix.go b/vendor/github.com/containers/storage/lockfile_otherunix.go deleted file mode 100644 index 041d54c05..000000000 --- a/vendor/github.com/containers/storage/lockfile_otherunix.go +++ /dev/null @@ -1,19 +0,0 @@ -// +build darwin freebsd - -package storage - -import ( - "time" - - "golang.org/x/sys/unix" -) - -func (l *lockfile) TouchedSince(when time.Time) bool { - st := unix.Stat_t{} - err := unix.Fstat(int(l.fd), &st) - if err != nil { - return true - } - touched := time.Unix(st.Mtimespec.Unix()) - return when.Before(touched) -} diff --git a/vendor/github.com/containers/storage/lockfile_unix.go b/vendor/github.com/containers/storage/lockfile_unix.go index 8e0f22cb5..00215e928 100644 --- a/vendor/github.com/containers/storage/lockfile_unix.go +++ b/vendor/github.com/containers/storage/lockfile_unix.go @@ -9,6 +9,7 @@ import ( "time" "github.com/containers/storage/pkg/stringid" + "github.com/containers/storage/pkg/system" "github.com/pkg/errors" "golang.org/x/sys/unix" ) @@ -25,6 +26,7 @@ type lockfile struct { locktype int16 locked bool ro bool + recursive bool } // openLock opens the file at path and returns the corresponding file @@ -75,7 +77,7 @@ func createLockerForPath(path string, ro bool) (Locker, error) { // lock locks the lockfile via FCTNL(2) based on the specified type and // command. -func (l *lockfile) lock(l_type int16) { +func (l *lockfile) lock(l_type int16, recursive bool) { lk := unix.Flock_t{ Type: l_type, Whence: int16(os.SEEK_SET), @@ -86,7 +88,13 @@ func (l *lockfile) lock(l_type int16) { case unix.F_RDLCK: l.rwMutex.RLock() case unix.F_WRLCK: - l.rwMutex.Lock() + if recursive { + // NOTE: that's okay as recursive is only set in RecursiveLock(), so + // there's no need to protect against hypothetical RDLCK cases. + l.rwMutex.RLock() + } else { + l.rwMutex.Lock() + } default: panic(fmt.Sprintf("attempted to acquire a file lock of unrecognized type %d", l_type)) } @@ -110,6 +118,7 @@ func (l *lockfile) lock(l_type int16) { } l.locktype = l_type l.locked = true + l.recursive = recursive l.counter++ } @@ -119,13 +128,24 @@ func (l *lockfile) Lock() { if l.ro { l.RLock() } else { - l.lock(unix.F_WRLCK) + l.lock(unix.F_WRLCK, false) + } +} + +// RecursiveLock locks the lockfile as a writer but allows for recursive +// acquisitions within the same process space. Note that RLock() will be called +// if it's a lockTypReader lock. +func (l *lockfile) RecursiveLock() { + if l.ro { + l.RLock() + } else { + l.lock(unix.F_WRLCK, true) } } // LockRead locks the lockfile as a reader. func (l *lockfile) RLock() { - l.lock(unix.F_RDLCK) + l.lock(unix.F_RDLCK, false) } // Unlock unlocks the lockfile. @@ -161,7 +181,7 @@ func (l *lockfile) Unlock() { // Close the file descriptor on the last unlock. unix.Close(int(l.fd)) } - if l.locktype == unix.F_RDLCK { + if l.locktype == unix.F_RDLCK || l.recursive { l.rwMutex.RUnlock() } else { l.rwMutex.Unlock() @@ -232,3 +252,14 @@ func (l *lockfile) Modified() (bool, error) { func (l *lockfile) IsReadWrite() bool { return !l.ro } + +// TouchedSince indicates if the lock file has been touched since the specified time +func (l *lockfile) TouchedSince(when time.Time) bool { + st, err := system.Fstat(int(l.fd)) + if err != nil { + return true + } + mtim := st.Mtim() + touched := time.Unix(mtim.Unix()) + return when.Before(touched) +} diff --git a/vendor/github.com/containers/storage/lockfile_windows.go b/vendor/github.com/containers/storage/lockfile_windows.go index c02069495..caf7c184a 100644 --- a/vendor/github.com/containers/storage/lockfile_windows.go +++ b/vendor/github.com/containers/storage/lockfile_windows.go @@ -36,6 +36,12 @@ func (l *lockfile) Lock() { l.locked = true } +func (l *lockfile) RecursiveLock() { + // We don't support Windows but a recursive writer-lock in one process-space + // is really a writer lock, so just panic. + panic("not supported") +} + func (l *lockfile) RLock() { l.mu.Lock() l.locked = true diff --git a/vendor/github.com/containers/storage/pkg/chrootarchive/archive.go b/vendor/github.com/containers/storage/pkg/chrootarchive/archive.go index a36ff1cb1..33ba6a128 100644 --- a/vendor/github.com/containers/storage/pkg/chrootarchive/archive.go +++ b/vendor/github.com/containers/storage/pkg/chrootarchive/archive.go @@ -1,7 +1,7 @@ package chrootarchive import ( - "archive/tar" + stdtar "archive/tar" "fmt" "io" "io/ioutil" @@ -34,18 +34,34 @@ func NewArchiverWithChown(tarIDMappings *idtools.IDMappings, chownOpts *idtools. // The archive may be compressed with one of the following algorithms: // identity (uncompressed), gzip, bzip2, xz. func Untar(tarArchive io.Reader, dest string, options *archive.TarOptions) error { - return untarHandler(tarArchive, dest, options, true) + return untarHandler(tarArchive, dest, options, true, dest) +} + +// UntarWithRoot is the same as `Untar`, but allows you to pass in a root directory +// The root directory is the directory that will be chrooted to. +// `dest` must be a path within `root`, if it is not an error will be returned. +// +// `root` should set to a directory which is not controlled by any potentially +// malicious process. +// +// This should be used to prevent a potential attacker from manipulating `dest` +// such that it would provide access to files outside of `dest` through things +// like symlinks. Normally `ResolveSymlinksInScope` would handle this, however +// sanitizing symlinks in this manner is inherrently racey: +// ref: CVE-2018-15664 +func UntarWithRoot(tarArchive io.Reader, dest string, options *archive.TarOptions, root string) error { + return untarHandler(tarArchive, dest, options, true, root) } // UntarUncompressed reads a stream of bytes from `archive`, parses it as a tar archive, // and unpacks it into the directory at `dest`. // The archive must be an uncompressed stream. func UntarUncompressed(tarArchive io.Reader, dest string, options *archive.TarOptions) error { - return untarHandler(tarArchive, dest, options, false) + return untarHandler(tarArchive, dest, options, false, dest) } // Handler for teasing out the automatic decompression -func untarHandler(tarArchive io.Reader, dest string, options *archive.TarOptions, decompress bool) error { +func untarHandler(tarArchive io.Reader, dest string, options *archive.TarOptions, decompress bool, root string) error { if tarArchive == nil { return fmt.Errorf("Empty archive") } @@ -77,7 +93,15 @@ func untarHandler(tarArchive io.Reader, dest string, options *archive.TarOptions r = decompressedArchive } - return invokeUnpack(r, dest, options) + return invokeUnpack(r, dest, options, root) +} + +// Tar tars the requested path while chrooted to the specified root. +func Tar(srcPath string, options *archive.TarOptions, root string) (io.ReadCloser, error) { + if options == nil { + options = &archive.TarOptions{} + } + return invokePack(srcPath, options, root) } // CopyFileWithTarAndChown returns a function which copies a single file from outside @@ -99,7 +123,7 @@ func CopyFileWithTarAndChown(chownOpts *idtools.IDPair, hasher io.Writer, uidmap var hashWorker sync.WaitGroup hashWorker.Add(1) go func() { - t := tar.NewReader(contentReader) + t := stdtar.NewReader(contentReader) _, err := t.Next() if err != nil { hashError = err diff --git a/vendor/github.com/containers/storage/pkg/chrootarchive/archive_unix.go b/vendor/github.com/containers/storage/pkg/chrootarchive/archive_unix.go index e04ed787c..ca9fb10d7 100644 --- a/vendor/github.com/containers/storage/pkg/chrootarchive/archive_unix.go +++ b/vendor/github.com/containers/storage/pkg/chrootarchive/archive_unix.go @@ -10,10 +10,13 @@ import ( "io" "io/ioutil" "os" + "path/filepath" "runtime" + "strings" "github.com/containers/storage/pkg/archive" "github.com/containers/storage/pkg/reexec" + "github.com/pkg/errors" ) // untar is the entry-point for storage-untar on re-exec. This is not used on @@ -23,18 +26,28 @@ func untar() { runtime.LockOSThread() flag.Parse() - var options *archive.TarOptions + var options archive.TarOptions //read the options from the pipe "ExtraFiles" if err := json.NewDecoder(os.NewFile(3, "options")).Decode(&options); err != nil { fatal(err) } - if err := chroot(flag.Arg(0)); err != nil { + dst := flag.Arg(0) + var root string + if len(flag.Args()) > 1 { + root = flag.Arg(1) + } + + if root == "" { + root = dst + } + + if err := chroot(root); err != nil { fatal(err) } - if err := archive.Unpack(os.Stdin, "/", options); err != nil { + if err := archive.Unpack(os.Stdin, dst, &options); err != nil { fatal(err) } // fully consume stdin in case it is zero padded @@ -45,7 +58,10 @@ func untar() { os.Exit(0) } -func invokeUnpack(decompressedArchive io.Reader, dest string, options *archive.TarOptions) error { +func invokeUnpack(decompressedArchive io.Reader, dest string, options *archive.TarOptions, root string) error { + if root == "" { + return errors.New("must specify a root to chroot to") + } // We can't pass a potentially large exclude list directly via cmd line // because we easily overrun the kernel's max argument/environment size @@ -57,7 +73,21 @@ func invokeUnpack(decompressedArchive io.Reader, dest string, options *archive.T return fmt.Errorf("Untar pipe failure: %v", err) } - cmd := reexec.Command("storage-untar", dest) + if root != "" { + relDest, err := filepath.Rel(root, dest) + if err != nil { + return err + } + if relDest == "." { + relDest = "/" + } + if relDest[0] != '/' { + relDest = "/" + relDest + } + dest = relDest + } + + cmd := reexec.Command("storage-untar", dest, root) cmd.Stdin = decompressedArchive cmd.ExtraFiles = append(cmd.ExtraFiles, r) @@ -68,6 +98,7 @@ func invokeUnpack(decompressedArchive io.Reader, dest string, options *archive.T if err := cmd.Start(); err != nil { return fmt.Errorf("Untar error on re-exec cmd: %v", err) } + //write the options to the pipe for the untar exec to read if err := json.NewEncoder(w).Encode(options); err != nil { return fmt.Errorf("Untar json encode to pipe failed: %v", err) @@ -84,3 +115,92 @@ func invokeUnpack(decompressedArchive io.Reader, dest string, options *archive.T } return nil } + +func tar() { + runtime.LockOSThread() + flag.Parse() + + src := flag.Arg(0) + var root string + if len(flag.Args()) > 1 { + root = flag.Arg(1) + } + + if root == "" { + root = src + } + + if err := realChroot(root); err != nil { + fatal(err) + } + + var options archive.TarOptions + if err := json.NewDecoder(os.Stdin).Decode(&options); err != nil { + fatal(err) + } + + rdr, err := archive.TarWithOptions(src, &options) + if err != nil { + fatal(err) + } + defer rdr.Close() + + if _, err := io.Copy(os.Stdout, rdr); err != nil { + fatal(err) + } + + os.Exit(0) +} + +func invokePack(srcPath string, options *archive.TarOptions, root string) (io.ReadCloser, error) { + if root == "" { + return nil, errors.New("root path must not be empty") + } + + relSrc, err := filepath.Rel(root, srcPath) + if err != nil { + return nil, err + } + if relSrc == "." { + relSrc = "/" + } + if relSrc[0] != '/' { + relSrc = "/" + relSrc + } + + // make sure we didn't trim a trailing slash with the call to `Rel` + if strings.HasSuffix(srcPath, "/") && !strings.HasSuffix(relSrc, "/") { + relSrc += "/" + } + + cmd := reexec.Command("storage-tar", relSrc, root) + + errBuff := bytes.NewBuffer(nil) + cmd.Stderr = errBuff + + tarR, tarW := io.Pipe() + cmd.Stdout = tarW + + stdin, err := cmd.StdinPipe() + if err != nil { + return nil, errors.Wrap(err, "error getting options pipe for tar process") + } + + if err := cmd.Start(); err != nil { + return nil, errors.Wrap(err, "tar error on re-exec cmd") + } + + go func() { + err := cmd.Wait() + err = errors.Wrapf(err, "error processing tar file: %s", errBuff) + tarW.CloseWithError(err) + }() + + if err := json.NewEncoder(stdin).Encode(options); err != nil { + stdin.Close() + return nil, errors.Wrap(err, "tar json encode to pipe failed") + } + stdin.Close() + + return tarR, nil +} diff --git a/vendor/github.com/containers/storage/pkg/chrootarchive/archive_windows.go b/vendor/github.com/containers/storage/pkg/chrootarchive/archive_windows.go index 93fde4220..8a5c680b1 100644 --- a/vendor/github.com/containers/storage/pkg/chrootarchive/archive_windows.go +++ b/vendor/github.com/containers/storage/pkg/chrootarchive/archive_windows.go @@ -14,9 +14,16 @@ func chroot(path string) error { func invokeUnpack(decompressedArchive io.ReadCloser, dest string, - options *archive.TarOptions) error { + options *archive.TarOptions, root string) error { // Windows is different to Linux here because Windows does not support // chroot. Hence there is no point sandboxing a chrooted process to // do the unpack. We call inline instead within the daemon process. return archive.Unpack(decompressedArchive, longpath.AddPrefix(dest), options) } + +func invokePack(srcPath string, options *archive.TarOptions, root string) (io.ReadCloser, error) { + // Windows is different to Linux here because Windows does not support + // chroot. Hence there is no point sandboxing a chrooted process to + // do the pack. We call inline instead within the daemon process. + return archive.TarWithOptions(srcPath, options) +} diff --git a/vendor/github.com/containers/storage/pkg/chrootarchive/chroot_unix.go b/vendor/github.com/containers/storage/pkg/chrootarchive/chroot_unix.go index f9b5dece8..83278ee50 100644 --- a/vendor/github.com/containers/storage/pkg/chrootarchive/chroot_unix.go +++ b/vendor/github.com/containers/storage/pkg/chrootarchive/chroot_unix.go @@ -4,9 +4,13 @@ package chrootarchive import "golang.org/x/sys/unix" -func chroot(path string) error { +func realChroot(path string) error { if err := unix.Chroot(path); err != nil { return err } return unix.Chdir("/") } + +func chroot(path string) error { + return realChroot(path) +} diff --git a/vendor/github.com/containers/storage/pkg/chrootarchive/init_unix.go b/vendor/github.com/containers/storage/pkg/chrootarchive/init_unix.go index 21cd87992..ea08135e4 100644 --- a/vendor/github.com/containers/storage/pkg/chrootarchive/init_unix.go +++ b/vendor/github.com/containers/storage/pkg/chrootarchive/init_unix.go @@ -14,6 +14,7 @@ import ( func init() { reexec.Register("storage-applyLayer", applyLayer) reexec.Register("storage-untar", untar) + reexec.Register("storage-tar", tar) } func fatal(err error) { diff --git a/vendor/github.com/containers/storage/pkg/system/stat_unix.go b/vendor/github.com/containers/storage/pkg/system/stat_unix.go index 91c7d121c..f9a1b4877 100644 --- a/vendor/github.com/containers/storage/pkg/system/stat_unix.go +++ b/vendor/github.com/containers/storage/pkg/system/stat_unix.go @@ -58,3 +58,15 @@ func Stat(path string) (*StatT, error) { } return fromStatT(s) } + +// Fstat takes an open file descriptor and returns +// a system.StatT type pertaining to that file. +// +// Throws an error if the file descriptor is invalid +func Fstat(fd int) (*StatT, error) { + s := &syscall.Stat_t{} + if err := syscall.Fstat(fd, s); err != nil { + return nil, err + } + return fromStatT(s) +} diff --git a/vendor/github.com/coreos/go-systemd/NOTICE b/vendor/github.com/coreos/go-systemd/NOTICE new file mode 100644 index 000000000..23a0ada2f --- /dev/null +++ b/vendor/github.com/coreos/go-systemd/NOTICE @@ -0,0 +1,5 @@ +CoreOS Project +Copyright 2018 CoreOS, Inc + +This product includes software developed at CoreOS, Inc. +(http://www.coreos.com/). diff --git a/vendor/github.com/coreos/go-systemd/README.md b/vendor/github.com/coreos/go-systemd/README.md index cb87a1124..cad04a803 100644 --- a/vendor/github.com/coreos/go-systemd/README.md +++ b/vendor/github.com/coreos/go-systemd/README.md @@ -6,9 +6,11 @@ Go bindings to systemd. The project has several packages: - `activation` - for writing and using socket activation from Go +- `daemon` - for notifying systemd of service status changes - `dbus` - for starting/stopping/inspecting running services and units - `journal` - for writing to systemd's logging service, journald - `sdjournal` - for reading from journald by wrapping its C API +- `login1` - for integration with the systemd logind API - `machine1` - for registering machines/containers with systemd - `unit` - for (de)serialization and comparison of unit files @@ -18,10 +20,9 @@ An example HTTP server using socket activation can be quickly set up by followin https://github.com/coreos/go-systemd/tree/master/examples/activation/httpserver -## Journal +## systemd Service Notification -Using the pure-Go `journal` package you can submit journal entries directly to systemd's journal, taking advantage of features like indexed key/value pairs for each log entry. -The `sdjournal` package provides read access to the journal by wrapping around journald's native C API; consequently it requires cgo and the journal headers to be available. +The `daemon` package is an implementation of the [sd_notify protocol](https://www.freedesktop.org/software/systemd/man/sd_notify.html#Description). It can be used to inform systemd of service start-up completion, watchdog events, and other status changes. ## D-Bus @@ -45,6 +46,20 @@ Create `/etc/dbus-1/system-local.conf` that looks like this: </busconfig> ``` +## Journal + +### Writing to the Journal + +Using the pure-Go `journal` package you can submit journal entries directly to systemd's journal, taking advantage of features like indexed key/value pairs for each log entry. + +### Reading from the Journal + +The `sdjournal` package provides read access to the journal by wrapping around journald's native C API; consequently it requires cgo and the journal headers to be available. + +## logind + +The `login1` package provides functions to integrate with the [systemd logind API](http://www.freedesktop.org/wiki/Software/systemd/logind/). + ## machined The `machine1` package allows interaction with the [systemd machined D-Bus API](http://www.freedesktop.org/wiki/Software/systemd/machined/). diff --git a/vendor/github.com/coreos/go-systemd/activation/files.go b/vendor/github.com/coreos/go-systemd/activation/files.go index c8e85fcd5..29dd18def 100644 --- a/vendor/github.com/coreos/go-systemd/activation/files.go +++ b/vendor/github.com/coreos/go-systemd/activation/files.go @@ -18,18 +18,26 @@ package activation import ( "os" "strconv" + "strings" "syscall" ) -// based on: https://gist.github.com/alberts/4640792 const ( + // listenFdsStart corresponds to `SD_LISTEN_FDS_START`. listenFdsStart = 3 ) +// Files returns a slice containing a `os.File` object for each +// file descriptor passed to this process via systemd fd-passing protocol. +// +// The order of the file descriptors is preserved in the returned slice. +// `unsetEnv` is typically set to `true` in order to avoid clashes in +// fd usage and to avoid leaking environment flags to child processes. func Files(unsetEnv bool) []*os.File { if unsetEnv { defer os.Unsetenv("LISTEN_PID") defer os.Unsetenv("LISTEN_FDS") + defer os.Unsetenv("LISTEN_FDNAMES") } pid, err := strconv.Atoi(os.Getenv("LISTEN_PID")) @@ -42,10 +50,17 @@ func Files(unsetEnv bool) []*os.File { return nil } + names := strings.Split(os.Getenv("LISTEN_FDNAMES"), ":") + files := make([]*os.File, 0, nfds) for fd := listenFdsStart; fd < listenFdsStart+nfds; fd++ { syscall.CloseOnExec(fd) - files = append(files, os.NewFile(uintptr(fd), "LISTEN_FD_"+strconv.Itoa(fd))) + name := "LISTEN_FD_" + strconv.Itoa(fd) + offset := fd - listenFdsStart + if offset < len(names) && len(names[offset]) > 0 { + name = names[offset] + } + files = append(files, os.NewFile(uintptr(fd), name)) } return files diff --git a/vendor/github.com/coreos/go-systemd/activation/listeners.go b/vendor/github.com/coreos/go-systemd/activation/listeners.go index fd5dfc709..bb5cc2311 100644 --- a/vendor/github.com/coreos/go-systemd/activation/listeners.go +++ b/vendor/github.com/coreos/go-systemd/activation/listeners.go @@ -25,13 +25,33 @@ import ( // The order of the file descriptors is preserved in the returned slice. // Nil values are used to fill any gaps. For example if systemd were to return file descriptors // corresponding with "udp, tcp, tcp", then the slice would contain {nil, net.Listener, net.Listener} -func Listeners(unsetEnv bool) ([]net.Listener, error) { - files := Files(unsetEnv) +func Listeners() ([]net.Listener, error) { + files := Files(true) listeners := make([]net.Listener, len(files)) for i, f := range files { if pc, err := net.FileListener(f); err == nil { listeners[i] = pc + f.Close() + } + } + return listeners, nil +} + +// ListenersWithNames maps a listener name to a set of net.Listener instances. +func ListenersWithNames() (map[string][]net.Listener, error) { + files := Files(true) + listeners := map[string][]net.Listener{} + + for _, f := range files { + if pc, err := net.FileListener(f); err == nil { + current, ok := listeners[f.Name()] + if !ok { + listeners[f.Name()] = []net.Listener{pc} + } else { + listeners[f.Name()] = append(current, pc) + } + f.Close() } } return listeners, nil @@ -40,8 +60,8 @@ func Listeners(unsetEnv bool) ([]net.Listener, error) { // TLSListeners returns a slice containing a net.listener for each matching TCP socket type // passed to this process. // It uses default Listeners func and forces TCP sockets handlers to use TLS based on tlsConfig. -func TLSListeners(unsetEnv bool, tlsConfig *tls.Config) ([]net.Listener, error) { - listeners, err := Listeners(unsetEnv) +func TLSListeners(tlsConfig *tls.Config) ([]net.Listener, error) { + listeners, err := Listeners() if listeners == nil || err != nil { return nil, err @@ -58,3 +78,26 @@ func TLSListeners(unsetEnv bool, tlsConfig *tls.Config) ([]net.Listener, error) return listeners, err } + +// TLSListenersWithNames maps a listener name to a net.Listener with +// the associated TLS configuration. +func TLSListenersWithNames(tlsConfig *tls.Config) (map[string][]net.Listener, error) { + listeners, err := ListenersWithNames() + + if listeners == nil || err != nil { + return nil, err + } + + if tlsConfig != nil && err == nil { + for _, ll := range listeners { + // Activate TLS only for TCP sockets + for i, l := range ll { + if l.Addr().Network() == "tcp" { + ll[i] = tls.NewListener(l, tlsConfig) + } + } + } + } + + return listeners, err +} diff --git a/vendor/github.com/coreos/go-systemd/activation/packetconns.go b/vendor/github.com/coreos/go-systemd/activation/packetconns.go index 48b2ca029..a97206785 100644 --- a/vendor/github.com/coreos/go-systemd/activation/packetconns.go +++ b/vendor/github.com/coreos/go-systemd/activation/packetconns.go @@ -24,13 +24,14 @@ import ( // The order of the file descriptors is preserved in the returned slice. // Nil values are used to fill any gaps. For example if systemd were to return file descriptors // corresponding with "udp, tcp, udp", then the slice would contain {net.PacketConn, nil, net.PacketConn} -func PacketConns(unsetEnv bool) ([]net.PacketConn, error) { - files := Files(unsetEnv) +func PacketConns() ([]net.PacketConn, error) { + files := Files(true) conns := make([]net.PacketConn, len(files)) for i, f := range files { if pc, err := net.FilePacketConn(f); err == nil { conns[i] = pc + f.Close() } } return conns, nil diff --git a/vendor/github.com/coreos/go-systemd/dbus/dbus.go b/vendor/github.com/coreos/go-systemd/dbus/dbus.go index c1694fb52..1d54810af 100644 --- a/vendor/github.com/coreos/go-systemd/dbus/dbus.go +++ b/vendor/github.com/coreos/go-systemd/dbus/dbus.go @@ -16,6 +16,7 @@ package dbus import ( + "encoding/hex" "fmt" "os" "strconv" @@ -60,6 +61,27 @@ func PathBusEscape(path string) string { return string(n) } +// pathBusUnescape is the inverse of PathBusEscape. +func pathBusUnescape(path string) string { + if path == "_" { + return "" + } + n := []byte{} + for i := 0; i < len(path); i++ { + c := path[i] + if c == '_' && i+2 < len(path) { + res, err := hex.DecodeString(path[i+1 : i+3]) + if err == nil { + n = append(n, res...) + } + i += 2 + } else { + n = append(n, c) + } + } + return string(n) +} + // Conn is a connection to systemd's dbus endpoint. type Conn struct { // sysconn/sysobj are only used to call dbus methods @@ -74,13 +96,18 @@ type Conn struct { jobs map[dbus.ObjectPath]chan<- string sync.Mutex } - subscriber struct { + subStateSubscriber struct { updateCh chan<- *SubStateUpdate errCh chan<- error sync.Mutex ignore map[dbus.ObjectPath]int64 cleanIgnore int64 } + propertiesSubscriber struct { + updateCh chan<- *PropertiesUpdate + errCh chan<- error + sync.Mutex + } } // New establishes a connection to any available bus and authenticates. @@ -152,7 +179,7 @@ func NewConnection(dialBus func() (*dbus.Conn, error)) (*Conn, error) { sigobj: systemdObject(sigconn), } - c.subscriber.ignore = make(map[dbus.ObjectPath]int64) + c.subStateSubscriber.ignore = make(map[dbus.ObjectPath]int64) c.jobListener.jobs = make(map[dbus.ObjectPath]chan<- string) // Setup the listeners on jobs so that we can get completions diff --git a/vendor/github.com/coreos/go-systemd/dbus/methods.go b/vendor/github.com/coreos/go-systemd/dbus/methods.go index ab17f7cc7..0b4207229 100644 --- a/vendor/github.com/coreos/go-systemd/dbus/methods.go +++ b/vendor/github.com/coreos/go-systemd/dbus/methods.go @@ -1,4 +1,4 @@ -// Copyright 2015 CoreOS, Inc. +// Copyright 2015, 2018 CoreOS, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -16,6 +16,7 @@ package dbus import ( "errors" + "fmt" "path" "strconv" @@ -148,14 +149,27 @@ func (c *Conn) ResetFailedUnit(name string) error { return c.sysobj.Call("org.freedesktop.systemd1.Manager.ResetFailedUnit", 0, name).Store() } -// getProperties takes the unit name and returns all of its dbus object properties, for the given dbus interface -func (c *Conn) getProperties(unit string, dbusInterface string) (map[string]interface{}, error) { +// SystemState returns the systemd state. Equivalent to `systemctl is-system-running`. +func (c *Conn) SystemState() (*Property, error) { + var err error + var prop dbus.Variant + + obj := c.sysconn.Object("org.freedesktop.systemd1", "/org/freedesktop/systemd1") + err = obj.Call("org.freedesktop.DBus.Properties.Get", 0, "org.freedesktop.systemd1.Manager", "SystemState").Store(&prop) + if err != nil { + return nil, err + } + + return &Property{Name: "SystemState", Value: prop}, nil +} + +// getProperties takes the unit path and returns all of its dbus object properties, for the given dbus interface +func (c *Conn) getProperties(path dbus.ObjectPath, dbusInterface string) (map[string]interface{}, error) { var err error var props map[string]dbus.Variant - path := unitPath(unit) if !path.IsValid() { - return nil, errors.New("invalid unit name: " + unit) + return nil, fmt.Errorf("invalid unit name: %v", path) } obj := c.sysconn.Object("org.freedesktop.systemd1", path) @@ -172,9 +186,15 @@ func (c *Conn) getProperties(unit string, dbusInterface string) (map[string]inte return out, nil } -// GetUnitProperties takes the unit name and returns all of its dbus object properties. +// GetUnitProperties takes the (unescaped) unit name and returns all of its dbus object properties. func (c *Conn) GetUnitProperties(unit string) (map[string]interface{}, error) { - return c.getProperties(unit, "org.freedesktop.systemd1.Unit") + path := unitPath(unit) + return c.getProperties(path, "org.freedesktop.systemd1.Unit") +} + +// GetUnitProperties takes the (escaped) unit path and returns all of its dbus object properties. +func (c *Conn) GetUnitPathProperties(path dbus.ObjectPath) (map[string]interface{}, error) { + return c.getProperties(path, "org.freedesktop.systemd1.Unit") } func (c *Conn) getProperty(unit string, dbusInterface string, propertyName string) (*Property, error) { @@ -208,7 +228,8 @@ func (c *Conn) GetServiceProperty(service string, propertyName string) (*Propert // Valid values for unitType: Service, Socket, Target, Device, Mount, Automount, Snapshot, Timer, Swap, Path, Slice, Scope // return "dbus.Error: Unknown interface" if the unitType is not the correct type of the unit func (c *Conn) GetUnitTypeProperties(unit string, unitType string) (map[string]interface{}, error) { - return c.getProperties(unit, "org.freedesktop.systemd1."+unitType) + path := unitPath(unit) + return c.getProperties(path, "org.freedesktop.systemd1."+unitType) } // SetUnitProperties() may be used to modify certain unit properties at runtime. @@ -292,6 +313,7 @@ func (c *Conn) ListUnitsByPatterns(states []string, patterns []string) ([]UnitSt // names and returns an UnitStatus array. Comparing to ListUnitsByPatterns // method, this method returns statuses even for inactive or non-existing // units. Input array should contain exact unit names, but not patterns. +// Note: Requires systemd v230 or higher func (c *Conn) ListUnitsByNames(units []string) ([]UnitStatus, error) { return c.listUnitsInternal(c.sysobj.Call("org.freedesktop.systemd1.Manager.ListUnitsByNames", 0, units).Store) } @@ -563,3 +585,8 @@ func (c *Conn) Reload() error { func unitPath(name string) dbus.ObjectPath { return dbus.ObjectPath("/org/freedesktop/systemd1/unit/" + PathBusEscape(name)) } + +// unitName returns the unescaped base element of the supplied escaped path +func unitName(dpath dbus.ObjectPath) string { + return pathBusUnescape(path.Base(string(dpath))) +} diff --git a/vendor/github.com/coreos/go-systemd/dbus/set.go b/vendor/github.com/coreos/go-systemd/dbus/set.go index f92e6fbed..17c5d4856 100644 --- a/vendor/github.com/coreos/go-systemd/dbus/set.go +++ b/vendor/github.com/coreos/go-systemd/dbus/set.go @@ -36,7 +36,7 @@ func (s *set) Length() int { } func (s *set) Values() (values []string) { - for val, _ := range s.data { + for val := range s.data { values = append(values, val) } return diff --git a/vendor/github.com/coreos/go-systemd/dbus/subscription.go b/vendor/github.com/coreos/go-systemd/dbus/subscription.go index 996451445..70e63a6f1 100644 --- a/vendor/github.com/coreos/go-systemd/dbus/subscription.go +++ b/vendor/github.com/coreos/go-systemd/dbus/subscription.go @@ -16,6 +16,7 @@ package dbus import ( "errors" + "log" "time" "github.com/godbus/dbus" @@ -36,22 +37,12 @@ func (c *Conn) Subscribe() error { c.sigconn.BusObject().Call("org.freedesktop.DBus.AddMatch", 0, "type='signal',interface='org.freedesktop.DBus.Properties',member='PropertiesChanged'") - err := c.sigobj.Call("org.freedesktop.systemd1.Manager.Subscribe", 0).Store() - if err != nil { - return err - } - - return nil + return c.sigobj.Call("org.freedesktop.systemd1.Manager.Subscribe", 0).Store() } // Unsubscribe this connection from systemd dbus events. func (c *Conn) Unsubscribe() error { - err := c.sigobj.Call("org.freedesktop.systemd1.Manager.Unsubscribe", 0).Store() - if err != nil { - return err - } - - return nil + return c.sigobj.Call("org.freedesktop.systemd1.Manager.Unsubscribe", 0).Store() } func (c *Conn) dispatch() { @@ -70,7 +61,8 @@ func (c *Conn) dispatch() { c.jobComplete(signal) } - if c.subscriber.updateCh == nil { + if c.subStateSubscriber.updateCh == nil && + c.propertiesSubscriber.updateCh == nil { continue } @@ -84,6 +76,12 @@ func (c *Conn) dispatch() { case "org.freedesktop.DBus.Properties.PropertiesChanged": if signal.Body[0].(string) == "org.freedesktop.systemd1.Unit" { unitPath = signal.Path + + if len(signal.Body) >= 2 { + if changed, ok := signal.Body[1].(map[string]dbus.Variant); ok { + c.sendPropertiesUpdate(unitPath, changed) + } + } } } @@ -169,42 +167,80 @@ type SubStateUpdate struct { // is full, it attempts to write an error to errCh; if errCh is full, the error // passes silently. func (c *Conn) SetSubStateSubscriber(updateCh chan<- *SubStateUpdate, errCh chan<- error) { - c.subscriber.Lock() - defer c.subscriber.Unlock() - c.subscriber.updateCh = updateCh - c.subscriber.errCh = errCh + if c == nil { + msg := "nil receiver" + select { + case errCh <- errors.New(msg): + default: + log.Printf("full error channel while reporting: %s\n", msg) + } + return + } + + c.subStateSubscriber.Lock() + defer c.subStateSubscriber.Unlock() + c.subStateSubscriber.updateCh = updateCh + c.subStateSubscriber.errCh = errCh } -func (c *Conn) sendSubStateUpdate(path dbus.ObjectPath) { - c.subscriber.Lock() - defer c.subscriber.Unlock() +func (c *Conn) sendSubStateUpdate(unitPath dbus.ObjectPath) { + c.subStateSubscriber.Lock() + defer c.subStateSubscriber.Unlock() + + if c.subStateSubscriber.updateCh == nil { + return + } - if c.shouldIgnore(path) { + isIgnored := c.shouldIgnore(unitPath) + defer c.cleanIgnore() + if isIgnored { return } - info, err := c.GetUnitProperties(string(path)) + info, err := c.GetUnitPathProperties(unitPath) if err != nil { select { - case c.subscriber.errCh <- err: + case c.subStateSubscriber.errCh <- err: default: + log.Printf("full error channel while reporting: %s\n", err) } + return } + defer c.updateIgnore(unitPath, info) - name := info["Id"].(string) - substate := info["SubState"].(string) + name, ok := info["Id"].(string) + if !ok { + msg := "failed to cast info.Id" + select { + case c.subStateSubscriber.errCh <- errors.New(msg): + default: + log.Printf("full error channel while reporting: %s\n", err) + } + return + } + substate, ok := info["SubState"].(string) + if !ok { + msg := "failed to cast info.SubState" + select { + case c.subStateSubscriber.errCh <- errors.New(msg): + default: + log.Printf("full error channel while reporting: %s\n", msg) + } + return + } update := &SubStateUpdate{name, substate} select { - case c.subscriber.updateCh <- update: + case c.subStateSubscriber.updateCh <- update: default: + msg := "update channel is full" select { - case c.subscriber.errCh <- errors.New("update channel full!"): + case c.subStateSubscriber.errCh <- errors.New(msg): default: + log.Printf("full error channel while reporting: %s\n", msg) } + return } - - c.updateIgnore(path, info) } // The ignore functions work around a wart in the systemd dbus interface. @@ -222,29 +258,76 @@ func (c *Conn) sendSubStateUpdate(path dbus.ObjectPath) { // the properties). func (c *Conn) shouldIgnore(path dbus.ObjectPath) bool { - t, ok := c.subscriber.ignore[path] + t, ok := c.subStateSubscriber.ignore[path] return ok && t >= time.Now().UnixNano() } func (c *Conn) updateIgnore(path dbus.ObjectPath, info map[string]interface{}) { - c.cleanIgnore() + loadState, ok := info["LoadState"].(string) + if !ok { + return + } // unit is unloaded - it will trigger bad systemd dbus behavior - if info["LoadState"].(string) == "not-found" { - c.subscriber.ignore[path] = time.Now().UnixNano() + ignoreInterval + if loadState == "not-found" { + c.subStateSubscriber.ignore[path] = time.Now().UnixNano() + ignoreInterval } } // without this, ignore would grow unboundedly over time func (c *Conn) cleanIgnore() { now := time.Now().UnixNano() - if c.subscriber.cleanIgnore < now { - c.subscriber.cleanIgnore = now + cleanIgnoreInterval + if c.subStateSubscriber.cleanIgnore < now { + c.subStateSubscriber.cleanIgnore = now + cleanIgnoreInterval - for p, t := range c.subscriber.ignore { + for p, t := range c.subStateSubscriber.ignore { if t < now { - delete(c.subscriber.ignore, p) + delete(c.subStateSubscriber.ignore, p) } } } } + +// PropertiesUpdate holds a map of a unit's changed properties +type PropertiesUpdate struct { + UnitName string + Changed map[string]dbus.Variant +} + +// SetPropertiesSubscriber writes to updateCh when any unit's properties +// change. Every property change reported by systemd will be sent; that is, no +// transitions will be "missed" (as they might be with SetSubStateSubscriber). +// However, state changes will only be written to the channel with non-blocking +// writes. If updateCh is full, it attempts to write an error to errCh; if +// errCh is full, the error passes silently. +func (c *Conn) SetPropertiesSubscriber(updateCh chan<- *PropertiesUpdate, errCh chan<- error) { + c.propertiesSubscriber.Lock() + defer c.propertiesSubscriber.Unlock() + c.propertiesSubscriber.updateCh = updateCh + c.propertiesSubscriber.errCh = errCh +} + +// we don't need to worry about shouldIgnore() here because +// sendPropertiesUpdate doesn't call GetProperties() +func (c *Conn) sendPropertiesUpdate(unitPath dbus.ObjectPath, changedProps map[string]dbus.Variant) { + c.propertiesSubscriber.Lock() + defer c.propertiesSubscriber.Unlock() + + if c.propertiesSubscriber.updateCh == nil { + return + } + + update := &PropertiesUpdate{unitName(unitPath), changedProps} + + select { + case c.propertiesSubscriber.updateCh <- update: + default: + msg := "update channel is full" + select { + case c.propertiesSubscriber.errCh <- errors.New(msg): + default: + log.Printf("full error channel while reporting: %s\n", msg) + } + return + } +} diff --git a/vendor/github.com/coreos/go-systemd/journal/journal.go b/vendor/github.com/coreos/go-systemd/journal/journal.go index 7f434990d..ef85a3ba2 100644 --- a/vendor/github.com/coreos/go-systemd/journal/journal.go +++ b/vendor/github.com/coreos/go-systemd/journal/journal.go @@ -103,7 +103,10 @@ func Send(message string, priority Priority, vars map[string]string) error { if !ok { return journalError("can't send file through non-Unix connection") } - unixConn.WriteMsgUnix([]byte{}, rights, nil) + _, _, err = unixConn.WriteMsgUnix([]byte{}, rights, nil) + if err != nil { + return journalError(err.Error()) + } } else if err != nil { return journalError(err.Error()) } @@ -165,7 +168,7 @@ func tempFd() (*os.File, error) { if err != nil { return nil, err } - syscall.Unlink(file.Name()) + err = syscall.Unlink(file.Name()) if err != nil { return nil, err } diff --git a/vendor/github.com/coreos/go-systemd/sdjournal/journal.go b/vendor/github.com/coreos/go-systemd/sdjournal/journal.go index b00d606c1..9f3d92342 100644 --- a/vendor/github.com/coreos/go-systemd/sdjournal/journal.go +++ b/vendor/github.com/coreos/go-systemd/sdjournal/journal.go @@ -47,6 +47,15 @@ package sdjournal // return sd_journal_open_directory(ret, path, flags); // } // +// int +// my_sd_journal_open_files(void *f, sd_journal **ret, const char **paths, int flags) +// { +// int (*sd_journal_open_files)(sd_journal **, const char **, int); +// +// sd_journal_open_files = f; +// return sd_journal_open_files(ret, paths, flags); +// } +// // void // my_sd_journal_close(void *f, sd_journal *j) // { @@ -282,9 +291,19 @@ package sdjournal // sd_journal_restart_unique(j); // } // +// int +// my_sd_journal_get_catalog(void *f, sd_journal *j, char **ret) +// { +// int(*sd_journal_get_catalog)(sd_journal *, char **); +// +// sd_journal_get_catalog = f; +// return sd_journal_get_catalog(j, ret); +// } +// import "C" import ( "bytes" + "errors" "fmt" "strings" "sync" @@ -352,6 +371,12 @@ const ( IndefiniteWait time.Duration = 1<<63 - 1 ) +var ( + // ErrNoTestCursor gets returned when using TestCursor function and cursor + // parameter is not the same as the current cursor position. + ErrNoTestCursor = errors.New("Cursor parameter is not the same as current position") +) + // Journal is a Go wrapper of an sd_journal structure. type Journal struct { cjournal *C.sd_journal @@ -396,8 +421,7 @@ func NewJournal() (j *Journal, err error) { } // NewJournalFromDir returns a new Journal instance pointing to a journal residing -// in a given directory. The supplied path may be relative or absolute; if -// relative, it will be converted to an absolute path before being opened. +// in a given directory. func NewJournalFromDir(path string) (j *Journal, err error) { j = &Journal{} @@ -417,6 +441,32 @@ func NewJournalFromDir(path string) (j *Journal, err error) { return j, nil } +// NewJournalFromFiles returns a new Journal instance pointing to a journals residing +// in a given files. +func NewJournalFromFiles(paths ...string) (j *Journal, err error) { + j = &Journal{} + + sd_journal_open_files, err := getFunction("sd_journal_open_files") + if err != nil { + return nil, err + } + + // by making the slice 1 elem too long, we guarantee it'll be null-terminated + cPaths := make([]*C.char, len(paths)+1) + for idx, path := range paths { + p := C.CString(path) + cPaths[idx] = p + defer C.free(unsafe.Pointer(p)) + } + + r := C.my_sd_journal_open_files(sd_journal_open_files, &j.cjournal, &cPaths[0], 0) + if r < 0 { + return nil, fmt.Errorf("failed to open journals in paths %q: %d", paths, syscall.Errno(-r)) + } + + return j, nil +} + // Close closes a journal opened with NewJournal. func (j *Journal) Close() error { sd_journal_close, err := getFunction("sd_journal_close") @@ -598,7 +648,8 @@ func (j *Journal) getData(field string) (unsafe.Pointer, C.int, error) { } // GetData gets the data object associated with a specific field from the -// current journal entry. +// the journal entry referenced by the last completed Next/Previous function +// call. To call GetData, you must have first called one of these functions. func (j *Journal) GetData(field string) (string, error) { d, l, err := j.getData(field) if err != nil { @@ -609,7 +660,9 @@ func (j *Journal) GetData(field string) (string, error) { } // GetDataValue gets the data object associated with a specific field from the -// current journal entry, returning only the value of the object. +// journal entry referenced by the last completed Next/Previous function call, +// returning only the value of the object. To call GetDataValue, you must first +// have called one of the Next/Previous functions. func (j *Journal) GetDataValue(field string) (string, error) { val, err := j.GetData(field) if err != nil { @@ -620,7 +673,8 @@ func (j *Journal) GetDataValue(field string) (string, error) { } // GetDataBytes gets the data object associated with a specific field from the -// current journal entry. +// journal entry referenced by the last completed Next/Previous function call. +// To call GetDataBytes, you must first have called one of these functions. func (j *Journal) GetDataBytes(field string) ([]byte, error) { d, l, err := j.getData(field) if err != nil { @@ -631,7 +685,9 @@ func (j *Journal) GetDataBytes(field string) ([]byte, error) { } // GetDataValueBytes gets the data object associated with a specific field from the -// current journal entry, returning only the value of the object. +// journal entry referenced by the last completed Next/Previous function call, +// returning only the value of the object. To call GetDataValueBytes, you must first +// have called one of the Next/Previous functions. func (j *Journal) GetDataValueBytes(field string) ([]byte, error) { val, err := j.GetDataBytes(field) if err != nil { @@ -641,9 +697,10 @@ func (j *Journal) GetDataValueBytes(field string) ([]byte, error) { return bytes.SplitN(val, []byte("="), 2)[1], nil } -// GetEntry returns a full representation of a journal entry with -// all key-value pairs of data as well as address fields (cursor, realtime -// timestamp and monotonic timestamp) +// GetEntry returns a full representation of the journal entry referenced by the +// last completed Next/Previous function call, with all key-value pairs of data +// as well as address fields (cursor, realtime timestamp and monotonic timestamp). +// To call GetEntry, you must first have called one of the Next/Previous functions. func (j *Journal) GetEntry() (*JournalEntry, error) { sd_journal_get_realtime_usec, err := getFunction("sd_journal_get_realtime_usec") if err != nil { @@ -731,7 +788,7 @@ func (j *Journal) GetEntry() (*JournalEntry, error) { return entry, nil } -// SetDataThresold sets the data field size threshold for data returned by +// SetDataThreshold sets the data field size threshold for data returned by // GetData. To retrieve the complete data fields this threshold should be // turned off by setting it to 0, so that the library always returns the // complete data objects. @@ -752,8 +809,10 @@ func (j *Journal) SetDataThreshold(threshold uint64) error { return nil } -// GetRealtimeUsec gets the realtime (wallclock) timestamp of the current -// journal entry. +// GetRealtimeUsec gets the realtime (wallclock) timestamp of the journal +// entry referenced by the last completed Next/Previous function call. To +// call GetRealtimeUsec, you must first have called one of the Next/Previous +// functions. func (j *Journal) GetRealtimeUsec() (uint64, error) { var usec C.uint64_t @@ -773,7 +832,10 @@ func (j *Journal) GetRealtimeUsec() (uint64, error) { return uint64(usec), nil } -// GetMonotonicUsec gets the monotonic timestamp of the current journal entry. +// GetMonotonicUsec gets the monotonic timestamp of the journal entry +// referenced by the last completed Next/Previous function call. To call +// GetMonotonicUsec, you must first have called one of the Next/Previous +// functions. func (j *Journal) GetMonotonicUsec() (uint64, error) { var usec C.uint64_t var boot_id C.sd_id128_t @@ -794,7 +856,9 @@ func (j *Journal) GetMonotonicUsec() (uint64, error) { return uint64(usec), nil } -// GetCursor gets the cursor of the current journal entry. +// GetCursor gets the cursor of the last journal entry reeferenced by the +// last completed Next/Previous function call. To call GetCursor, you must +// first have called one of the Next/Previous functions. func (j *Journal) GetCursor() (string, error) { sd_journal_get_cursor, err := getFunction("sd_journal_get_cursor") if err != nil { @@ -836,13 +900,16 @@ func (j *Journal) TestCursor(cursor string) error { if r < 0 { return fmt.Errorf("failed to test to cursor %q: %d", cursor, syscall.Errno(-r)) + } else if r == 0 { + return ErrNoTestCursor } return nil } // SeekHead seeks to the beginning of the journal, i.e. the oldest available -// entry. +// entry. This call must be followed by a call to Next before any call to +// Get* will return data about the first element. func (j *Journal) SeekHead() error { sd_journal_seek_head, err := getFunction("sd_journal_seek_head") if err != nil { @@ -861,7 +928,8 @@ func (j *Journal) SeekHead() error { } // SeekTail may be used to seek to the end of the journal, i.e. the most recent -// available entry. +// available entry. This call must be followed by a call to Next before any +// call to Get* will return data about the last element. func (j *Journal) SeekTail() error { sd_journal_seek_tail, err := getFunction("sd_journal_seek_tail") if err != nil { @@ -880,7 +948,8 @@ func (j *Journal) SeekTail() error { } // SeekRealtimeUsec seeks to the entry with the specified realtime (wallclock) -// timestamp, i.e. CLOCK_REALTIME. +// timestamp, i.e. CLOCK_REALTIME. This call must be followed by a call to +// Next/Previous before any call to Get* will return data about the sought entry. func (j *Journal) SeekRealtimeUsec(usec uint64) error { sd_journal_seek_realtime_usec, err := getFunction("sd_journal_seek_realtime_usec") if err != nil { @@ -898,7 +967,9 @@ func (j *Journal) SeekRealtimeUsec(usec uint64) error { return nil } -// SeekCursor seeks to a concrete journal cursor. +// SeekCursor seeks to a concrete journal cursor. This call must be +// followed by a call to Next/Previous before any call to Get* will return +// data about the sought entry. func (j *Journal) SeekCursor(cursor string) error { sd_journal_seek_cursor, err := getFunction("sd_journal_seek_cursor") if err != nil { @@ -937,7 +1008,7 @@ func (j *Journal) Wait(timeout time.Duration) int { // equivalent hex value. to = 0xffffffffffffffff } else { - to = uint64(time.Now().Add(timeout).Unix() / 1000) + to = uint64(timeout / time.Microsecond) } j.mu.Lock() r := C.my_sd_journal_wait(sd_journal_wait, j.cjournal, C.uint64_t(to)) @@ -1022,3 +1093,28 @@ func (j *Journal) GetUniqueValues(field string) ([]string, error) { return result, nil } + +// GetCatalog retrieves a message catalog entry for the journal entry referenced +// by the last completed Next/Previous function call. To call GetCatalog, you +// must first have called one of these functions. +func (j *Journal) GetCatalog() (string, error) { + sd_journal_get_catalog, err := getFunction("sd_journal_get_catalog") + if err != nil { + return "", err + } + + var c *C.char + + j.mu.Lock() + r := C.my_sd_journal_get_catalog(sd_journal_get_catalog, j.cjournal, &c) + j.mu.Unlock() + defer C.free(unsafe.Pointer(c)) + + if r < 0 { + return "", fmt.Errorf("failed to retrieve catalog entry for current journal entry: %d", syscall.Errno(-r)) + } + + catalog := C.GoString(c) + + return catalog, nil +} diff --git a/vendor/github.com/coreos/go-systemd/sdjournal/read.go b/vendor/github.com/coreos/go-systemd/sdjournal/read.go index b581f03b4..51a060fb5 100644 --- a/vendor/github.com/coreos/go-systemd/sdjournal/read.go +++ b/vendor/github.com/coreos/go-systemd/sdjournal/read.go @@ -21,10 +21,13 @@ import ( "io" "log" "strings" + "sync" "time" ) var ( + // ErrExpired gets returned when the Follow function runs into the + // specified timeout. ErrExpired = errors.New("Timeout expired") ) @@ -44,6 +47,11 @@ type JournalReaderConfig struct { // If not empty, the journal instance will point to a journal residing // in this directory. The supplied path may be relative or absolute. Path string + + // If not nil, Formatter will be used to translate the resulting entries + // into strings. If not set, the default format (timestamp and message field) + // will be used. If Formatter returns an error, Read will stop and return the error. + Formatter func(entry *JournalEntry) (string, error) } // JournalReader is an io.ReadCloser which provides a simple interface for iterating through the @@ -51,12 +59,20 @@ type JournalReaderConfig struct { type JournalReader struct { journal *Journal msgReader *strings.Reader + formatter func(entry *JournalEntry) (string, error) } // NewJournalReader creates a new JournalReader with configuration options that are similar to the // systemd journalctl tool's iteration and filtering features. func NewJournalReader(config JournalReaderConfig) (*JournalReader, error) { - r := &JournalReader{} + // use simpleMessageFormatter as default formatter. + if config.Formatter == nil { + config.Formatter = simpleMessageFormatter + } + + r := &JournalReader{ + formatter: config.Formatter, + } // Open the journal var err error @@ -71,7 +87,9 @@ func NewJournalReader(config JournalReaderConfig) (*JournalReader, error) { // Add any supplied matches for _, m := range config.Matches { - r.journal.AddMatch(m.String()) + if err = r.journal.AddMatch(m.String()); err != nil { + return nil, err + } } // Set the start position based on options @@ -118,14 +136,10 @@ func NewJournalReader(config JournalReaderConfig) (*JournalReader, error) { // don't fit in the read buffer. Callers should keep calling until 0 and/or an // error is returned. func (r *JournalReader) Read(b []byte) (int, error) { - var err error - if r.msgReader == nil { - var c uint64 - // Advance the journal cursor. It has to be called at least one time // before reading - c, err = r.journal.Next() + c, err := r.journal.Next() // An unexpected error if err != nil { @@ -137,10 +151,13 @@ func (r *JournalReader) Read(b []byte) (int, error) { return 0, io.EOF } - // Build a message - var msg string - msg, err = r.buildMessage() + entry, err := r.journal.GetEntry() + if err != nil { + return 0, err + } + // Build a message + msg, err := r.formatter(entry) if err != nil { return 0, err } @@ -148,8 +165,7 @@ func (r *JournalReader) Read(b []byte) (int, error) { } // Copy and return the message - var sz int - sz, err = r.msgReader.Read(b) + sz, err := r.msgReader.Read(b) if err == io.EOF { // The current entry has been fully read. Don't propagate this // EOF, so the next entry can be read at the next Read() @@ -180,80 +196,76 @@ func (r *JournalReader) Rewind() error { // Follow synchronously follows the JournalReader, writing each new journal entry to writer. The // follow will continue until a single time.Time is received on the until channel. -func (r *JournalReader) Follow(until <-chan time.Time, writer io.Writer) (err error) { +func (r *JournalReader) Follow(until <-chan time.Time, writer io.Writer) error { // Process journal entries and events. Entries are flushed until the tail or // timeout is reached, and then we wait for new events or the timeout. var msg = make([]byte, 64*1<<(10)) + var waitCh = make(chan int, 1) + var waitGroup sync.WaitGroup + defer waitGroup.Wait() + process: for { c, err := r.Read(msg) if err != nil && err != io.EOF { - break process + return err } select { case <-until: return ErrExpired default: - if c > 0 { - if _, err = writer.Write(msg[:c]); err != nil { - break process - } - continue process + } + if c > 0 { + if _, err = writer.Write(msg[:c]); err != nil { + return err } + continue process } // We're at the tail, so wait for new events or time out. // Holds journal events to process. Tightly bounded for now unless there's a // reason to unblock the journal watch routine more quickly. - events := make(chan int, 1) - pollDone := make(chan bool, 1) - go func() { - for { - select { - case <-pollDone: - return + for { + waitGroup.Add(1) + go func() { + status := r.journal.Wait(100 * time.Millisecond) + waitCh <- status + waitGroup.Done() + }() + + select { + case <-until: + return ErrExpired + case e := <-waitCh: + switch e { + case SD_JOURNAL_NOP: + // the journal did not change since the last invocation + case SD_JOURNAL_APPEND, SD_JOURNAL_INVALIDATE: + continue process default: - events <- r.journal.Wait(time.Duration(1) * time.Second) - } - } - }() + if e < 0 { + return fmt.Errorf("received error event: %d", e) + } - select { - case <-until: - pollDone <- true - return ErrExpired - case e := <-events: - pollDone <- true - switch e { - case SD_JOURNAL_NOP, SD_JOURNAL_APPEND, SD_JOURNAL_INVALIDATE: - // TODO: need to account for any of these? - default: - log.Printf("Received unknown event: %d\n", e) + log.Printf("received unknown event: %d\n", e) + } } - continue process } } - - return } -// buildMessage returns a string representing the current journal entry in a simple format which +// simpleMessageFormatter is the default formatter. +// It returns a string representing the current journal entry in a simple format which // includes the entry timestamp and MESSAGE field. -func (r *JournalReader) buildMessage() (string, error) { - var msg string - var usec uint64 - var err error - - if msg, err = r.journal.GetData("MESSAGE"); err != nil { - return "", err - } - - if usec, err = r.journal.GetRealtimeUsec(); err != nil { - return "", err +func simpleMessageFormatter(entry *JournalEntry) (string, error) { + msg, ok := entry.Fields["MESSAGE"] + if !ok { + return "", fmt.Errorf("no MESSAGE field present in journal entry") } + usec := entry.RealtimeTimestamp timestamp := time.Unix(0, int64(usec)*int64(time.Microsecond)) return fmt.Sprintf("%s %s\n", timestamp, msg), nil diff --git a/version/version.go b/version/version.go index a917931b7..c3917c016 100644 --- a/version/version.go +++ b/version/version.go @@ -4,7 +4,7 @@ package version // NOTE: remember to bump the version at the top // of the top-level README.md file when this is // bumped. -const Version = "1.3.2-dev" +const Version = "1.4.1-dev" // RemoteAPIVersion is the version for the remote // client API. It is used to determine compatibility |