diff options
99 files changed, 2685 insertions, 805 deletions
diff --git a/.cirrus.yml b/.cirrus.yml index 28aa528a4..cc645e601 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -223,6 +223,9 @@ meta_task: depends_on: - "gating" + - "vendor" + - "varlink_api" + - "build_each_commit" container: image: "quay.io/libpod/imgts:latest" # see contrib/imgts @@ -253,8 +256,8 @@ testing_task: depends_on: - "gating" - - "varlink_api" - "vendor" + - "varlink_api" - "build_each_commit" # Only test build cache-images, if that's what's requested @@ -264,7 +267,7 @@ testing_task: matrix: # Images are generated separately, from build_images_task (below) image_name: "${FEDORA_CACHE_IMAGE_NAME}" - image_name: "${PRIOR_FEDORA_CACHE_IMAGE_NAME}" + #image_name: "${PRIOR_FEDORA_CACHE_IMAGE_NAME}" image_name: "${UBUNTU_CACHE_IMAGE_NAME}" timeout_in: 120m @@ -352,32 +355,6 @@ special_testing_in_podman_task: failed_audit_log_script: 'cat /var/log/audit/audit.log || cat /var/log/kern.log || echo "Uh oh, cat audit.log failed"' failed_journalctl_b_script: 'journalctl -b || echo "Uh oh, journalctl -b failed"' -# Because system tests are stored within the repository, it is sometimes -# necessary to execute them within a PR to validate changes. -optional_testing_task: - - depends_on: - - "gating" - - # Only run system tests in PRs (not on merge) if magic string is present - # in the PR description. Post-merge system testing is assumed to happen - # later from OS distribution's build systems. - only_if: >- - $CIRRUS_BRANCH != 'master' && - $CIRRUS_CHANGE_MESSAGE !=~ '.*\*\*\*\s*CIRRUS:\s*TEST\s*IMAGES\s*\*\*\*.*' && - $CIRRUS_CHANGE_MESSAGE =~ '.*\*\*\*\s*CIRRUS:\s*SYSTEM\s*TEST\s*\*\*\*.*' - - gce_instance: - matrix: - image_name: "${FEDORA_CACHE_IMAGE_NAME}" - image_name: "${PRIOR_FEDORA_CACHE_IMAGE_NAME}" - image_name: "${UBUNTU_CACHE_IMAGE_NAME}" - - timeout_in: 60m - - setup_environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}' - system_test_script: '$SCRIPT_BASE/system_test.sh |& ${TIMESTAMP}' - # Test building of new cache-images for future PR testing, in this PR. test_build_cache_images_task: @@ -424,7 +401,7 @@ verify_test_built_images_task: gce_instance: matrix: # Images are generated separately, from build_images_task (below) - image_name: "fedora-28${BUILT_IMAGE_SUFFIX}" + #image_name: "fedora-28${BUILT_IMAGE_SUFFIX}" image_name: "fedora-29${BUILT_IMAGE_SUFFIX}" image_name: "ubuntu-18${BUILT_IMAGE_SUFFIX}" @@ -508,12 +485,13 @@ success_task: depends_on: # ignores any dependent task conditions - "gating" - - "build_each_commit" - "vendor" - "varlink_api" + - "build_each_commit" + - "meta" - "testing" - - "special_testing" - - "optional_testing" + - "special_testing_rootless" + - "special_testing_in_podman" - "test_build_cache_images" - "verify_test_built_images" - "build_cache_images" @@ -11,7 +11,7 @@ in the [API.md](https://github.com/containers/libpod/blob/master/API.md) file in [func BuildImageHierarchyMap(name: string) string](#BuildImageHierarchyMap) -[func Commit(name: string, image_name: string, changes: []string, author: string, message: string, pause: bool, manifestType: string) string](#Commit) +[func Commit(name: string, image_name: string, changes: []string, author: string, message: string, pause: bool, manifestType: string) MoreResponse](#Commit) [func ContainerArtifacts(name: string, artifactName: string) string](#ContainerArtifacts) @@ -308,14 +308,14 @@ BuildImageHierarchyMap is for the development of Podman and should not be used. ### <a name="Commit"></a>func Commit <div style="background-color: #E8E8E8; padding: 15px; margin: 10px; border-radius: 10px;"> -method Commit(name: [string](https://godoc.org/builtin#string), image_name: [string](https://godoc.org/builtin#string), changes: [[]string](#[]string), author: [string](https://godoc.org/builtin#string), message: [string](https://godoc.org/builtin#string), pause: [bool](https://godoc.org/builtin#bool), manifestType: [string](https://godoc.org/builtin#string)) [string](https://godoc.org/builtin#string)</div> +method Commit(name: [string](https://godoc.org/builtin#string), image_name: [string](https://godoc.org/builtin#string), changes: [[]string](#[]string), author: [string](https://godoc.org/builtin#string), message: [string](https://godoc.org/builtin#string), pause: [bool](https://godoc.org/builtin#bool), manifestType: [string](https://godoc.org/builtin#string)) [MoreResponse](#MoreResponse)</div> Commit, creates an image from an existing container. It requires the name or ID of the container as well as the resulting image name. Optionally, you can define an author and message to be added to the resulting image. You can also define changes to the resulting image for the following attributes: _CMD, ENTRYPOINT, ENV, EXPOSE, LABEL, ONBUILD, STOPSIGNAL, USER, VOLUME, and WORKDIR_. To pause the container while it is being committed, pass a _true_ bool for the pause argument. If the container cannot be found by the ID or name provided, a (ContainerNotFound)[#ContainerNotFound] error will be returned; otherwise, -the resulting image's ID will be returned as a string. +the resulting image's ID will be returned as a string inside a MoreResponse. ### <a name="ContainerArtifacts"></a>func ContainerArtifacts <div style="background-color: #E8E8E8; padding: 15px; margin: 10px; border-radius: 10px;"> @@ -1,6 +1,6 @@ GO ?= go DESTDIR ?= / -EPOCH_TEST_COMMIT ?= 8161802f7df857e0850f842261079c83290f9891 +EPOCH_TEST_COMMIT ?= 1f31892a9fd8573d4b25274b208e6b9f860cdf81 HEAD ?= HEAD CHANGELOG_BASE ?= HEAD~ CHANGELOG_TARGET ?= HEAD diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 5eb85d0bc..e36757edc 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,5 +1,36 @@ # Release Notes +## 1.4.0 +### Features +- The `podman checkpoint` and `podman restore` commands can now be used to migrate containers between Podman installations on different systems ([#1618](https://github.com/containers/libpod/issues/1618)) +- The `podman cp` command now supports a `pause` flag to pause containers while copying into them +- The remote client now supports a configuration file for pre-configuring connections to remote Podman installations + +### Bugfixes +- Fixed CVE-2019-10152 - The `podman cp` command improperly dereferenced symlinks in host context +- Fixed a bug where `podman commit` could improperly set environment variables that contained `=` characters ([#3132](https://github.com/containers/libpod/issues/3132)) +- Fixed a bug where rootless Podman would sometimes fail to start containers with forwarded ports ([#2942](https://github.com/containers/libpod/issues/2942)) +- Fixed a bug where `podman version` on the remote client could segfault ([#3145](https://github.com/containers/libpod/issues/3145)) +- Fixed a bug where `podman container runlabel` would use `/proc/self/exe` instead of the path of the Podman command when printing the command being executed +- Fixed a bug where filtering images by label did not work ([#3163](https://github.com/containers/libpod/issues/3163)) +- Fixed a bug where specifying a bing mount or tmpfs mount over an image volume would cause a container to be unable to start ([#3174](https://github.com/containers/libpod/issues/3174)) +- Fixed a bug where `podman generate kube` did not work with containers with named volumes +- Fixed a bug where rootless Podman would receive `permission denied` errors accessing `conmon.pid` ([#3187](https://github.com/containers/libpod/issues/3187)) +- Fixed a bug where `podman cp` with a folder specified as target would replace the folder, as opposed to copying into it ([#3184](https://github.com/containers/libpod/issues/3184)) +- Fixed a bug where rootless Podman commands could double-unlock a lock, causing a crash ([#3207](https://github.com/containers/libpod/issues/3207)) +- Fixed a bug where Podman incorrectly set `tmpcopyup` on `/dev/` mounts, causing errors when using the Kata containers runtime ([#3229](https://github.com/containers/libpod/issues/3229)) +- Fixed a bug where `podman exec` would fail on older kernels ([#2968](https://github.com/containers/libpod/issues/2968)) + +### Misc +- The `podman commit` command is now usable with the Podman remote client +- The `--signature-policy` flag (used with several image-related commands) has been deprecated +- The `podman unshare` command now defines two environment variables in the spawned shell: `CONTAINERS_RUNROOT` and `CONTAINERS_GRAPHROOT`, pointing to temporary and permanent storage for rootless containers +- Updated vendored containers/storage and containers/image libraries with numerous bugfixes +- Updated vendored Buildah to v1.8.3 +- Podman now requires [Conmon v0.2.0](https://github.com/containers/conmon/releases/tag/v0.2.0) +- The `podman cp` command is now aliased as `podman container cp` +- Rootless Podman will now default `init_path` using root Podman's configuration files (`/etc/containers/libpod.conf` and `/usr/share/containers/libpod.conf`) if not overridden in the rootless configuration + ## 1.3.1 ### Features - The `podman cp` command can now read input redirected to `STDIN`, and output to `STDOUT` instead of a file, using `-` instead of an argument. diff --git a/changelog.txt b/changelog.txt index ec0a62f26..ece82e15b 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,3 +1,137 @@ +- Changelog for v1.4.0 (2019-06-07) + * Update release notes for v1.4.0 + * Update release notes for v1.4.0 + * Disable a very badly flaking healthcheck test + * rootless: skip NS_GET_PARENT on old kernels + * Cirrus: Track VM Image calling GCE project + * remove -c for podman remote global options + * Vendor Buildah v1.8.3 + * Cirrus: Disable testing on F28 (EOL) + * migration: add possibility to restore a container with a new name + * Inherit rootless init_path from system libpod.conf + * Also download container images during restore + * Include container migration into tutorial + * Add man-pages for container migration + * Added bash completion for container migration + * Add test case for container migration + * Added support to migrate containers + * Added helper functions for container migration + * Fix restore options help text and comments + * fix timing issues with some tests + * pkg/varlinkapi/virtwriter/virtwriter.go: simplify func Reader + * rootless: block signals on re-exec + * cirrus: minor cleanup and refactoring + * manpage: podman-tool table: un-confuse version and varlink + * Create Dockerfiles for podmanimage + * rootless: use TEMP_FAILURE_RETRY macro + * rootless: fix return type + * rootless: make sure the buffer is NUL terminated + * split rootless local and remote testing + * Fix podman cp test by reordering operations + * Small fix to readme to force tests to run + * Do not set tmpcopyup on /dev + * do not run remote tests inside container + * podman remote-client commit + * Fix podman cp tests + * podman-remote.conf enablement + * Error when trying to copy into a running rootless ctr + * rootless: skip check fo /etc/containers/registries.conf + * We can't pause rootless containers during cp + * Fix bug in e2e tests for podman cp + * Tolerate non-running containers in paused cp + * Add test to ensure symlinks are resolved in ctr scope + * Add --pause to podman cp manpage and bash completions + * Pause containers while copying into them + * Use securejoin to merge paths in `podman cp` + * use imagecaches for local tests + * add dns flags to docs + * add missing container cp command + * Podman logs man page shouldn't include timestamps + * Fix the varlink upgraded calls + * hack: support setting local region/zone + * document missing container update command + * Add --follow to journald ctr logging + * Address comments + * Implement podman logs with log-driver journald + * bump go-systemd version + * Added --log-driver and journald logging + * Update completions and docs to use k8s file as log driver + * bump conmon to v0.2.0 + * runtime: unlock the alive lock only once + * rootless: make JoinUserAndMountNS private + * Revert "rootless: change default path for conmon.pid" + * rootless: enable loginctl linger + * rootless: new function to join existing conmon processes + * rootless: block signals for pause + * Update install.md ostree Debian dependencies. + * fix bug dest path of copying tar + * podman: honor env variable PODMAN_USERNS + * userns: add new option --userns=keep-id + * warn when --security-opt and --privileged + * baseline tests: apparmor with --privileged + * rootless: store also the original GID in the host + * Fix a potential flake in the tests for podman cp + * cirrus: update images w/ zip pkg + * Cirrus: Add zip package to images + * rootless: fix top huser and hgroup + * vendor: update psgo to v1.3.0 + * apparmor: don't load/set profile in privileged mode + * hack: ignore from all VCS files when tarballing + * hack: shrink xfer tarball size + * hack: Display IP address of VM from script + * document nullable types + * Add test cases for login and logout + * Remove unused return statement in kube volume code + * Fix play kube when a pod is specified + * Fix a 'generate kube' bug on ctrs with named volumes + * Add test for image volume conflict with user volume + * Cirrus: Fix missing CRIO_COMMIT -> CONMON_COMMIT + * When superceding mounts, check for opposite types + * make remote resize channel buffered + * Cirrus: workaround root expand failure + * Cirrus: Stub in F30 support + * Cirrus: fixups based on review feedback + * Cirrus: Overhaul/Simplify env. var setup + * Cirrus: Run tests on test-built cache-images + * Cirrus: Support testing of VM cache-image changes + * Cirrus: Remove "too new" runc hack + * libpod: prefer WaitForFile to polling + * Remove conmon from fedora install instructions + * rootless: force resources to be nil on cgroup v1 + * Fixup Flags + * Minor fix filtering images by label + * container: move channel close to its writer + * util: fix race condition in WaitForFile + * Update vendor of buildah and containers/images + * Add Jhon Honce (@jwhonce on github) to OWNERS + * Don't set apparmor if --priviliged + * docs/libpod.conf.5: Add "have" to "higher precedence" typo + * Output name of process on runlabel command + * Minor fix splitting env vars in podman-commit + * Fixup conmon documentation + * troubleshooting.md: add note about updating subuid/subgid + * system: migrate stops the pause process + * rootless: join namespace immediately when possible + * rootless: use a pause process + * migrate: not create a new namespace + * install.remote should be separate for install.bin + * Cirrus: Confirm networking is working + * Use containers/conmon + * Fix a typo in release notes, and bump README version + * s|kubernetes-sigs/cri-o|cri-o/cri-o|g + * Bump github.com/containers/storage to v1.12.7 + * remote: version: fix nil dereference + * Bump gitvalidation epoch + * Bump to v1.3.2-dev + * Add connection information to podman-remote info + * unshare: define CONTAINERS_GRAPHROOT and CONTAINERS_RUNROOT + * Touchup run man page + * unshare: use rootless from libpod + * Replace root-based rootless tests + * rootless: default --cgroup-manager=systemd in unified mode + * create: skip resources validation with cgroup v2 + * rootless, spec: allow resources with cgroup v2 + - Changelog for v1.3.1 (2019-05-16) * More release notes * Add unshare to podman diff --git a/cmd/podman/checkpoint.go b/cmd/podman/checkpoint.go index 234d683bb..86bc8b973 100644 --- a/cmd/podman/checkpoint.go +++ b/cmd/podman/checkpoint.go @@ -46,6 +46,7 @@ func init() { flags.BoolVar(&checkpointCommand.TcpEstablished, "tcp-established", false, "Checkpoint a container with established TCP connections") flags.BoolVarP(&checkpointCommand.All, "all", "a", false, "Checkpoint all running containers") flags.BoolVarP(&checkpointCommand.Latest, "latest", "l", false, "Act on the latest container podman is aware of") + flags.StringVarP(&checkpointCommand.Export, "export", "e", "", "Export the checkpoint image to a tar.gz") markFlagHiddenForRemoteClient("latest", flags) } @@ -64,6 +65,7 @@ func checkpointCmd(c *cliconfig.CheckpointValues) error { Keep: c.Keep, KeepRunning: c.LeaveRunning, TCPEstablished: c.TcpEstablished, + TargetFile: c.Export, } return runtime.Checkpoint(c, options) } diff --git a/cmd/podman/cliconfig/config.go b/cmd/podman/cliconfig/config.go index aaa4513d8..b8b1648b8 100644 --- a/cmd/podman/cliconfig/config.go +++ b/cmd/podman/cliconfig/config.go @@ -33,9 +33,11 @@ type MainFlags struct { LogLevel string TmpDir string - RemoteUserName string - RemoteHost string - VarlinkAddress string + RemoteUserName string + RemoteHost string + VarlinkAddress string + ConnectionName string + RemoteConfigFilePath string } type AttachValues struct { @@ -89,6 +91,7 @@ type CheckpointValues struct { TcpEstablished bool All bool Latest bool + Export string } type CommitValues struct { @@ -426,6 +429,8 @@ type RestoreValues struct { Keep bool Latest bool TcpEstablished bool + Import string + Name string } type RmValues struct { diff --git a/cmd/podman/commands.go b/cmd/podman/commands.go index 2ac465b9d..18b0b7857 100644 --- a/cmd/podman/commands.go +++ b/cmd/podman/commands.go @@ -11,7 +11,6 @@ const remoteclient = false // Commands that the local client implements func getMainCommands() []*cobra.Command { rootCommands := []*cobra.Command{ - _commitCommand, _execCommand, _playCommand, _loginCommand, @@ -41,7 +40,6 @@ func getContainerSubCommands() []*cobra.Command { return []*cobra.Command{ _cleanupCommand, - _commitCommand, _execCommand, _mountCommand, _refreshCommand, diff --git a/cmd/podman/commit.go b/cmd/podman/commit.go index 2b38bab35..01e2ec701 100644 --- a/cmd/podman/commit.go +++ b/cmd/podman/commit.go @@ -2,16 +2,11 @@ package main import ( "fmt" - "io" - "os" "strings" - "github.com/containers/buildah" - "github.com/containers/image/manifest" "github.com/containers/libpod/cmd/podman/cliconfig" - "github.com/containers/libpod/cmd/podman/libpodruntime" "github.com/containers/libpod/libpod" - "github.com/containers/libpod/libpod/image" + "github.com/containers/libpod/pkg/adapter" "github.com/containers/libpod/pkg/util" "github.com/pkg/errors" "github.com/spf13/cobra" @@ -52,32 +47,17 @@ func init() { } func commitCmd(c *cliconfig.CommitValues) error { - runtime, err := libpodruntime.GetRuntime(getContext(), &c.PodmanCommand) + runtime, err := adapter.GetRuntime(getContext(), &c.PodmanCommand) if err != nil { return errors.Wrapf(err, "could not get runtime") } defer runtime.Shutdown(false) - var ( - writer io.Writer - mimeType string - ) args := c.InputArgs if len(args) != 2 { return errors.Errorf("you must provide a container name or ID and a target image name") } - switch c.Format { - case "oci": - mimeType = buildah.OCIv1ImageManifest - if c.Flag("message").Changed { - return errors.Errorf("messages are only compatible with the docker image format (-f docker)") - } - case "docker": - mimeType = manifest.DockerV2Schema2MediaType - default: - return errors.Errorf("unrecognized image format %q", c.Format) - } container := args[0] reference := args[1] if c.Flag("change").Changed { @@ -92,38 +72,10 @@ func commitCmd(c *cliconfig.CommitValues) error { } } - if !c.Quiet { - writer = os.Stderr - } - ctr, err := runtime.LookupContainer(container) - if err != nil { - return errors.Wrapf(err, "error looking up container %q", container) - } - - rtc, err := runtime.GetConfig() - if err != nil { - return err - } - - sc := image.GetSystemContext(rtc.SignaturePolicyPath, "", false) - coptions := buildah.CommitOptions{ - SignaturePolicyPath: rtc.SignaturePolicyPath, - ReportWriter: writer, - SystemContext: sc, - PreferredManifestType: mimeType, - } - options := libpod.ContainerCommitOptions{ - CommitOptions: coptions, - Pause: c.Pause, - IncludeVolumes: c.IncludeVolumes, - Message: c.Message, - Changes: c.Change, - Author: c.Author, - } - newImage, err := ctr.Commit(getContext(), reference, options) + iid, err := runtime.Commit(getContext(), c, container, reference) if err != nil { return err } - fmt.Println(newImage.ID()) + fmt.Println(iid) return nil } diff --git a/cmd/podman/container.go b/cmd/podman/container.go index 839ae3a0e..cb54317c0 100644 --- a/cmd/podman/container.go +++ b/cmd/podman/container.go @@ -52,6 +52,7 @@ var ( containerCommands = []*cobra.Command{ _attachCommand, _checkpointCommand, + _commitCommand, _containerExistsCommand, _contInspectSubCommand, _cpCommand, diff --git a/cmd/podman/cp.go b/cmd/podman/cp.go index 907bde4b9..7679ebcf1 100644 --- a/cmd/podman/cp.go +++ b/cmd/podman/cp.go @@ -51,7 +51,7 @@ func init() { cpCommand.Command = _cpCommand flags := cpCommand.Flags() flags.BoolVar(&cpCommand.Extract, "extract", false, "Extract the tar file into the destination directory.") - flags.BoolVar(&cpCommand.Pause, "pause", true, "Pause the container while copying") + flags.BoolVar(&cpCommand.Pause, "pause", false, "Pause the container while copying") cpCommand.SetHelpTemplate(HelpTemplate()) cpCommand.SetUsageTemplate(UsageTemplate()) rootCmd.AddCommand(cpCommand.Command) @@ -211,7 +211,7 @@ func copyBetweenHostAndContainer(runtime *libpod.Runtime, src string, dest strin } func getUser(mountPoint string, userspec string) (specs.User, error) { - uid, gid, err := chrootuser.GetUser(mountPoint, userspec) + uid, gid, _, err := chrootuser.GetUser(mountPoint, userspec) u := specs.User{ UID: uid, GID: gid, diff --git a/cmd/podman/main.go b/cmd/podman/main.go index 787dd55c0..a149a47f9 100644 --- a/cmd/podman/main.go +++ b/cmd/podman/main.go @@ -30,6 +30,7 @@ var ( var mainCommands = []*cobra.Command{ _attachCommand, _buildCommand, + _commitCommand, _diffCommand, _createCommand, _eventsCommand, diff --git a/cmd/podman/main_remote.go b/cmd/podman/main_remote.go index c8bb3ad3e..de6c2c760 100644 --- a/cmd/podman/main_remote.go +++ b/cmd/podman/main_remote.go @@ -9,6 +9,8 @@ import ( const remote = true func init() { + rootCmd.PersistentFlags().StringVar(&MainGlobalOpts.ConnectionName, "connection", "", "remote connection name") + rootCmd.PersistentFlags().StringVar(&MainGlobalOpts.RemoteConfigFilePath, "remote-config-path", "", "alternate path for configuration file") rootCmd.PersistentFlags().StringVar(&MainGlobalOpts.RemoteUserName, "username", "", "username on the remote host") rootCmd.PersistentFlags().StringVar(&MainGlobalOpts.RemoteHost, "remote-host", "", "remote host") // TODO maybe we allow the altering of this for bridge connections? diff --git a/cmd/podman/remoteclientconfig/config.go b/cmd/podman/remoteclientconfig/config.go new file mode 100644 index 000000000..01f293ec3 --- /dev/null +++ b/cmd/podman/remoteclientconfig/config.go @@ -0,0 +1,21 @@ +package remoteclientconfig + +const remoteConfigFileName string = "podman-remote.conf" + +// RemoteConfig describes the podman remote configuration file +type RemoteConfig struct { + Connections map[string]RemoteConnection +} + +// RemoteConnection describes the attributes of a podman-remote endpoint +type RemoteConnection struct { + Destination string `toml:"destination"` + Username string `toml:"username"` + IsDefault bool `toml:"default"` +} + +// GetConfigFilePath is a simple helper to export the configuration file's +// path based on arch, etc +func GetConfigFilePath() string { + return getConfigFilePath() +} diff --git a/cmd/podman/remoteclientconfig/config_darwin.go b/cmd/podman/remoteclientconfig/config_darwin.go new file mode 100644 index 000000000..b94941381 --- /dev/null +++ b/cmd/podman/remoteclientconfig/config_darwin.go @@ -0,0 +1,12 @@ +package remoteclientconfig + +import ( + "path/filepath" + + "github.com/docker/docker/pkg/homedir" +) + +func getConfigFilePath() string { + homeDir := homedir.Get() + return filepath.Join(homeDir, ".config", "containers", remoteConfigFileName) +} diff --git a/cmd/podman/remoteclientconfig/config_linux.go b/cmd/podman/remoteclientconfig/config_linux.go new file mode 100644 index 000000000..b94941381 --- /dev/null +++ b/cmd/podman/remoteclientconfig/config_linux.go @@ -0,0 +1,12 @@ +package remoteclientconfig + +import ( + "path/filepath" + + "github.com/docker/docker/pkg/homedir" +) + +func getConfigFilePath() string { + homeDir := homedir.Get() + return filepath.Join(homeDir, ".config", "containers", remoteConfigFileName) +} diff --git a/cmd/podman/remoteclientconfig/config_windows.go b/cmd/podman/remoteclientconfig/config_windows.go new file mode 100644 index 000000000..fa6ffca63 --- /dev/null +++ b/cmd/podman/remoteclientconfig/config_windows.go @@ -0,0 +1,12 @@ +package remoteclientconfig + +import ( + "path/filepath" + + "github.com/docker/docker/pkg/homedir" +) + +func getConfigFilePath() string { + homeDir := homedir.Get() + return filepath.Join(homeDir, "AppData", "podman", remoteConfigFileName) +} diff --git a/cmd/podman/remoteclientconfig/configfile.go b/cmd/podman/remoteclientconfig/configfile.go new file mode 100644 index 000000000..aa3e82a31 --- /dev/null +++ b/cmd/podman/remoteclientconfig/configfile.go @@ -0,0 +1,62 @@ +package remoteclientconfig + +import ( + "io" + + "github.com/BurntSushi/toml" + "github.com/pkg/errors" +) + +// ReadRemoteConfig takes an io.Reader representing the remote configuration +// file and returns a remoteconfig +func ReadRemoteConfig(reader io.Reader) (*RemoteConfig, error) { + var remoteConfig RemoteConfig + // the configuration file does not exist + if reader == nil { + return &remoteConfig, ErrNoConfigationFile + } + _, err := toml.DecodeReader(reader, &remoteConfig) + if err != nil { + return nil, err + } + // We need to validate each remote connection has fields filled out + for name, conn := range remoteConfig.Connections { + if len(conn.Destination) < 1 { + return nil, errors.Errorf("connection %s has no destination defined", name) + } + } + return &remoteConfig, err +} + +// GetDefault returns the default RemoteConnection. If there is only one +// connection, we assume it is the default as well +func (r *RemoteConfig) GetDefault() (*RemoteConnection, error) { + if len(r.Connections) == 0 { + return nil, ErrNoDefinedConnections + } + for _, v := range r.Connections { + if len(r.Connections) == 1 { + // if there is only one defined connection, we assume it is + // the default whether tagged as such or not + return &v, nil + } + if v.IsDefault { + return &v, nil + } + } + return nil, ErrNoDefaultConnection +} + +// GetRemoteConnection "looks up" a remote connection by name and returns it in the +// form of a RemoteConnection +func (r *RemoteConfig) GetRemoteConnection(name string) (*RemoteConnection, error) { + if len(r.Connections) == 0 { + return nil, ErrNoDefinedConnections + } + for k, v := range r.Connections { + if k == name { + return &v, nil + } + } + return nil, errors.Wrap(ErrConnectionNotFound, name) +} diff --git a/cmd/podman/remoteclientconfig/configfile_test.go b/cmd/podman/remoteclientconfig/configfile_test.go new file mode 100644 index 000000000..66e0a4693 --- /dev/null +++ b/cmd/podman/remoteclientconfig/configfile_test.go @@ -0,0 +1,201 @@ +package remoteclientconfig + +import ( + "io" + "reflect" + "strings" + "testing" +) + +var goodConfig = ` +[connections] + +[connections.homer] +destination = "192.168.1.1" +username = "myuser" +default = true + +[connections.bart] +destination = "foobar.com" +username = "root" +` +var noDest = ` +[connections] + +[connections.homer] +destination = "192.168.1.1" +username = "myuser" +default = true + +[connections.bart] +username = "root" +` + +var noUser = ` +[connections] + +[connections.homer] +destination = "192.168.1.1" +` + +func makeGoodResult() *RemoteConfig { + var goodConnections = make(map[string]RemoteConnection) + goodConnections["homer"] = RemoteConnection{ + Destination: "192.168.1.1", + Username: "myuser", + IsDefault: true, + } + goodConnections["bart"] = RemoteConnection{ + Destination: "foobar.com", + Username: "root", + } + var goodResult = RemoteConfig{ + Connections: goodConnections, + } + return &goodResult +} + +func makeNoUserResult() *RemoteConfig { + var goodConnections = make(map[string]RemoteConnection) + goodConnections["homer"] = RemoteConnection{ + Destination: "192.168.1.1", + } + var goodResult = RemoteConfig{ + Connections: goodConnections, + } + return &goodResult +} + +func TestReadRemoteConfig(t *testing.T) { + type args struct { + reader io.Reader + } + tests := []struct { + name string + args args + want *RemoteConfig + wantErr bool + }{ + // good test should pass + {"good", args{reader: strings.NewReader(goodConfig)}, makeGoodResult(), false}, + // a connection with no destination is an error + {"nodest", args{reader: strings.NewReader(noDest)}, nil, true}, + // a connnection with no user is OK + {"nouser", args{reader: strings.NewReader(noUser)}, makeNoUserResult(), false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ReadRemoteConfig(tt.args.reader) + if (err != nil) != tt.wantErr { + t.Errorf("ReadRemoteConfig() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("ReadRemoteConfig() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestRemoteConfig_GetDefault(t *testing.T) { + good := make(map[string]RemoteConnection) + good["homer"] = RemoteConnection{ + Username: "myuser", + Destination: "192.168.1.1", + IsDefault: true, + } + good["bart"] = RemoteConnection{ + Username: "root", + Destination: "foobar.com", + } + noDefault := make(map[string]RemoteConnection) + noDefault["homer"] = RemoteConnection{ + Username: "myuser", + Destination: "192.168.1.1", + } + noDefault["bart"] = RemoteConnection{ + Username: "root", + Destination: "foobar.com", + } + single := make(map[string]RemoteConnection) + single["homer"] = RemoteConnection{ + Username: "myuser", + Destination: "192.168.1.1", + } + + none := make(map[string]RemoteConnection) + + type fields struct { + Connections map[string]RemoteConnection + } + tests := []struct { + name string + fields fields + want *RemoteConnection + wantErr bool + }{ + // A good toml should return the connection that is marked isDefault + {"good", fields{Connections: makeGoodResult().Connections}, &RemoteConnection{"192.168.1.1", "myuser", true}, false}, + // If nothing is marked as isDefault and there is more than one connection, error should occur + {"nodefault", fields{Connections: noDefault}, nil, true}, + // if nothing is marked as isDefault but there is only one connection, the one connection is considered the default + {"single", fields{Connections: none}, nil, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := &RemoteConfig{ + Connections: tt.fields.Connections, + } + got, err := r.GetDefault() + if (err != nil) != tt.wantErr { + t.Errorf("RemoteConfig.GetDefault() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("RemoteConfig.GetDefault() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestRemoteConfig_GetRemoteConnection(t *testing.T) { + type fields struct { + Connections map[string]RemoteConnection + } + type args struct { + name string + } + + blank := make(map[string]RemoteConnection) + tests := []struct { + name string + fields fields + args args + want *RemoteConnection + wantErr bool + }{ + // Good connection + {"goodhomer", fields{Connections: makeGoodResult().Connections}, args{name: "homer"}, &RemoteConnection{"192.168.1.1", "myuser", true}, false}, + // Good connection + {"goodbart", fields{Connections: makeGoodResult().Connections}, args{name: "bart"}, &RemoteConnection{"foobar.com", "root", false}, false}, + // Getting an unknown connection should result in error + {"noexist", fields{Connections: makeGoodResult().Connections}, args{name: "foobar"}, nil, true}, + // Getting a connection when there are none should result in an error + {"none", fields{Connections: blank}, args{name: "foobar"}, nil, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := &RemoteConfig{ + Connections: tt.fields.Connections, + } + got, err := r.GetRemoteConnection(tt.args.name) + if (err != nil) != tt.wantErr { + t.Errorf("RemoteConfig.GetRemoteConnection() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("RemoteConfig.GetRemoteConnection() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/cmd/podman/remoteclientconfig/errors.go b/cmd/podman/remoteclientconfig/errors.go new file mode 100644 index 000000000..2689d3b49 --- /dev/null +++ b/cmd/podman/remoteclientconfig/errors.go @@ -0,0 +1,14 @@ +package remoteclientconfig + +import "errors" + +var ( + // ErrNoDefaultConnection no default connection is defined in the podman-remote.conf file + ErrNoDefaultConnection = errors.New("no default connection is defined") + // ErrNoDefinedConnections no connections are defined in the podman-remote.conf file + ErrNoDefinedConnections = errors.New("no remote connections have been defined") + // ErrConnectionNotFound unable to lookup connection by name + ErrConnectionNotFound = errors.New("remote connection not found by name") + // ErrNoConfigationFile no config file found + ErrNoConfigationFile = errors.New("no configuration file found") +) diff --git a/cmd/podman/restore.go b/cmd/podman/restore.go index 8cfd5ca0d..9c77d4a5e 100644 --- a/cmd/podman/restore.go +++ b/cmd/podman/restore.go @@ -24,10 +24,10 @@ var ( restoreCommand.InputArgs = args restoreCommand.GlobalFlags = MainGlobalOpts restoreCommand.Remote = remoteclient - return restoreCmd(&restoreCommand) + return restoreCmd(&restoreCommand, cmd) }, Args: func(cmd *cobra.Command, args []string) error { - return checkAllAndLatest(cmd, args, false) + return checkAllAndLatest(cmd, args, true) }, Example: `podman container restore ctrID podman container restore --latest @@ -43,13 +43,14 @@ func init() { flags.BoolVarP(&restoreCommand.All, "all", "a", false, "Restore all checkpointed containers") flags.BoolVarP(&restoreCommand.Keep, "keep", "k", false, "Keep all temporary checkpoint files") flags.BoolVarP(&restoreCommand.Latest, "latest", "l", false, "Act on the latest container podman is aware of") - // TODO: add ContainerStateCheckpointed - flags.BoolVar(&restoreCommand.TcpEstablished, "tcp-established", false, "Checkpoint a container with established TCP connections") + flags.BoolVar(&restoreCommand.TcpEstablished, "tcp-established", false, "Restore a container with established TCP connections") + flags.StringVarP(&restoreCommand.Import, "import", "i", "", "Restore from exported checkpoint archive (tar.gz)") + flags.StringVarP(&restoreCommand.Name, "name", "n", "", "Specify new name for container restored from exported checkpoint (only works with --import)") markFlagHiddenForRemoteClient("latest", flags) } -func restoreCmd(c *cliconfig.RestoreValues) error { +func restoreCmd(c *cliconfig.RestoreValues, cmd *cobra.Command) error { if rootless.IsRootless() { return errors.New("restoring a container requires root") } @@ -63,6 +64,20 @@ func restoreCmd(c *cliconfig.RestoreValues) error { options := libpod.ContainerCheckpointOptions{ Keep: c.Keep, TCPEstablished: c.TcpEstablished, + TargetFile: c.Import, + Name: c.Name, } - return runtime.Restore(c, options) + + if c.Import == "" && c.Name != "" { + return errors.Errorf("--name can only used with --import") + } + + if c.Name != "" && c.TcpEstablished { + return errors.Errorf("--tcp-established cannot be used with --name") + } + + if (c.Import != "") && (c.All || c.Latest) { + return errors.Errorf("Cannot use --import and --all or --latest at the same time") + } + return runtime.Restore(getContext(), c, options) } diff --git a/cmd/podman/varlink/io.podman.varlink b/cmd/podman/varlink/io.podman.varlink index ed7b49c68..5b3d5ae4c 100644 --- a/cmd/podman/varlink/io.podman.varlink +++ b/cmd/podman/varlink/io.podman.varlink @@ -802,8 +802,8 @@ method DeleteUnusedImages() -> (images: []string) # attributes: _CMD, ENTRYPOINT, ENV, EXPOSE, LABEL, ONBUILD, STOPSIGNAL, USER, VOLUME, and WORKDIR_. To pause the # container while it is being committed, pass a _true_ bool for the pause argument. If the container cannot # be found by the ID or name provided, a (ContainerNotFound)[#ContainerNotFound] error will be returned; otherwise, -# the resulting image's ID will be returned as a string. -method Commit(name: string, image_name: string, changes: []string, author: string, message: string, pause: bool, manifestType: string) -> (image: string) +# the resulting image's ID will be returned as a string inside a MoreResponse. +method Commit(name: string, image_name: string, changes: []string, author: string, message: string, pause: bool, manifestType: string) -> (reply: MoreResponse) # ImportImage imports an image from a source (like tarball) into local storage. The image can have additional # descriptions added to it using the message and changes options. See also [ExportImage](ExportImage). diff --git a/completions/bash/podman b/completions/bash/podman index 49c8c0e52..efb8a6a9b 100644 --- a/completions/bash/podman +++ b/completions/bash/podman @@ -742,6 +742,10 @@ _podman_container_attach() { } _podman_container_checkpoint() { + local options_with_args=" + -e + --export + " local boolean_options=" -a --all @@ -755,9 +759,15 @@ _podman_container_checkpoint() { --leave-running --tcp-established " + case "$prev" in + -e|--export) + _filedir + return + ;; + esac case "$cur" in -*) - COMPREPLY=($(compgen -W "$boolean_options" -- "$cur")) + COMPREPLY=($(compgen -W "$boolean_options $options_with_args" -- "$cur")) ;; *) __podman_complete_containers_running @@ -844,6 +854,12 @@ _podman_container_restart() { } _podman_container_restore() { + local options_with_args=" + -i + --import + -n + --name + " local boolean_options=" -a --all @@ -855,9 +871,15 @@ _podman_container_restore() { --latest --tcp-established " + case "$prev" in + -i|--import) + _filedir + return + ;; + esac case "$cur" in -*) - COMPREPLY=($(compgen -W "$boolean_options" -- "$cur")) + COMPREPLY=($(compgen -W "$boolean_options $options_with_args" -- "$cur")) ;; *) __podman_complete_containers_created diff --git a/contrib/cirrus/README.md b/contrib/cirrus/README.md index 69d8653fe..94494a558 100644 --- a/contrib/cirrus/README.md +++ b/contrib/cirrus/README.md @@ -63,41 +63,6 @@ task (pass or fail) is set based on the exit status of the last script to execut Total execution time is capped at 2-hours (includes all the above) but this script normally completes in less than an hour. -### ``special_testing`` Task - -This task exercises podman under specialized environments or conditions. -The specific differences from the ``testing`` task depend upon the -contents of the ``$SPECIALMODE`` environment variable. - -| Value | Meaning | -| rootless | Setup a regular user to build/run integration tests. | -| in_podman | Setup a container image, build/run integration tests inside container | - -***N/B: Steps below are performed by automation*** - -1. After `gating` passes, spin up one VM per - `matrix: image_name` item. - -2. ``setup_environment.sh``: Mostly the same as - in ``testing`` task, then specialized depending on ``$SPECIALMODE``. - -3. Which tests and how they execute depends on ``$SPECIALMODE``. - - -### ``optional_testing`` Task - -***N/B: Steps below are performed by automation*** - -1. Optionally executes in parallel with ``testing``. Requires - **prior** to job-start, the magic string ``***CIRRUS: SYSTEM TEST***`` - is found in the pull-request *description*. The *description* is the first - text-box under the main *summary* line in the github WebUI. - -2. ``setup_environment.sh``: Same as for other tasks. - -3. ``system_test.sh``: Build both dependencies and libpod, install them, - then execute `make localsystem` from the repository root. - ### ``test_build_cache_images_task`` Task diff --git a/contrib/cirrus/container_test.sh b/contrib/cirrus/container_test.sh index a0542f4a0..27baf0ad7 100644 --- a/contrib/cirrus/container_test.sh +++ b/contrib/cirrus/container_test.sh @@ -133,8 +133,8 @@ fi if [ $integrationtest -eq 1 ]; then make TAGS="${TAGS}" test-binaries make varlink_generate - make ginkgo $INTEGRATION_TEST_ENVS + make localintegration $INTEGRATION_TEST_ENVS if [ $remote -eq 1 ]; then - make ginkgo-remote $INTEGRATION_TEST_ENVS + make remoteintegration $INTEGRATION_TEST_ENVS fi fi diff --git a/contrib/cirrus/integration_test.sh b/contrib/cirrus/integration_test.sh index 88e57fa53..f9ba010cd 100755 --- a/contrib/cirrus/integration_test.sh +++ b/contrib/cirrus/integration_test.sh @@ -5,6 +5,14 @@ source $(dirname $0)/lib.sh req_env_var GOSRC SCRIPT_BASE OS_RELEASE_ID OS_RELEASE_VER CONTAINER_RUNTIME +# Our name must be of the form xxxx_test or xxxx_test.sh, where xxxx is +# the test suite to run; currently (2019-05) the only option is 'integration' +# but pr2947 intends to add 'system'. +TESTSUITE=$(expr $(basename $0) : '\(.*\)_test') +if [[ -z $TESTSUITE ]]; then + die 1 "Script name is not of the form xxxx_test.sh" +fi + cd "$GOSRC" if [[ "$SPECIALMODE" == "in_podman" ]] @@ -28,11 +36,11 @@ then if [[ "$USER" == "$ROOTLESS_USER" ]] then - $GOSRC/$SCRIPT_BASE/rootless_test.sh + $GOSRC/$SCRIPT_BASE/rootless_test.sh ${TESTSUITE} else ssh $ROOTLESS_USER@localhost \ -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o CheckHostIP=no \ - $GOSRC/$SCRIPT_BASE/rootless_test.sh + $GOSRC/$SCRIPT_BASE/rootless_test.sh ${TESTSUITE} fi else make @@ -40,9 +48,9 @@ else make test-binaries if [[ "$TEST_REMOTE_CLIENT" == "true" ]] then - make remoteintegration + make remote${TESTSUITE} else - make localintegration + make local${TESTSUITE} fi exit $? fi diff --git a/contrib/cirrus/rootless_test.sh b/contrib/cirrus/rootless_test.sh index ef0d05cfd..b5744671b 100755 --- a/contrib/cirrus/rootless_test.sh +++ b/contrib/cirrus/rootless_test.sh @@ -18,6 +18,12 @@ then exit 1 fi +# Which set of tests to run; possible alternative is "system" +TESTSUITE=integration +if [[ -n "$*" ]]; then + TESTSUITE="$1" +fi + # Ensure environment setup correctly req_env_var GOSRC ROOTLESS_USER @@ -34,7 +40,7 @@ make make varlink_generate make test-binaries if [ $remote -eq 0 ]; then - make ginkgo + make local${TESTSUITE} else - make ginkgo-remote + make remote${TESTSUITE} fi diff --git a/contrib/imgts/entrypoint.sh b/contrib/imgts/entrypoint.sh index 65a76d8e4..610e1f3b6 100755 --- a/contrib/imgts/entrypoint.sh +++ b/contrib/imgts/entrypoint.sh @@ -32,6 +32,7 @@ ARGS="--update-labels=last-used=$(date +%s)" # optional [[ -z "$BUILDID" ]] || ARGS="$ARGS --update-labels=build-id=$BUILDID" [[ -z "$REPOREF" ]] || ARGS="$ARGS --update-labels=repo-ref=$REPOREF" +[[ -z "$GCPPROJECT" ]] || ARGS="$ARGS --update-labels=project=$GCPPROJECT" gcloud config set account "$GCPNAME" gcloud config set project "$GCPPROJECT" diff --git a/contrib/podmanimage/README.md b/contrib/podmanimage/README.md new file mode 100644 index 000000000..79484d4a3 --- /dev/null +++ b/contrib/podmanimage/README.md @@ -0,0 +1,44 @@ + + +# podmanimage + +## Overview + +This directory contains the Dockerfiles necessary to create the three podmanimage container +images that are housed on quay.io under the podman account. All three repositories where +the images live are public and can be pulled without credentials. These container images are secured and the +resulting containers can run safely with privileges within the container. The container images are built +using the latest Fedora and then Podman is installed into them: + + * quay.io/podman/stable - This image is built using the latest stable version of Podman in a Fedora based container. Built with podman/stable/Dockerfile. + * quay.io/podman/upstream - This image is built using the latest code found in this GitHub repository. When someone creates a commit and pushes it, the image is created. Due to that the image changes frequently and is not guaranteed to be stable. Built with podmanimage/upstream/Dockerfile. + * quay.io/podman/testing - This image is built using the latest version of Podman that is or was in updates testing for Fedora. At times this may be the same as the stable image. This container image will primarily be used by the development teams for verification testing when a new package is created. Built with podmanimage/testing/Dockerfile. + +## Sample Usage + + +``` +podman pull docker://quay.io/podman/stable:latest + +podman run --privileged stable podman version + +# Create a directory on the host to mount the container's +# /var/lib/container directory to so containers can be +# run within the container. +mkdir /var/lib/mycontainer + +# Run the image detached using the host's network in a container name +# podmanctr, turn off label and seccomp confinement in the container +# and then do a little shell hackery to keep the container up and running. +podman run --detach --name=podmanctr --net=host --security-opt label=disable --security-opt seccomp=unconfined --device /dev/fuse:rw -v /var/lib/mycontainer:/var/lib/containers:Z --privileged stable sh -c 'while true ;do wait; done' + +podman exec -it podmanctr /bin/sh + +# Now inside of the container + +podman pull alpine + +podman images + +exit +``` diff --git a/contrib/podmanimage/stable/Dockerfile b/contrib/podmanimage/stable/Dockerfile new file mode 100644 index 000000000..056f62624 --- /dev/null +++ b/contrib/podmanimage/stable/Dockerfile @@ -0,0 +1,26 @@ +# stable/Dockerfile +# +# Build a Podman container image from the latest +# stable version of Podman on the Fedoras Updates System. +# https://bodhi.fedoraproject.org/updates/?search=podman +# This image can be used to create a secured container +# that runs safely with privileges within the container. +# +FROM fedora:latest + +# Don't include container-selinux and remove +# directories used by dnf that are just taking +# up space. +RUN yum -y install podman fuse-overlayfs --exclude container-selinux; rm -rf /var/cache /var/log/dnf* /var/log/yum.* + +# Adjust storage.conf to enable Fuse storage. +RUN sed -i -e 's|^#mount_program|mount_program|g' -e '/additionalimage.*/a "/var/lib/shared",' /etc/containers/storage.conf +RUN mkdir -p /var/lib/shared/overlay-images /var/lib/shared/overlay-layers; touch /var/lib/shared/overlay-images/images.lock; touch /var/lib/shared/overlay-layers/layers.lock + +# Adjust libpod.conf to write logging to a file +RUN sed -i 's/events_logger = "journald"/events_logger = "file"/g' /usr/share/containers/libpod.conf + +# Set up environment variables to note that this is +# not starting with usernamespace and default to +# isolate the filesystem with chroot. +ENV _BUILDAH_STARTED_IN_USERNS="" BUILDAH_ISOLATION=chroot diff --git a/contrib/podmanimage/testing/Dockerfile b/contrib/podmanimage/testing/Dockerfile new file mode 100644 index 000000000..50d8ed7f2 --- /dev/null +++ b/contrib/podmanimage/testing/Dockerfile @@ -0,0 +1,28 @@ +# testing/Dockerfile +# +# Build a Podman image using the latest +# version of Podman that is in updates-testing +# on the Fedoras Updates System. At times this +# may be the same the latest stable version. +# https://bodhi.fedoraproject.org/updates/?search=podman +# This image can be used to create a secured container +# that runs safely with privileges within the container. +# +FROM fedora:latest + +# Don't include container-selinux and remove +# directories used by dnf that are just taking +# up space. +RUN yum -y install podman fuse-overlayfs --exclude container-selinux --enablerepo updates-testing; rm -rf /var/cache /var/log/dnf* /var/log/yum.* + +# Adjust storage.conf to enable Fuse storage. +RUN sed -i -e 's|^#mount_program|mount_program|g' -e '/additionalimage.*/a "/var/lib/shared",' /etc/containers/storage.conf +RUN mkdir -p /var/lib/shared/overlay-images /var/lib/shared/overlay-layers; touch /var/lib/shared/overlay-images/images.lock; touch /var/lib/shared/overlay-layers/layers.lock + +# Adjust libpod.conf to write logging to a file +RUN sed -i 's/events_logger = "journald"/events_logger = "file"/g' /usr/share/containers/libpod.conf + +# Set up environment variables to note that this is +# not starting with usernamespace and default to +# isolate the filesystem with chroot. +ENV _BUILDAH_STARTED_IN_USERNS="" BUILDAH_ISOLATION=chroot diff --git a/contrib/podmanimage/upstream/Dockerfile b/contrib/podmanimage/upstream/Dockerfile new file mode 100644 index 000000000..6ab082efc --- /dev/null +++ b/contrib/podmanimage/upstream/Dockerfile @@ -0,0 +1,78 @@ +# git/Dockerfile +# +# Build a Podman container image from the latest +# upstream version of Podman on GitHub. +# https://github.com/containers/libpod +# This image can be used to create a secured container +# that runs safely with privileges within the container. +# The containers created by this image also come with a +# Podman development environment in /root/podman. +# +FROM fedora:latest +ENV GOPATH=/root/podman + +# Install the software required to build Podman. +# Then create a directory and clone from the Podman +# GitHub repository, make and install Podman +# to the container. +# Finally remove the podman directory and a few other packages +# that are needed for building but not running Podman +RUN dnf -y install --exclude container-selinux \ + --enablerepo=updates-testing \ + atomic-registries \ + btrfs-progs-devel \ + conmon \ + containernetworking-cni \ + device-mapper-devel \ + git \ + glib2-devel \ + glibc-devel \ + glibc-static \ + go \ + golang-github-cpuguy83-go-md2man \ + gpgme-devel \ + iptables \ + libassuan-devel \ + libgpg-error-devel \ + libseccomp-devel \ + libselinux-devel \ + make \ + ostree-devel \ + pkgconfig \ + runc \ + fuse-overlayfs \ + fuse3 \ + containers-common; \ + mkdir /root/podman; \ + git clone https://github.com/containers/libpod /root/podman/src/github.com/containers/libpod; \ + cd /root/podman/src/github.com/containers/libpod; \ + make BUILDTAGS="selinux seccomp"; \ + make install PREFIX=/usr; \ + cd /root/podman; \ + git clone https://github.com/containers/conmon; \ + cd conmon; \ + make; \ + install -D -m 755 bin/conmon /usr/libexec/podman/conmon; \ + git clone https://github.com/containernetworking/plugins.git $GOPATH/src/github.com/containernetworking/plugins; \ + cd $GOPATH/src/github.com/containernetworking/plugins; \ + ./build_linux.sh; \ + mkdir -p /usr/libexec/cni; \ + cp bin/* /usr/libexec/cni; \ + mkdir -p /etc/cni/net.d; \ + curl -qsSL https://raw.githubusercontent.com/containers/libpod/master/cni/87-podman-bridge.conflist | sudo tee /etc/cni/net.d/99-loopback.conf; \ + mkdir -p /usr/share/containers; \ + cp $GOPATH/podman/src/github.com/containers/libpod/libpod.conf /usr/share/containers; \ + # Adjust libpod.conf to write logging to a file + sed -i 's/events_logger = "journald"/events_logger = "file"/g' /usr/share/containers/libpod.conf; \ + rm -rf /root/podman/*; \ + dnf -y remove bats git golang go-md2man make; \ + dnf clean all; + +# Adjust storage.conf to enable Fuse storage. +RUN sed -i -e 's|^#mount_program|mount_program|g' -e '/additionalimage.*/a "/var/lib/shared",' /etc/containers/storage.conf +RUN mkdir -p /var/lib/shared/overlay-images /var/lib/shared/overlay-layers; touch /var/lib/shared/overlay-images/images.lock; touch /var/lib/shared/overlay-layers/layers.lock + +# Set up environment variables to note that this is +# not starting with usernamespace and default to +# isolate the filesystem with chroot. +ENV _BUILDAH_STARTED_IN_USERNS="" BUILDAH_ISOLATION=chroot diff --git a/contrib/spec/podman.spec.in b/contrib/spec/podman.spec.in index a1c11a5a6..8c2ccd4b0 100644 --- a/contrib/spec/podman.spec.in +++ b/contrib/spec/podman.spec.in @@ -39,7 +39,7 @@ %global shortcommit_conmon %(c=%{commit_conmon}; echo ${c:0:7}) Name: podman -Version: 1.3.2 +Version: 1.4.1 Release: #COMMITDATE#.git%{shortcommit0}%{?dist} Summary: Manage Pods, Containers and Container Images License: ASL 2.0 diff --git a/docs/podman-build.1.md b/docs/podman-build.1.md index e2769c2a9..1b86992d9 100644 --- a/docs/podman-build.1.md +++ b/docs/podman-build.1.md @@ -529,7 +529,7 @@ Only the current container can use a private volume. `Overlay Volume Mounts` - The `:O` flag tells Buildah to mount the directory from the host as a temporary storage using the Overlay file system. The `RUN` command containers are allowed to modify contents within the mountpoint and are stored in the container storage in a separate directory. In Ovelay FS terms the source directory will be the lower, and the container storage directory will be the upper. Modifications to the mount point are destroyed when the `RUN` command finishes executing, similar to a tmpfs mount point. + The `:O` flag tells Buildah to mount the directory from the host as a temporary storage using the Overlay file system. The `RUN` command containers are allowed to modify contents within the mountpoint and are stored in the container storage in a separate directory. In Overlay FS terms the source directory will be the lower, and the container storage directory will be the upper. Modifications to the mount point are destroyed when the `RUN` command finishes executing, similar to a tmpfs mount point. Any subsequent execution of `RUN` commands sees the original source directory content, any changes from previous RUN commands no longer exists. diff --git a/docs/podman-container-checkpoint.1.md b/docs/podman-container-checkpoint.1.md index 79dc12261..afccdf59a 100644 --- a/docs/podman-container-checkpoint.1.md +++ b/docs/podman-container-checkpoint.1.md @@ -38,6 +38,12 @@ image contains established TCP connections, this options is required during restore. Defaults to not checkpointing containers with established TCP connections. +**--export, -e** + +Export the checkpoint to a tar.gz file. The exported checkpoint can be used +to import the container on another system and thus enabling container live +migration. + ## EXAMPLE podman container checkpoint mywebserver diff --git a/docs/podman-container-restore.1.md b/docs/podman-container-restore.1.md index e41f7c1d8..5efc280fe 100644 --- a/docs/podman-container-restore.1.md +++ b/docs/podman-container-restore.1.md @@ -42,6 +42,24 @@ If the checkpoint image does not contain established TCP connections this option is ignored. Defaults to not restoring containers with established TCP connections. +**--import, -i** + +Import a checkpoint tar.gz file, which was exported by Podman. This can be used +to import a checkpointed container from another host. It is not necessary to specify +a container when restoring from an exported checkpoint. + +**--name, -n** + +This is only available in combination with **--import, -i**. If a container is restored +from a checkpoint tar.gz file it is possible to rename it with **--name, -n**. This +way it is possible to restore a container from a checkpoint multiple times with different +names. + +If the **--name, -n** option is used, Podman will not attempt to assign the same IP +address to the container it was using before checkpointing as each IP address can only +be used once and the restored container will have another IP address. This also means +that **--name, -n** cannot be used in combination with **--tcp-established**. + ## EXAMPLE podman container restore mywebserver diff --git a/docs/podman-cp.1.md b/docs/podman-cp.1.md index 76fe57a9e..ee218d66a 100644 --- a/docs/podman-cp.1.md +++ b/docs/podman-cp.1.md @@ -63,7 +63,7 @@ Extract the tar file into the destination directory. If the destination director **--pause** -Pause the container while copying into it to avoid potential security issues around symlinks. Defaults to *true*. +Pause the container while copying into it to avoid potential security issues around symlinks. Defaults to *false*. ## ALTERNATIVES diff --git a/docs/podman-remote.conf.5.md b/docs/podman-remote.conf.5.md new file mode 100644 index 000000000..3e1cffb02 --- /dev/null +++ b/docs/podman-remote.conf.5.md @@ -0,0 +1,47 @@ +% podman-remote.conf(5) + +## NAME +podman-remote.conf - configuration file for the podman remote client + +## DESCRIPTION +The libpod.conf file is the default configuration file for all tools using +libpod to manage containers. + +The podman-remote.conf file is the default configuration file for the podman +remote client. It is in the TOML format. It is primarily used to keep track +of the user's remote connections. + +## CONNECTION OPTIONS +**destination** = "" + The hostname or IP address of the remote system + +**username** = "" + The username to use when connecting to the remote system + +**default** = bool + Denotes whether the connection is the default connection for the user. The default connection + is used when the user does not specify a destination or connection name to `podman`. + + +## EXAMPLE + +The following example depicts a configuration file with two connections. One of the connections +is designated as the default connection. +``` +[connections] + [connections.host1] + destination = "host1" + username = "homer" + default = true + + [connections.host2] + destination = "192.168.122.133" + username = "fedora" +``` + +## FILES + `/$HOME/.config/containers/podman-remote.conf`, default location for the podman remote +configuration file + +## HISTORY +May 2019, Originally compiled by Brent Baude<bbaude@redhat.com> diff --git a/docs/tutorials/podman_tutorial.md b/docs/tutorials/podman_tutorial.md index 032b7c851..d2f8e08fa 100644 --- a/docs/tutorials/podman_tutorial.md +++ b/docs/tutorials/podman_tutorial.md @@ -10,7 +10,7 @@ root escalation is required. ## Installing Podman -For installing or building Podman, please see the [installation instructions](install.md). +For installing or building Podman, please see the [installation instructions](https://github.com/containers/libpod/blob/master/install.md). ## Familiarizing yourself with Podman @@ -96,6 +96,28 @@ After being restored, the container will answer requests again as it did before curl http://<IP_address>:8080 ``` +### Migrate the container +To live migrate a container from one host to another the container is checkpointed on the source +system of the migration, transferred to the destination system and then restored on the destination +system. When transferring the checkpoint, it is possible to specify an output-file. + +On the source system: +```console +sudo podman container checkpoint <container_id> -e /tmp/checkpoint.tar.gz +scp /tmp/checkpoint.tar.gz <destination_system>:/tmp +``` + +On the destination system: +```console +sudo podman container restore -i /tmp/checkpoint.tar.gz +``` + +After being restored, the container will answer requests again as it did before checkpointing. This +time the container will continue to run on the destination system. +```console +curl http://<IP_address>:8080 +``` + ### Stopping the container To stop the httpd container: ```console diff --git a/install.md b/install.md index 94de2ea74..4b2c5a119 100644 --- a/install.md +++ b/install.md @@ -273,7 +273,7 @@ First, ensure that the go version that is found first on the $PATH (in case you git clone https://github.com/containers/libpod/ $GOPATH/src/github.com/containers/libpod cd $GOPATH/src/github.com/containers/libpod make BUILDTAGS="selinux seccomp" -sudo make install PREFIX= +sudo make install PREFIX=/usr ``` #### Build Tags diff --git a/libpod/container_api.go b/libpod/container_api.go index 60333104e..1894780de 100644 --- a/libpod/container_api.go +++ b/libpod/container_api.go @@ -787,11 +787,27 @@ type ContainerCheckpointOptions struct { // TCPEstablished tells the API to checkpoint a container // even if it contains established TCP connections TCPEstablished bool + // Export tells the API to write the checkpoint image to + // the filename set in TargetFile + // Import tells the API to read the checkpoint image from + // the filename set in TargetFile + TargetFile string + // Name tells the API that during restore from an exported + // checkpoint archive a new name should be used for the + // restored container + Name string } // Checkpoint checkpoints a container func (c *Container) Checkpoint(ctx context.Context, options ContainerCheckpointOptions) error { logrus.Debugf("Trying to checkpoint container %s", c.ID()) + + if options.TargetFile != "" { + if err := c.prepareCheckpointExport(); err != nil { + return err + } + } + if !c.batched { c.lock.Lock() defer c.lock.Unlock() diff --git a/libpod/container_internal.go b/libpod/container_internal.go index 5f8dd1c72..c0b5e4302 100644 --- a/libpod/container_internal.go +++ b/libpod/container_internal.go @@ -21,6 +21,7 @@ import ( "github.com/containers/storage/pkg/archive" "github.com/containers/storage/pkg/mount" spec "github.com/opencontainers/runtime-spec/specs-go" + "github.com/opencontainers/runtime-tools/generate" "github.com/opencontainers/selinux/go-selinux/label" opentracing "github.com/opentracing/opentracing-go" "github.com/pkg/errors" @@ -1345,7 +1346,7 @@ func (c *Container) appendStringToRundir(destFile, output string) (string, error return filepath.Join(c.state.RunDir, destFile), nil } -// Save OCI spec to disk, replacing any existing specs for the container +// saveSpec saves the OCI spec to disk, replacing any existing specs for the container func (c *Container) saveSpec(spec *spec.Spec) error { // If the OCI spec already exists, we need to replace it // Cannot guarantee some things, e.g. network namespaces, have the same @@ -1501,3 +1502,40 @@ func (c *Container) checkReadyForRemoval() error { return nil } + +// writeJSONFile marshalls and writes the given data to a JSON file +// in the bundle path +func (c *Container) writeJSONFile(v interface{}, file string) (err error) { + fileJSON, err := json.MarshalIndent(v, "", " ") + if err != nil { + return errors.Wrapf(err, "error writing JSON to %s for container %s", file, c.ID()) + } + file = filepath.Join(c.bundlePath(), file) + if err := ioutil.WriteFile(file, fileJSON, 0644); err != nil { + return err + } + + return nil +} + +// prepareCheckpointExport writes the config and spec to +// JSON files for later export +func (c *Container) prepareCheckpointExport() (err error) { + // save live config + if err := c.writeJSONFile(c.Config(), "config.dump"); err != nil { + return err + } + + // save spec + jsonPath := filepath.Join(c.bundlePath(), "config.json") + g, err := generate.NewFromFile(jsonPath) + if err != nil { + logrus.Debugf("generating spec for container %q failed with %v", c.ID(), err) + return err + } + if err := c.writeJSONFile(g.Spec(), "spec.dump"); err != nil { + return err + } + + return nil +} diff --git a/libpod/container_internal_linux.go b/libpod/container_internal_linux.go index f25f76092..4acc77afa 100644 --- a/libpod/container_internal_linux.go +++ b/libpod/container_internal_linux.go @@ -5,6 +5,7 @@ package libpod import ( "context" "fmt" + "io" "io/ioutil" "net" "os" @@ -25,6 +26,7 @@ import ( "github.com/containers/libpod/pkg/lookup" "github.com/containers/libpod/pkg/resolvconf" "github.com/containers/libpod/pkg/rootless" + "github.com/containers/storage/pkg/archive" securejoin "github.com/cyphar/filepath-securejoin" "github.com/opencontainers/runc/libcontainer/user" spec "github.com/opencontainers/runtime-spec/specs-go" @@ -496,6 +498,45 @@ func (c *Container) addNamespaceContainer(g *generate.Generator, ns LinuxNS, ctr return nil } +func (c *Container) exportCheckpoint(dest string) (err error) { + if (len(c.config.NamedVolumes) > 0) || (len(c.Dependencies()) > 0) { + return errors.Errorf("Cannot export checkpoints of containers with named volumes or dependencies") + } + logrus.Debugf("Exporting checkpoint image of container %q to %q", c.ID(), dest) + input, err := archive.TarWithOptions(c.bundlePath(), &archive.TarOptions{ + Compression: archive.Gzip, + IncludeSourceDir: true, + IncludeFiles: []string{ + "checkpoint", + "artifacts", + "ctr.log", + "config.dump", + "spec.dump", + "network.status"}, + }) + + if err != nil { + return errors.Wrapf(err, "error reading checkpoint directory %q", c.ID()) + } + + outFile, err := os.Create(dest) + if err != nil { + return errors.Wrapf(err, "error creating checkpoint export file %q", dest) + } + defer outFile.Close() + + if err := os.Chmod(dest, 0600); err != nil { + return errors.Wrapf(err, "cannot chmod %q", dest) + } + + _, err = io.Copy(outFile, input) + if err != nil { + return err + } + + return nil +} + func (c *Container) checkpointRestoreSupported() (err error) { if !criu.CheckForCriu() { return errors.Errorf("Checkpoint/Restore requires at least CRIU %d", criu.MinCriuVersion) @@ -549,6 +590,12 @@ func (c *Container) checkpoint(ctx context.Context, options ContainerCheckpointO return err } + if options.TargetFile != "" { + if err = c.exportCheckpoint(options.TargetFile); err != nil { + return err + } + } + logrus.Debugf("Checkpointed container %s", c.ID()) if !options.KeepRunning { @@ -561,15 +608,50 @@ func (c *Container) checkpoint(ctx context.Context, options ContainerCheckpointO } if !options.Keep { - // Remove log file - os.Remove(filepath.Join(c.bundlePath(), "dump.log")) - // Remove statistic file - os.Remove(filepath.Join(c.bundlePath(), "stats-dump")) + cleanup := []string{ + "dump.log", + "stats-dump", + "config.dump", + "spec.dump", + } + for _, delete := range cleanup { + file := filepath.Join(c.bundlePath(), delete) + os.Remove(file) + } } return c.save() } +func (c *Container) importCheckpoint(input string) (err error) { + archiveFile, err := os.Open(input) + if err != nil { + return errors.Wrapf(err, "Failed to open checkpoint archive %s for import", input) + } + + defer archiveFile.Close() + options := &archive.TarOptions{ + ExcludePatterns: []string{ + // config.dump and spec.dump are only required + // container creation + "config.dump", + "spec.dump", + }, + } + err = archive.Untar(archiveFile, c.bundlePath(), options) + if err != nil { + return errors.Wrapf(err, "Unpacking of checkpoint archive %s failed", input) + } + + // Make sure the newly created config.json exists on disk + g := generate.NewFromSpec(c.config.Spec) + if err = c.saveSpec(g.Spec()); err != nil { + return errors.Wrap(err, "Saving imported container specification for restore failed") + } + + return nil +} + func (c *Container) restore(ctx context.Context, options ContainerCheckpointOptions) (err error) { if err := c.checkpointRestoreSupported(); err != nil { @@ -580,6 +662,12 @@ func (c *Container) restore(ctx context.Context, options ContainerCheckpointOpti return errors.Wrapf(ErrCtrStateInvalid, "container %s is running or paused, cannot restore", c.ID()) } + if options.TargetFile != "" { + if err = c.importCheckpoint(options.TargetFile); err != nil { + return err + } + } + // Let's try to stat() CRIU's inventory file. If it does not exist, it makes // no sense to try a restore. This is a minimal check if a checkpoint exist. if _, err := os.Stat(filepath.Join(c.CheckpointPath(), "inventory.img")); os.IsNotExist(err) { @@ -593,7 +681,13 @@ func (c *Container) restore(ctx context.Context, options ContainerCheckpointOpti // Read network configuration from checkpoint // Currently only one interface with one IP is supported. networkStatusFile, err := os.Open(filepath.Join(c.bundlePath(), "network.status")) - if err == nil { + // If the restored container should get a new name, the IP address of + // the container will not be restored. This assumes that if a new name is + // specified, the container is restored multiple times. + // TODO: This implicit restoring with or without IP depending on an + // unrelated restore parameter (--name) does not seem like the + // best solution. + if err == nil && options.Name == "" { // The file with the network.status does exist. Let's restore the // container with the same IP address as during checkpointing. defer networkStatusFile.Close() @@ -637,23 +731,44 @@ func (c *Container) restore(ctx context.Context, options ContainerCheckpointOpti return err } + // Restoring from an import means that we are doing migration + if options.TargetFile != "" { + g.SetRootPath(c.state.Mountpoint) + } + // We want to have the same network namespace as before. if c.config.CreateNetNS { g.AddOrReplaceLinuxNamespace(spec.NetworkNamespace, c.state.NetNS.Path()) } - // Save the OCI spec to disk - if err := c.saveSpec(g.Spec()); err != nil { + if err := c.makeBindMounts(); err != nil { return err } - if err := c.makeBindMounts(); err != nil { - return err + if options.TargetFile != "" { + for dstPath, srcPath := range c.state.BindMounts { + newMount := spec.Mount{ + Type: "bind", + Source: srcPath, + Destination: dstPath, + Options: []string{"bind", "private"}, + } + if c.IsReadOnly() && dstPath != "/dev/shm" { + newMount.Options = append(newMount.Options, "ro", "nosuid", "noexec", "nodev") + } + if !MountExists(g.Mounts(), dstPath) { + g.AddMount(newMount) + } + } } // Cleanup for a working restore. c.removeConmonFiles() + // Save the OCI spec to disk + if err := c.saveSpec(g.Spec()); err != nil { + return err + } if err := c.runtime.ociRuntime.createContainer(c, c.config.CgroupParent, &options); err != nil { return err } diff --git a/libpod/runtime.go b/libpod/runtime.go index 1f8dd98b4..098607b63 100644 --- a/libpod/runtime.go +++ b/libpod/runtime.go @@ -250,6 +250,7 @@ type runtimeConfiguredFrom struct { volPathSet bool conmonPath bool conmonEnvVars bool + initPath bool ociRuntimes bool runtimePath bool cniPluginDir bool @@ -475,6 +476,9 @@ func newRuntimeFromConfig(ctx context.Context, userConfigPath string, options .. if tmpConfig.ConmonEnvVars != nil { runtime.configuredFrom.conmonEnvVars = true } + if tmpConfig.InitPath != "" { + runtime.configuredFrom.initPath = true + } if tmpConfig.OCIRuntimes != nil { runtime.configuredFrom.ociRuntimes = true } @@ -512,6 +516,9 @@ func newRuntimeFromConfig(ctx context.Context, userConfigPath string, options .. if !runtime.configuredFrom.conmonEnvVars { runtime.config.ConmonEnvVars = tmpConfig.ConmonEnvVars } + if !runtime.configuredFrom.initPath { + runtime.config.InitPath = tmpConfig.InitPath + } if !runtime.configuredFrom.ociRuntimes { runtime.config.OCIRuntimes = tmpConfig.OCIRuntimes } diff --git a/libpod/runtime_ctr.go b/libpod/runtime_ctr.go index 0c8d3edab..cf1f5701d 100644 --- a/libpod/runtime_ctr.go +++ b/libpod/runtime_ctr.go @@ -14,6 +14,7 @@ import ( "github.com/containers/storage" "github.com/containers/storage/pkg/stringid" spec "github.com/opencontainers/runtime-spec/specs-go" + "github.com/opencontainers/runtime-tools/generate" opentracing "github.com/opentracing/opentracing-go" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -34,7 +35,7 @@ type CtrCreateOption func(*Container) error // A true return will include the container, a false return will exclude it. type ContainerFilter func(*Container) bool -// NewContainer creates a new container from a given OCI config +// NewContainer creates a new container from a given OCI config. func (r *Runtime) NewContainer(ctx context.Context, rSpec *spec.Spec, options ...CtrCreateOption) (c *Container, err error) { r.lock.Lock() defer r.lock.Unlock() @@ -44,20 +45,46 @@ func (r *Runtime) NewContainer(ctx context.Context, rSpec *spec.Spec, options .. return r.newContainer(ctx, rSpec, options...) } -func (r *Runtime) newContainer(ctx context.Context, rSpec *spec.Spec, options ...CtrCreateOption) (c *Container, err error) { - span, _ := opentracing.StartSpanFromContext(ctx, "newContainer") - span.SetTag("type", "runtime") - defer span.Finish() +// RestoreContainer re-creates a container from an imported checkpoint +func (r *Runtime) RestoreContainer(ctx context.Context, rSpec *spec.Spec, config *ContainerConfig) (c *Container, err error) { + r.lock.Lock() + defer r.lock.Unlock() + if !r.valid { + return nil, ErrRuntimeStopped + } + ctr, err := r.initContainerVariables(rSpec, config) + if err != nil { + return nil, errors.Wrapf(err, "error initializing container variables") + } + return r.setupContainer(ctx, ctr, true) +} + +func (r *Runtime) initContainerVariables(rSpec *spec.Spec, config *ContainerConfig) (c *Container, err error) { if rSpec == nil { return nil, errors.Wrapf(ErrInvalidArg, "must provide a valid runtime spec to create container") } - ctr := new(Container) ctr.config = new(ContainerConfig) ctr.state = new(ContainerState) - ctr.config.ID = stringid.GenerateNonCryptoID() + if config == nil { + ctr.config.ID = stringid.GenerateNonCryptoID() + ctr.config.ShmSize = DefaultShmSize + } else { + // This is a restore from an imported checkpoint + if err := JSONDeepCopy(config, ctr.config); err != nil { + return nil, errors.Wrapf(err, "error copying container config for restore") + } + // If the ID is empty a new name for the restored container was requested + if ctr.config.ID == "" { + ctr.config.ID = stringid.GenerateNonCryptoID() + // Fixup ExitCommand with new ID + ctr.config.ExitCommand[len(ctr.config.ExitCommand)-1] = ctr.config.ID + } + // Reset the log path to point to the default + ctr.config.LogPath = "" + } ctr.config.Spec = new(spec.Spec) if err := JSONDeepCopy(rSpec, ctr.config.Spec); err != nil { @@ -65,8 +92,6 @@ func (r *Runtime) newContainer(ctx context.Context, rSpec *spec.Spec, options .. } ctr.config.CreatedTime = time.Now() - ctr.config.ShmSize = DefaultShmSize - ctr.state.BindMounts = make(map[string]string) ctr.config.StopTimeout = CtrRemoveTimeout @@ -80,12 +105,29 @@ func (r *Runtime) newContainer(ctx context.Context, rSpec *spec.Spec, options .. } ctr.runtime = r + + return ctr, nil +} + +func (r *Runtime) newContainer(ctx context.Context, rSpec *spec.Spec, options ...CtrCreateOption) (c *Container, err error) { + span, _ := opentracing.StartSpanFromContext(ctx, "newContainer") + span.SetTag("type", "runtime") + defer span.Finish() + + ctr, err := r.initContainerVariables(rSpec, nil) + if err != nil { + return nil, errors.Wrapf(err, "error initializing container variables") + } + for _, option := range options { if err := option(ctr); err != nil { return nil, errors.Wrapf(err, "error running container create option") } } + return r.setupContainer(ctx, ctr, false) +} +func (r *Runtime) setupContainer(ctx context.Context, ctr *Container, restore bool) (c *Container, err error) { // Allocate a lock for the container lock, err := r.lockManager.AllocateLock() if err != nil { @@ -154,6 +196,19 @@ func (r *Runtime) newContainer(ctx context.Context, rSpec *spec.Spec, options .. return nil, errors.Wrapf(ErrInvalidArg, "unsupported CGroup manager: %s - cannot validate cgroup parent", r.config.CgroupManager) } + if restore { + // Remove information about bind mount + // for new container from imported checkpoint + g := generate.Generator{Config: ctr.config.Spec} + g.RemoveMount("/dev/shm") + ctr.config.ShmDir = "" + g.RemoveMount("/etc/resolv.conf") + g.RemoveMount("/etc/hostname") + g.RemoveMount("/etc/hosts") + g.RemoveMount("/run/.containerenv") + g.RemoveMount("/run/secrets") + } + // Set up storage for the container if err := ctr.setupStorage(ctx); err != nil { return nil, err diff --git a/pkg/adapter/checkpoint_restore.go b/pkg/adapter/checkpoint_restore.go new file mode 100644 index 000000000..97ba5ecf7 --- /dev/null +++ b/pkg/adapter/checkpoint_restore.go @@ -0,0 +1,145 @@ +// +build !remoteclient + +package adapter + +import ( + "context" + "github.com/containers/libpod/libpod" + "github.com/containers/libpod/libpod/image" + "github.com/containers/storage/pkg/archive" + jsoniter "github.com/json-iterator/go" + spec "github.com/opencontainers/runtime-spec/specs-go" + "github.com/pkg/errors" + "io" + "io/ioutil" + "os" + "path/filepath" +) + +// Prefixing the checkpoint/restore related functions with 'cr' + +// crImportFromJSON imports the JSON files stored in the exported +// checkpoint tarball +func crImportFromJSON(filePath string, v interface{}) error { + jsonFile, err := os.Open(filePath) + if err != nil { + return errors.Wrapf(err, "Failed to open container definition %s for restore", filePath) + } + defer jsonFile.Close() + + content, err := ioutil.ReadAll(jsonFile) + if err != nil { + return errors.Wrapf(err, "Failed to read container definition %s for restore", filePath) + } + json := jsoniter.ConfigCompatibleWithStandardLibrary + if err = json.Unmarshal([]byte(content), v); err != nil { + return errors.Wrapf(err, "Failed to unmarshal container definition %s for restore", filePath) + } + + return nil +} + +// crImportCheckpoint it the function which imports the information +// from checkpoint tarball and re-creates the container from that information +func crImportCheckpoint(ctx context.Context, runtime *libpod.Runtime, input string, name string) ([]*libpod.Container, error) { + // First get the container definition from the + // tarball to a temporary directory + archiveFile, err := os.Open(input) + if err != nil { + return nil, errors.Wrapf(err, "Failed to open checkpoint archive %s for import", input) + } + defer archiveFile.Close() + options := &archive.TarOptions{ + // Here we only need the files config.dump and spec.dump + ExcludePatterns: []string{ + "checkpoint", + "artifacts", + "ctr.log", + "network.status", + }, + } + dir, err := ioutil.TempDir("", "checkpoint") + if err != nil { + return nil, err + } + defer os.RemoveAll(dir) + err = archive.Untar(archiveFile, dir, options) + if err != nil { + return nil, errors.Wrapf(err, "Unpacking of checkpoint archive %s failed", input) + } + + // Load spec.dump from temporary directory + spec := new(spec.Spec) + if err := crImportFromJSON(filepath.Join(dir, "spec.dump"), spec); err != nil { + return nil, err + } + + // Load config.dump from temporary directory + config := new(libpod.ContainerConfig) + if err = crImportFromJSON(filepath.Join(dir, "config.dump"), config); err != nil { + return nil, err + } + + // This should not happen as checkpoints with these options are not exported. + if (len(config.Dependencies) > 0) || (len(config.NamedVolumes) > 0) { + return nil, errors.Errorf("Cannot import checkpoints of containers with named volumes or dependencies") + } + + ctrID := config.ID + newName := false + + // Check if the restored container gets a new name + if name != "" { + config.ID = "" + config.Name = name + newName = true + } + + ctrName := config.Name + + // The code to load the images is copied from create.go + var writer io.Writer + // In create.go this only set if '--quiet' does not exist. + writer = os.Stderr + rtc, err := runtime.GetConfig() + if err != nil { + return nil, err + } + + _, err = runtime.ImageRuntime().New(ctx, config.RootfsImageName, rtc.SignaturePolicyPath, "", writer, nil, image.SigningOptions{}, false, nil) + if err != nil { + return nil, err + } + + // Now create a new container from the just loaded information + container, err := runtime.RestoreContainer(ctx, spec, config) + if err != nil { + return nil, err + } + + var containers []*libpod.Container + if container == nil { + return nil, nil + } + + containerConfig := container.Config() + if containerConfig.Name != ctrName { + return nil, errors.Errorf("Name of restored container (%s) does not match requested name (%s)", containerConfig.Name, ctrName) + } + + if newName == false { + // Only check ID for a restore with the same name. + // Using -n to request a new name for the restored container, will also create a new ID + if containerConfig.ID != ctrID { + return nil, errors.Errorf("ID of restored container (%s) does not match requested ID (%s)", containerConfig.ID, ctrID) + } + } + + // Check if the ExitCommand points to the correct container ID + if containerConfig.ExitCommand[len(containerConfig.ExitCommand)-1] != containerConfig.ID { + return nil, errors.Errorf("'ExitCommandID' uses ID %s instead of container ID %s", containerConfig.ExitCommand[len(containerConfig.ExitCommand)-1], containerConfig.ID) + } + + containers = append(containers, container) + return containers, nil +} diff --git a/pkg/adapter/client.go b/pkg/adapter/client.go index 01914834f..69aa3220a 100644 --- a/pkg/adapter/client.go +++ b/pkg/adapter/client.go @@ -6,42 +6,52 @@ import ( "fmt" "os" + "github.com/containers/libpod/cmd/podman/remoteclientconfig" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "github.com/varlink/go/varlink" ) var remoteEndpoint *Endpoint func (r RemoteRuntime) RemoteEndpoint() (remoteEndpoint *Endpoint, err error) { - if remoteEndpoint == nil { - remoteEndpoint = &Endpoint{Unknown, ""} - } else { - return remoteEndpoint, nil - } + remoteConfigConnections, _ := remoteclientconfig.ReadRemoteConfig(r.config) - // I'm leaving this here for now as a document of the birdge format. It can be removed later once the bridge - // function is more flushed out. - // bridge := `ssh -T root@192.168.122.1 "/usr/bin/varlink -A '/usr/bin/podman varlink \$VARLINK_ADDRESS' bridge"` - if len(r.cmd.RemoteHost) > 0 { - // The user has provided a remote host endpoint + // If the user defines an env variable for podman_varlink_bridge + // we use that as passed. + if bridge := os.Getenv("PODMAN_VARLINK_BRIDGE"); bridge != "" { + logrus.Debug("creating a varlink bridge based on env variable") + remoteEndpoint, err = newBridgeConnection(bridge, nil, r.cmd.LogLevel) + // if an environment variable for podman_varlink_address is defined, + // we used that as passed + } else if address := os.Getenv("PODMAN_VARLINK_ADDRESS"); address != "" { + logrus.Debug("creating a varlink address based on env variable: %s", address) + remoteEndpoint, err = newSocketConnection(address) + // if the user provides a remote host, we use it to configure a bridge connection + } else if len(r.cmd.RemoteHost) > 0 { + logrus.Debug("creating a varlink bridge based on user input") if len(r.cmd.RemoteUserName) < 1 { return nil, errors.New("you must provide a username when providing a remote host name") } - remoteEndpoint.Type = BridgeConnection - remoteEndpoint.Connection = fmt.Sprintf( - `ssh -T %s@%s /usr/bin/varlink -A \'/usr/bin/podman --log-level=%s varlink \\\$VARLINK_ADDRESS\' bridge`, - r.cmd.RemoteUserName, r.cmd.RemoteHost, r.cmd.LogLevel) - - } else if bridge := os.Getenv("PODMAN_VARLINK_BRIDGE"); bridge != "" { - remoteEndpoint.Type = BridgeConnection - remoteEndpoint.Connection = bridge - } else { - address := os.Getenv("PODMAN_VARLINK_ADDRESS") - if address == "" { - address = DefaultAddress + rc := remoteclientconfig.RemoteConnection{r.cmd.RemoteHost, r.cmd.RemoteUserName, false} + remoteEndpoint, err = newBridgeConnection("", &rc, r.cmd.LogLevel) + // if the user has a config file with connections in it + } else if len(remoteConfigConnections.Connections) > 0 { + logrus.Debug("creating a varlink bridge based configuration file") + var rc *remoteclientconfig.RemoteConnection + if len(r.cmd.ConnectionName) > 0 { + rc, err = remoteConfigConnections.GetRemoteConnection(r.cmd.ConnectionName) + } else { + rc, err = remoteConfigConnections.GetDefault() + } + if err != nil { + return nil, err } - remoteEndpoint.Type = DirectConnection - remoteEndpoint.Connection = address + remoteEndpoint, err = newBridgeConnection("", rc, r.cmd.LogLevel) + // last resort is to make a socket connection with the default varlink address for root user + } else { + logrus.Debug("creating a varlink address based default root address") + remoteEndpoint, err = newSocketConnection(DefaultAddress) } return } @@ -72,3 +82,12 @@ func (r RemoteRuntime) RefreshConnection() error { r.Conn = newConn return nil } + +// newSocketConnection returns an endpoint for a uds based connection +func newSocketConnection(address string) (*Endpoint, error) { + endpoint := Endpoint{ + Type: DirectConnection, + Connection: address, + } + return &endpoint, nil +} diff --git a/pkg/adapter/client_unix.go b/pkg/adapter/client_unix.go new file mode 100644 index 000000000..e0406567c --- /dev/null +++ b/pkg/adapter/client_unix.go @@ -0,0 +1,30 @@ +// +build linux darwin +// +build remoteclient + +package adapter + +import ( + "fmt" + + "github.com/containers/libpod/cmd/podman/remoteclientconfig" + "github.com/pkg/errors" +) + +// newBridgeConnection creates a bridge type endpoint with username, destination, and log-level +func newBridgeConnection(formattedBridge string, remoteConn *remoteclientconfig.RemoteConnection, logLevel string) (*Endpoint, error) { + endpoint := Endpoint{ + Type: BridgeConnection, + } + + if len(formattedBridge) < 1 && remoteConn == nil { + return nil, errors.New("bridge connections must either be created by string or remoteconnection") + } + if len(formattedBridge) > 0 { + endpoint.Connection = formattedBridge + return &endpoint, nil + } + endpoint.Connection = fmt.Sprintf( + `ssh -T %s@%s -- /usr/bin/varlink -A \'/usr/bin/podman --log-level=%s varlink \\\$VARLINK_ADDRESS\' bridge`, + remoteConn.Username, remoteConn.Destination, logLevel) + return &endpoint, nil +} diff --git a/pkg/adapter/client_windows.go b/pkg/adapter/client_windows.go new file mode 100644 index 000000000..088550667 --- /dev/null +++ b/pkg/adapter/client_windows.go @@ -0,0 +1,15 @@ +// +build remoteclient + +package adapter + +import ( + "github.com/containers/libpod/cmd/podman/remoteclientconfig" + "github.com/containers/libpod/libpod" +) + +func newBridgeConnection(formattedBridge string, remoteConn *remoteclientconfig.RemoteConnection, logLevel string) (*Endpoint, error) { + // TODO + // Unix and Windows appear to quote their ssh implementations differently therefore once we figure out what + // windows ssh is doing here, we can then get the format correct. + return nil, libpod.ErrNotImplemented +} diff --git a/pkg/adapter/containers.go b/pkg/adapter/containers.go index ff7b6377a..29297fbd5 100644 --- a/pkg/adapter/containers.go +++ b/pkg/adapter/containers.go @@ -6,6 +6,7 @@ import ( "bufio" "context" "fmt" + "io" "io/ioutil" "os" "path/filepath" @@ -15,9 +16,12 @@ import ( "syscall" "time" + "github.com/containers/buildah" + "github.com/containers/image/manifest" "github.com/containers/libpod/cmd/podman/cliconfig" "github.com/containers/libpod/cmd/podman/shared" "github.com/containers/libpod/libpod" + "github.com/containers/libpod/libpod/image" "github.com/containers/libpod/pkg/adapter/shortcuts" "github.com/containers/libpod/pkg/systemdgen" "github.com/containers/psgo" @@ -522,7 +526,7 @@ func (r *LocalRuntime) Checkpoint(c *cliconfig.CheckpointValues, options libpod. } // Restore one or more containers -func (r *LocalRuntime) Restore(c *cliconfig.RestoreValues, options libpod.ContainerCheckpointOptions) error { +func (r *LocalRuntime) Restore(ctx context.Context, c *cliconfig.RestoreValues, options libpod.ContainerCheckpointOptions) error { var ( containers []*libpod.Container err, lastError error @@ -534,7 +538,9 @@ func (r *LocalRuntime) Restore(c *cliconfig.RestoreValues, options libpod.Contai return state == libpod.ContainerStateExited }) - if c.All { + if c.Import != "" { + containers, err = crImportCheckpoint(ctx, r.Runtime, c.Import, c.Name) + } else if c.All { containers, err = r.GetContainers(filterFuncs...) } else { containers, err = shortcuts.GetContainersByContext(false, c.Latest, c.InputArgs, r.Runtime) @@ -1030,3 +1036,55 @@ func (r *LocalRuntime) GenerateSystemd(c *cliconfig.GenerateSystemdValues) (stri func (r *LocalRuntime) GetNamespaces(container shared.PsContainerOutput) *shared.Namespace { return shared.GetNamespaces(container.Pid) } + +// Commit creates a local image from a container +func (r *LocalRuntime) Commit(ctx context.Context, c *cliconfig.CommitValues, container, imageName string) (string, error) { + var ( + writer io.Writer + mimeType string + ) + switch c.Format { + case "oci": + mimeType = buildah.OCIv1ImageManifest + if c.Flag("message").Changed { + return "", errors.Errorf("messages are only compatible with the docker image format (-f docker)") + } + case "docker": + mimeType = manifest.DockerV2Schema2MediaType + default: + return "", errors.Errorf("unrecognized image format %q", c.Format) + } + if !c.Quiet { + writer = os.Stderr + } + ctr, err := r.Runtime.LookupContainer(container) + if err != nil { + return "", errors.Wrapf(err, "error looking up container %q", container) + } + + rtc, err := r.Runtime.GetConfig() + if err != nil { + return "", err + } + + sc := image.GetSystemContext(rtc.SignaturePolicyPath, "", false) + coptions := buildah.CommitOptions{ + SignaturePolicyPath: rtc.SignaturePolicyPath, + ReportWriter: writer, + SystemContext: sc, + PreferredManifestType: mimeType, + } + options := libpod.ContainerCommitOptions{ + CommitOptions: coptions, + Pause: c.Pause, + IncludeVolumes: c.IncludeVolumes, + Message: c.Message, + Changes: c.Change, + Author: c.Author, + } + newImage, err := ctr.Commit(ctx, imageName, options) + if err != nil { + return "", err + } + return newImage.ID(), nil +} diff --git a/pkg/adapter/containers_remote.go b/pkg/adapter/containers_remote.go index ce4b96370..cf0b90b3a 100644 --- a/pkg/adapter/containers_remote.go +++ b/pkg/adapter/containers_remote.go @@ -663,6 +663,10 @@ func (r *LocalRuntime) Attach(ctx context.Context, c *cliconfig.AttachValues) er // Checkpoint one or more containers func (r *LocalRuntime) Checkpoint(c *cliconfig.CheckpointValues, options libpod.ContainerCheckpointOptions) error { + if c.Export != "" { + return errors.New("the remote client does not support exporting checkpoints") + } + var lastError error ids, err := iopodman.GetContainersByContext().Call(r.Conn, c.All, c.Latest, c.InputArgs) if err != nil { @@ -698,7 +702,11 @@ func (r *LocalRuntime) Checkpoint(c *cliconfig.CheckpointValues, options libpod. } // Restore one or more containers -func (r *LocalRuntime) Restore(c *cliconfig.RestoreValues, options libpod.ContainerCheckpointOptions) error { +func (r *LocalRuntime) Restore(ctx context.Context, c *cliconfig.RestoreValues, options libpod.ContainerCheckpointOptions) error { + if c.Import != "" { + return errors.New("the remote client does not support importing checkpoints") + } + var lastError error ids, err := iopodman.GetContainersByContext().Call(r.Conn, c.All, c.Latest, c.InputArgs) if err != nil { @@ -993,3 +1001,26 @@ func (r *LocalRuntime) GetNamespaces(container shared.PsContainerOutput) *shared } return &ns } + +// Commit creates a local image from a container +func (r *LocalRuntime) Commit(ctx context.Context, c *cliconfig.CommitValues, container, imageName string) (string, error) { + var iid string + reply, err := iopodman.Commit().Send(r.Conn, varlink.More, container, imageName, c.Change, c.Author, c.Message, c.Pause, c.Format) + if err != nil { + return "", err + } + for { + responses, flags, err := reply() + if err != nil { + return "", err + } + for _, line := range responses.Logs { + fmt.Fprintln(os.Stderr, line) + } + iid = responses.Id + if flags&varlink.Continues == 0 { + break + } + } + return iid, nil +} diff --git a/pkg/adapter/runtime_remote.go b/pkg/adapter/runtime_remote.go index e0c0898bd..a1d358f68 100644 --- a/pkg/adapter/runtime_remote.go +++ b/pkg/adapter/runtime_remote.go @@ -20,6 +20,7 @@ import ( "github.com/containers/image/docker/reference" "github.com/containers/image/types" "github.com/containers/libpod/cmd/podman/cliconfig" + "github.com/containers/libpod/cmd/podman/remoteclientconfig" "github.com/containers/libpod/cmd/podman/varlink" "github.com/containers/libpod/libpod" "github.com/containers/libpod/libpod/events" @@ -40,6 +41,7 @@ type RemoteRuntime struct { Conn *varlink.Connection Remote bool cmd cliconfig.MainFlags + config io.Reader } // LocalRuntime describes a typical libpod runtime @@ -49,10 +51,35 @@ type LocalRuntime struct { // GetRuntime returns a LocalRuntime struct with the actual runtime embedded in it func GetRuntime(ctx context.Context, c *cliconfig.PodmanCommand) (*LocalRuntime, error) { + var ( + customConfig bool + err error + f *os.File + ) runtime := RemoteRuntime{ Remote: true, cmd: c.GlobalFlags, } + configPath := remoteclientconfig.GetConfigFilePath() + if len(c.GlobalFlags.RemoteConfigFilePath) > 0 { + configPath = c.GlobalFlags.RemoteConfigFilePath + customConfig = true + } + + f, err = os.Open(configPath) + if err != nil { + // If user does not explicitly provide a configuration file path and we cannot + // find a default, no error should occur. + if os.IsNotExist(err) && !customConfig { + logrus.Debugf("unable to load configuration file at %s", configPath) + runtime.config = nil + } else { + return nil, errors.Wrapf(err, "unable to load configuration file at %s", configPath) + } + } else { + // create the io reader for the remote client + runtime.config = bufio.NewReader(f) + } conn, err := runtime.Connect() if err != nil { return nil, err diff --git a/pkg/rootless/rootless_linux.c b/pkg/rootless/rootless_linux.c index a2425c83e..eb62d55e9 100644 --- a/pkg/rootless/rootless_linux.c +++ b/pkg/rootless/rootless_linux.c @@ -489,6 +489,7 @@ reexec_userns_join (int userns, int mountns, char *pause_pid_file_path) char **argv; int pid; char *cwd = getcwd (NULL, 0); + sigset_t sigset, oldsigset; if (cwd == NULL) { @@ -522,6 +523,22 @@ reexec_userns_join (int userns, int mountns, char *pause_pid_file_path) return pid; } + if (sigfillset (&sigset) < 0) + { + fprintf (stderr, "cannot fill sigset: %s\n", strerror (errno)); + _exit (EXIT_FAILURE); + } + if (sigdelset (&sigset, SIGCHLD) < 0) + { + fprintf (stderr, "cannot sigdelset(SIGCHLD): %s\n", strerror (errno)); + _exit (EXIT_FAILURE); + } + if (sigprocmask (SIG_BLOCK, &sigset, &oldsigset) < 0) + { + fprintf (stderr, "cannot block signals: %s\n", strerror (errno)); + _exit (EXIT_FAILURE); + } + setenv ("_CONTAINERS_USERNS_CONFIGURED", "init", 1); setenv ("_CONTAINERS_ROOTLESS_UID", uid, 1); setenv ("_CONTAINERS_ROOTLESS_GID", gid, 1); @@ -570,6 +587,11 @@ reexec_userns_join (int userns, int mountns, char *pause_pid_file_path) /* We ignore errors here as we didn't create the namespace anyway. */ create_pause_process (pause_pid_file_path, argv); } + if (sigprocmask (SIG_SETMASK, &oldsigset, NULL) < 0) + { + fprintf (stderr, "cannot block signals: %s\n", strerror (errno)); + _exit (EXIT_FAILURE); + } execvp (argv[0], argv); diff --git a/pkg/rootless/rootless_linux.go b/pkg/rootless/rootless_linux.go index d302b1777..0cac50fc0 100644 --- a/pkg/rootless/rootless_linux.go +++ b/pkg/rootless/rootless_linux.go @@ -169,6 +169,9 @@ func getUserNSFirstChild(fd uintptr) (*os.File, error) { for { nextFd, err := getParentUserNs(fd) if err != nil { + if err == syscall.ENOTTY { + return os.NewFile(fd, "userns child"), nil + } return nil, errors.Wrapf(err, "cannot get parent user namespace") } diff --git a/pkg/varlinkapi/images.go b/pkg/varlinkapi/images.go index fa1a0a109..1abc4f086 100644 --- a/pkg/varlinkapi/images.go +++ b/pkg/varlinkapi/images.go @@ -371,7 +371,6 @@ func (i *LibpodAPI) PushImage(call iopodman.VarlinkCall, name, tag string, compr done = true default: if !call.WantsMore() { - time.Sleep(1 * time.Second) break } br := iopodman.MoreResponse{ @@ -495,6 +494,9 @@ func (i *LibpodAPI) DeleteUnusedImages(call iopodman.VarlinkCall) error { // Commit ... func (i *LibpodAPI) Commit(call iopodman.VarlinkCall, name, imageName string, changes []string, author, message string, pause bool, manifestType string) error { + var newImage *image.Image + + output := bytes.NewBuffer([]byte{}) ctr, err := i.Runtime.LookupContainer(name) if err != nil { return call.ReplyContainerNotFound(name, err.Error()) @@ -515,7 +517,7 @@ func (i *LibpodAPI) Commit(call iopodman.VarlinkCall, name, imageName string, ch } coptions := buildah.CommitOptions{ SignaturePolicyPath: rtc.SignaturePolicyPath, - ReportWriter: nil, + ReportWriter: output, SystemContext: sc, PreferredManifestType: mimeType, } @@ -527,11 +529,61 @@ func (i *LibpodAPI) Commit(call iopodman.VarlinkCall, name, imageName string, ch Author: author, } - newImage, err := ctr.Commit(getContext(), imageName, options) - if err != nil { - return call.ReplyErrorOccurred(err.Error()) + if call.WantsMore() { + call.Continues = true } - return call.ReplyCommit(newImage.ID()) + + c := make(chan error) + + go func() { + newImage, err = ctr.Commit(getContext(), imageName, options) + if err != nil { + c <- err + } + c <- nil + close(c) + }() + + var log []string + done := false + for { + line, err := output.ReadString('\n') + if err == nil { + log = append(log, line) + continue + } else if err == io.EOF { + select { + case err := <-c: + if err != nil { + logrus.Errorf("reading of output during commit failed for %s", name) + return call.ReplyErrorOccurred(err.Error()) + } + done = true + default: + if !call.WantsMore() { + break + } + br := iopodman.MoreResponse{ + Logs: log, + } + call.ReplyCommit(br) + log = []string{} + } + } else { + return call.ReplyErrorOccurred(err.Error()) + } + if done { + break + } + } + call.Continues = false + + br := iopodman.MoreResponse{ + Logs: log, + Id: newImage.ID(), + } + + return call.ReplyCommit(br) } // ImportImage imports an image from a tarball to the image store @@ -633,7 +685,6 @@ func (i *LibpodAPI) PullImage(call iopodman.VarlinkCall, name string) error { done = true default: if !call.WantsMore() { - time.Sleep(1 * time.Second) break } br := iopodman.MoreResponse{ @@ -764,7 +815,6 @@ func (i *LibpodAPI) ImageSave(call iopodman.VarlinkCall, options iopodman.ImageS done = true default: if !call.WantsMore() { - time.Sleep(1 * time.Second) break } br := iopodman.MoreResponse{ @@ -844,7 +894,6 @@ func (i *LibpodAPI) LoadImage(call iopodman.VarlinkCall, name, inputFile string, done = true default: if !call.WantsMore() { - time.Sleep(1 * time.Second) break } br := iopodman.MoreResponse{ diff --git a/pkg/varlinkapi/virtwriter/virtwriter.go b/pkg/varlinkapi/virtwriter/virtwriter.go index 3adaf6e17..e747984c7 100644 --- a/pkg/varlinkapi/virtwriter/virtwriter.go +++ b/pkg/varlinkapi/virtwriter/virtwriter.go @@ -91,65 +91,65 @@ func (v VirtWriteCloser) Write(input []byte) (int, error) { // Reader decodes the content that comes over the wire and directs it to the proper destination. func Reader(r *bufio.Reader, output, errput *os.File, input *io.PipeWriter, resize chan remotecommand.TerminalSize) error { - var saveb []byte - var eom int + var messageSize int64 + headerBytes := make([]byte, 8) + for { - readb := make([]byte, 32*1024) - n, err := r.Read(readb) - // TODO, later may be worth checking in len of the read is 0 + n, err := io.ReadFull(r, headerBytes) if err != nil { return err } - b := append(saveb, readb[0:n]...) - // no sense in reading less than the header len - for len(b) > 7 { - eom = int(binary.BigEndian.Uint32(b[4:8])) + 8 - // The message and header are togther - if len(b) >= eom { - out := append([]byte{}, b[8:eom]...) - - switch IntToSocketDest(int(b[0])) { - case ToStdout: - n, err := output.Write(out) - if err != nil { - return err - } - if n < len(out) { - return errors.New("short write error occurred on stdout") - } - case ToStderr: - n, err := errput.Write(out) - if err != nil { - return err - } - if n < len(out) { - return errors.New("short write error occurred on stderr") - } - case ToStdin: - n, err := input.Write(out) - if err != nil { - return err - } - if n < len(out) { - return errors.New("short write error occurred on stdin") - } - case TerminalResize: - // Resize events come over in bytes, need to be reserialized - resizeEvent := remotecommand.TerminalSize{} - if err := json.Unmarshal(out, &resizeEvent); err != nil { - return err - } - resize <- resizeEvent - case Quit: - return nil + if n < 8 { + return errors.New("short read and no full header read") + } + + messageSize = int64(binary.BigEndian.Uint32(headerBytes[4:8])) + + switch IntToSocketDest(int(headerBytes[0])) { + case ToStdout: + _, err := io.CopyN(output, r, messageSize) + if err != nil { + return err + } + case ToStderr: + _, err := io.CopyN(errput, r, messageSize) + if err != nil { + return err + } + case ToStdin: + _, err := io.CopyN(input, r, messageSize) + if err != nil { + return err + } + case TerminalResize: + out := make([]byte, messageSize) + if messageSize > 0 { + _, err = io.ReadFull(r, out) + + if err != nil { + return err } - b = b[eom:] - } else { - // We do not have the header and full message, need to slurp again - saveb = b - break } + // Resize events come over in bytes, need to be reserialized + resizeEvent := remotecommand.TerminalSize{} + if err := json.Unmarshal(out, &resizeEvent); err != nil { + return err + } + resize <- resizeEvent + case Quit: + out := make([]byte, messageSize) + if messageSize > 0 { + _, err = io.ReadFull(r, out) + + if err != nil { + return err + } + } + return nil + + default: + // Something really went wrong + return errors.New("Unknown multiplex destination") } } - return nil } diff --git a/test/e2e/checkpoint_test.go b/test/e2e/checkpoint_test.go index 95ec21433..d452a062b 100644 --- a/test/e2e/checkpoint_test.go +++ b/test/e2e/checkpoint_test.go @@ -347,4 +347,49 @@ var _ = Describe("Podman checkpoint", func() { Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0)) }) + // This test does the same steps which are necessary for migrating + // a container from one host to another + It("podman checkpoint container with export (migration)", func() { + // CRIU does not work with seccomp correctly on RHEL7 + session := podmanTest.Podman([]string{"run", "-it", "--security-opt", "seccomp=unconfined", "-d", ALPINE, "top"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1)) + + result := podmanTest.Podman([]string{"container", "checkpoint", "-l", "-e", "/tmp/checkpoint.tar.gz"}) + result.WaitWithDefaultTimeout() + + Expect(result.ExitCode()).To(Equal(0)) + Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0)) + Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Exited")) + + // Remove all containers to simulate migration + result = podmanTest.Podman([]string{"rm", "-fa"}) + result.WaitWithDefaultTimeout() + Expect(result.ExitCode()).To(Equal(0)) + Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0)) + + result = podmanTest.Podman([]string{"container", "restore", "-i", "/tmp/checkpoint.tar.gz"}) + result.WaitWithDefaultTimeout() + + Expect(result.ExitCode()).To(Equal(0)) + Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1)) + Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Up")) + + // Restore container a second time with different name + result = podmanTest.Podman([]string{"container", "restore", "-i", "/tmp/checkpoint.tar.gz", "-n", "restore_again"}) + result.WaitWithDefaultTimeout() + + Expect(result.ExitCode()).To(Equal(0)) + Expect(podmanTest.NumberOfContainersRunning()).To(Equal(2)) + Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Up")) + + result = podmanTest.Podman([]string{"rm", "-fa"}) + result.WaitWithDefaultTimeout() + Expect(result.ExitCode()).To(Equal(0)) + Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0)) + + // Remove exported checkpoint + os.Remove("/tmp/checkpoint.tar.gz") + }) }) diff --git a/test/e2e/commit_test.go b/test/e2e/commit_test.go index 50577fdfb..e9d274649 100644 --- a/test/e2e/commit_test.go +++ b/test/e2e/commit_test.go @@ -1,5 +1,3 @@ -// +build !remoteclient - package integration import ( @@ -174,6 +172,9 @@ var _ = Describe("Podman commit", func() { }) It("podman commit with volume mounts and --include-volumes", func() { + // We need to figure out how volumes are going to work correctly with the remote + // client. This does not currently work. + SkipIfRemote() s := podmanTest.Podman([]string{"run", "--name", "test1", "-v", "/tmp:/foo", "alpine", "date"}) s.WaitWithDefaultTimeout() Expect(s.ExitCode()).To(Equal(0)) diff --git a/test/e2e/healthcheck_run_test.go b/test/e2e/healthcheck_run_test.go index be54cf1bd..125002bf9 100644 --- a/test/e2e/healthcheck_run_test.go +++ b/test/e2e/healthcheck_run_test.go @@ -5,6 +5,7 @@ package integration import ( "fmt" "os" + "time" . "github.com/containers/libpod/test/utils" . "github.com/onsi/ginkgo" @@ -43,13 +44,24 @@ var _ = Describe("Podman healthcheck run", func() { }) It("podman healthcheck on valid container", func() { + Skip("Extremely consistent flake - reenable on debugging") session := podmanTest.Podman([]string{"run", "-dt", "--name", "hc", healthcheck}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) - hc := podmanTest.Podman([]string{"healthcheck", "run", "hc"}) - hc.WaitWithDefaultTimeout() - Expect(hc.ExitCode()).To(Equal(0)) + exitCode := 999 + + // Buy a little time to get container running + for i := 0; i < 5; i++ { + hc := podmanTest.Podman([]string{"healthcheck", "run", "hc"}) + hc.WaitWithDefaultTimeout() + exitCode = hc.ExitCode() + if exitCode == 0 || i == 4 { + break + } + time.Sleep(1 * time.Second) + } + Expect(exitCode).To(Equal(0)) }) It("podman healthcheck that should fail", func() { diff --git a/vendor.conf b/vendor.conf index 65bfcd77a..a33a52786 100644 --- a/vendor.conf +++ b/vendor.conf @@ -3,7 +3,7 @@ # # TODO: no release, can we find an alternative? github.com/Azure/go-ansiterm d6e3b3328b783f23731bc4d058875b0371ff8109 -github.com/BurntSushi/toml v0.2.0 +github.com/BurntSushi/toml v0.3.1 github.com/Microsoft/go-winio v0.4.11 github.com/Microsoft/hcsshim v0.8.3 github.com/blang/semver v3.5.0 @@ -19,7 +19,7 @@ github.com/containers/image 2c0349c99af7d90694b3faa0e9bde404d407b145 github.com/vbauerster/mpb v3.3.4 github.com/mattn/go-isatty v0.0.4 github.com/VividCortex/ewma v1.1.1 -github.com/containers/storage v1.12.7 +github.com/containers/storage 9b10041d7b2ef767ce9c42b5862b6c51eeb82214 github.com/containers/psgo v1.3.0 github.com/coreos/go-systemd v17 github.com/coreos/pkg v4 @@ -93,7 +93,7 @@ k8s.io/api kubernetes-1.10.13-beta.0 https://github.com/kubernetes/api k8s.io/apimachinery kubernetes-1.10.13-beta.0 https://github.com/kubernetes/apimachinery k8s.io/client-go kubernetes-1.10.13-beta.0 https://github.com/kubernetes/client-go github.com/mrunalp/fileutils 7d4729fb36185a7c1719923406c9d40e54fb93c7 -github.com/containers/buildah bcc5e51a948c1847fab1c932f1a343696a96cafd +github.com/containers/buildah v1.8.3 github.com/varlink/go 0f1d566d194b9d6d48e0d47c5e4d822628919066 # TODO: Gotty has not been updated since 2012. Can we find replacement? github.com/Nvveen/Gotty cd527374f1e5bff4938207604a14f2e38a9cf512 diff --git a/vendor/github.com/BurntSushi/toml/COPYING b/vendor/github.com/BurntSushi/toml/COPYING index 5a8e33254..01b574320 100644 --- a/vendor/github.com/BurntSushi/toml/COPYING +++ b/vendor/github.com/BurntSushi/toml/COPYING @@ -1,14 +1,21 @@ - DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE - Version 2, December 2004 +The MIT License (MIT) - Copyright (C) 2004 Sam Hocevar <sam@hocevar.net> +Copyright (c) 2013 TOML authors - Everyone is permitted to copy and distribute verbatim or modified - copies of this license document, and changing it is allowed as long - as the name is changed. +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: - DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. You just DO WHAT THE FUCK YOU WANT TO. +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/vendor/github.com/BurntSushi/toml/README.md b/vendor/github.com/BurntSushi/toml/README.md index 5a5df6370..7c1b37ecc 100644 --- a/vendor/github.com/BurntSushi/toml/README.md +++ b/vendor/github.com/BurntSushi/toml/README.md @@ -1,17 +1,17 @@ ## TOML parser and encoder for Go with reflection TOML stands for Tom's Obvious, Minimal Language. This Go package provides a -reflection interface similar to Go's standard library `json` and `xml` +reflection interface similar to Go's standard library `json` and `xml` packages. This package also supports the `encoding.TextUnmarshaler` and -`encoding.TextMarshaler` interfaces so that you can define custom data +`encoding.TextMarshaler` interfaces so that you can define custom data representations. (There is an example of this below.) -Spec: https://github.com/mojombo/toml +Spec: https://github.com/toml-lang/toml Compatible with TOML version -[v0.2.0](https://github.com/toml-lang/toml/blob/master/versions/en/toml-v0.2.0.md) +[v0.4.0](https://github.com/toml-lang/toml/blob/master/versions/en/toml-v0.4.0.md) -Documentation: http://godoc.org/github.com/BurntSushi/toml +Documentation: https://godoc.org/github.com/BurntSushi/toml Installation: @@ -26,8 +26,7 @@ go get github.com/BurntSushi/toml/cmd/tomlv tomlv some-toml-file.toml ``` -[](https://travis-ci.org/BurntSushi/toml) - +[](https://travis-ci.org/BurntSushi/toml) [](https://godoc.org/github.com/BurntSushi/toml) ### Testing @@ -87,7 +86,7 @@ type TOML struct { ### Using the `encoding.TextUnmarshaler` interface -Here's an example that automatically parses duration strings into +Here's an example that automatically parses duration strings into `time.Duration` values: ```toml @@ -120,7 +119,7 @@ for _, s := range favorites.Song { } ``` -And you'll also need a `duration` type that satisfies the +And you'll also need a `duration` type that satisfies the `encoding.TextUnmarshaler` interface: ```go @@ -217,4 +216,3 @@ Note that a case insensitive match will be tried if an exact match can't be found. A working example of the above can be found in `_examples/example.{go,toml}`. - diff --git a/vendor/github.com/BurntSushi/toml/decode.go b/vendor/github.com/BurntSushi/toml/decode.go index c26b00c01..b0fd51d5b 100644 --- a/vendor/github.com/BurntSushi/toml/decode.go +++ b/vendor/github.com/BurntSushi/toml/decode.go @@ -10,7 +10,9 @@ import ( "time" ) -var e = fmt.Errorf +func e(format string, args ...interface{}) error { + return fmt.Errorf("toml: "+format, args...) +} // Unmarshaler is the interface implemented by objects that can unmarshal a // TOML description of themselves. @@ -103,6 +105,13 @@ func (md *MetaData) PrimitiveDecode(primValue Primitive, v interface{}) error { // This decoder will not handle cyclic types. If a cyclic type is passed, // `Decode` will not terminate. func Decode(data string, v interface{}) (MetaData, error) { + rv := reflect.ValueOf(v) + if rv.Kind() != reflect.Ptr { + return MetaData{}, e("Decode of non-pointer %s", reflect.TypeOf(v)) + } + if rv.IsNil() { + return MetaData{}, e("Decode of nil %s", reflect.TypeOf(v)) + } p, err := parse(data) if err != nil { return MetaData{}, err @@ -111,7 +120,7 @@ func Decode(data string, v interface{}) (MetaData, error) { p.mapping, p.types, p.ordered, make(map[string]bool, len(p.ordered)), nil, } - return md, md.unify(p.mapping, rvalue(v)) + return md, md.unify(p.mapping, indirect(rv)) } // DecodeFile is just like Decode, except it will automatically read the @@ -211,7 +220,7 @@ func (md *MetaData) unify(data interface{}, rv reflect.Value) error { case reflect.Interface: // we only support empty interfaces. if rv.NumMethod() > 0 { - return e("Unsupported type '%s'.", rv.Kind()) + return e("unsupported type %s", rv.Type()) } return md.unifyAnything(data, rv) case reflect.Float32: @@ -219,7 +228,7 @@ func (md *MetaData) unify(data interface{}, rv reflect.Value) error { case reflect.Float64: return md.unifyFloat64(data, rv) } - return e("Unsupported type '%s'.", rv.Kind()) + return e("unsupported type %s", rv.Kind()) } func (md *MetaData) unifyStruct(mapping interface{}, rv reflect.Value) error { @@ -228,7 +237,8 @@ func (md *MetaData) unifyStruct(mapping interface{}, rv reflect.Value) error { if mapping == nil { return nil } - return mismatch(rv, "map", mapping) + return e("type mismatch for %s: expected table but found %T", + rv.Type().String(), mapping) } for key, datum := range tmap { @@ -253,14 +263,13 @@ func (md *MetaData) unifyStruct(mapping interface{}, rv reflect.Value) error { md.decoded[md.context.add(key).String()] = true md.context = append(md.context, key) if err := md.unify(datum, subv); err != nil { - return e("Type mismatch for '%s.%s': %s", - rv.Type().String(), f.name, err) + return err } md.context = md.context[0 : len(md.context)-1] } else if f.name != "" { // Bad user! No soup for you! - return e("Field '%s.%s' is unexported, and therefore cannot "+ - "be loaded with reflection.", rv.Type().String(), f.name) + return e("cannot write unexported field %s.%s", + rv.Type().String(), f.name) } } } @@ -378,15 +387,15 @@ func (md *MetaData) unifyInt(data interface{}, rv reflect.Value) error { // No bounds checking necessary. case reflect.Int8: if num < math.MinInt8 || num > math.MaxInt8 { - return e("Value '%d' is out of range for int8.", num) + return e("value %d is out of range for int8", num) } case reflect.Int16: if num < math.MinInt16 || num > math.MaxInt16 { - return e("Value '%d' is out of range for int16.", num) + return e("value %d is out of range for int16", num) } case reflect.Int32: if num < math.MinInt32 || num > math.MaxInt32 { - return e("Value '%d' is out of range for int32.", num) + return e("value %d is out of range for int32", num) } } rv.SetInt(num) @@ -397,15 +406,15 @@ func (md *MetaData) unifyInt(data interface{}, rv reflect.Value) error { // No bounds checking necessary. case reflect.Uint8: if num < 0 || unum > math.MaxUint8 { - return e("Value '%d' is out of range for uint8.", num) + return e("value %d is out of range for uint8", num) } case reflect.Uint16: if num < 0 || unum > math.MaxUint16 { - return e("Value '%d' is out of range for uint16.", num) + return e("value %d is out of range for uint16", num) } case reflect.Uint32: if num < 0 || unum > math.MaxUint32 { - return e("Value '%d' is out of range for uint32.", num) + return e("value %d is out of range for uint32", num) } } rv.SetUint(unum) @@ -471,7 +480,7 @@ func rvalue(v interface{}) reflect.Value { // interest to us (like encoding.TextUnmarshaler). func indirect(v reflect.Value) reflect.Value { if v.Kind() != reflect.Ptr { - if v.CanAddr() { + if v.CanSet() { pv := v.Addr() if _, ok := pv.Interface().(TextUnmarshaler); ok { return pv @@ -496,10 +505,5 @@ func isUnifiable(rv reflect.Value) bool { } func badtype(expected string, data interface{}) error { - return e("Expected %s but found '%T'.", expected, data) -} - -func mismatch(user reflect.Value, expected string, data interface{}) error { - return e("Type mismatch for %s. Expected %s but found '%T'.", - user.Type().String(), expected, data) + return e("cannot load TOML value of type %T into a Go %s", data, expected) } diff --git a/vendor/github.com/BurntSushi/toml/decode_meta.go b/vendor/github.com/BurntSushi/toml/decode_meta.go index ef6f545fa..b9914a679 100644 --- a/vendor/github.com/BurntSushi/toml/decode_meta.go +++ b/vendor/github.com/BurntSushi/toml/decode_meta.go @@ -77,9 +77,8 @@ func (k Key) maybeQuoted(i int) string { } if quote { return "\"" + strings.Replace(k[i], "\"", "\\\"", -1) + "\"" - } else { - return k[i] } + return k[i] } func (k Key) add(piece string) Key { diff --git a/vendor/github.com/BurntSushi/toml/doc.go b/vendor/github.com/BurntSushi/toml/doc.go index fe2680004..b371f396e 100644 --- a/vendor/github.com/BurntSushi/toml/doc.go +++ b/vendor/github.com/BurntSushi/toml/doc.go @@ -4,7 +4,7 @@ files via reflection. There is also support for delaying decoding with the Primitive type, and querying the set of keys in a TOML document with the MetaData type. -The specification implemented: https://github.com/mojombo/toml +The specification implemented: https://github.com/toml-lang/toml The sub-command github.com/BurntSushi/toml/cmd/tomlv can be used to verify whether a file is a valid TOML document. It can also be used to print the diff --git a/vendor/github.com/BurntSushi/toml/encode.go b/vendor/github.com/BurntSushi/toml/encode.go index 4e4c97aed..d905c21a2 100644 --- a/vendor/github.com/BurntSushi/toml/encode.go +++ b/vendor/github.com/BurntSushi/toml/encode.go @@ -16,17 +16,17 @@ type tomlEncodeError struct{ error } var ( errArrayMixedElementTypes = errors.New( - "can't encode array with mixed element types") + "toml: cannot encode array with mixed element types") errArrayNilElement = errors.New( - "can't encode array with nil element") + "toml: cannot encode array with nil element") errNonString = errors.New( - "can't encode a map with non-string key type") + "toml: cannot encode a map with non-string key type") errAnonNonStruct = errors.New( - "can't encode an anonymous field that is not a struct") + "toml: cannot encode an anonymous field that is not a struct") errArrayNoTable = errors.New( - "TOML array element can't contain a table") + "toml: TOML array element cannot contain a table") errNoKey = errors.New( - "top-level values must be a Go map or struct") + "toml: top-level values must be Go maps or structs") errAnything = errors.New("") // used in testing ) @@ -148,7 +148,7 @@ func (enc *Encoder) encode(key Key, rv reflect.Value) { case reflect.Struct: enc.eTable(key, rv) default: - panic(e("Unsupported type for key '%s': %s", key, k)) + panic(e("unsupported type for key '%s': %s", key, k)) } } @@ -160,7 +160,7 @@ func (enc *Encoder) eElement(rv reflect.Value) { // Special case time.Time as a primitive. Has to come before // TextMarshaler below because time.Time implements // encoding.TextMarshaler, but we need to always use UTC. - enc.wf(v.In(time.FixedZone("UTC", 0)).Format("2006-01-02T15:04:05Z")) + enc.wf(v.UTC().Format("2006-01-02T15:04:05Z")) return case TextMarshaler: // Special case. Use text marshaler if it's available for this value. @@ -191,7 +191,7 @@ func (enc *Encoder) eElement(rv reflect.Value) { case reflect.String: enc.writeQuoted(rv.String()) default: - panic(e("Unexpected primitive type: %s", rv.Kind())) + panic(e("unexpected primitive type: %s", rv.Kind())) } } @@ -241,7 +241,7 @@ func (enc *Encoder) eArrayOfTables(key Key, rv reflect.Value) { func (enc *Encoder) eTable(key Key, rv reflect.Value) { panicIfInvalidKey(key) if len(key) == 1 { - // Output an extra new line between top-level tables. + // Output an extra newline between top-level tables. // (The newline isn't written if nothing else has been written though.) enc.newline() } @@ -315,10 +315,16 @@ func (enc *Encoder) eStruct(key Key, rv reflect.Value) { t := f.Type switch t.Kind() { case reflect.Struct: - addFields(t, frv, f.Index) - continue + // Treat anonymous struct fields with + // tag names as though they are not + // anonymous, like encoding/json does. + if getOptions(f.Tag).name == "" { + addFields(t, frv, f.Index) + continue + } case reflect.Ptr: - if t.Elem().Kind() == reflect.Struct { + if t.Elem().Kind() == reflect.Struct && + getOptions(f.Tag).name == "" { if !frv.IsNil() { addFields(t.Elem(), frv.Elem(), f.Index) } @@ -347,17 +353,18 @@ func (enc *Encoder) eStruct(key Key, rv reflect.Value) { continue } - tag := sft.Tag.Get("toml") - if tag == "-" { + opts := getOptions(sft.Tag) + if opts.skip { continue } - keyName, opts := getOptions(tag) - if keyName == "" { - keyName = sft.Name + keyName := sft.Name + if opts.name != "" { + keyName = opts.name } - if _, ok := opts["omitempty"]; ok && isEmpty(sf) { + if opts.omitempty && isEmpty(sf) { continue - } else if _, ok := opts["omitzero"]; ok && isZero(sf) { + } + if opts.omitzero && isZero(sf) { continue } @@ -392,9 +399,8 @@ func tomlTypeOfGo(rv reflect.Value) tomlType { case reflect.Array, reflect.Slice: if typeEqual(tomlHash, tomlArrayType(rv)) { return tomlArrayHash - } else { - return tomlArray } + return tomlArray case reflect.Ptr, reflect.Interface: return tomlTypeOfGo(rv.Elem()) case reflect.String: @@ -451,17 +457,30 @@ func tomlArrayType(rv reflect.Value) tomlType { return firstType } -func getOptions(keyName string) (string, map[string]struct{}) { - opts := make(map[string]struct{}) - ss := strings.Split(keyName, ",") - name := ss[0] - if len(ss) > 1 { - for _, opt := range ss { - opts[opt] = struct{}{} +type tagOptions struct { + skip bool // "-" + name string + omitempty bool + omitzero bool +} + +func getOptions(tag reflect.StructTag) tagOptions { + t := tag.Get("toml") + if t == "-" { + return tagOptions{skip: true} + } + var opts tagOptions + parts := strings.Split(t, ",") + opts.name = parts[0] + for _, s := range parts[1:] { + switch s { + case "omitempty": + opts.omitempty = true + case "omitzero": + opts.omitzero = true } } - - return name, opts + return opts } func isZero(rv reflect.Value) bool { diff --git a/vendor/github.com/BurntSushi/toml/lex.go b/vendor/github.com/BurntSushi/toml/lex.go index 9b20b3a81..e0a742a88 100644 --- a/vendor/github.com/BurntSushi/toml/lex.go +++ b/vendor/github.com/BurntSushi/toml/lex.go @@ -3,6 +3,7 @@ package toml import ( "fmt" "strings" + "unicode" "unicode/utf8" ) @@ -29,24 +30,28 @@ const ( itemArrayTableEnd itemKeyStart itemCommentStart + itemInlineTableStart + itemInlineTableEnd ) const ( - eof = 0 - tableStart = '[' - tableEnd = ']' - arrayTableStart = '[' - arrayTableEnd = ']' - tableSep = '.' - keySep = '=' - arrayStart = '[' - arrayEnd = ']' - arrayValTerm = ',' - commentStart = '#' - stringStart = '"' - stringEnd = '"' - rawStringStart = '\'' - rawStringEnd = '\'' + eof = 0 + comma = ',' + tableStart = '[' + tableEnd = ']' + arrayTableStart = '[' + arrayTableEnd = ']' + tableSep = '.' + keySep = '=' + arrayStart = '[' + arrayEnd = ']' + commentStart = '#' + stringStart = '"' + stringEnd = '"' + rawStringStart = '\'' + rawStringEnd = '\'' + inlineTableStart = '{' + inlineTableEnd = '}' ) type stateFn func(lx *lexer) stateFn @@ -55,11 +60,18 @@ type lexer struct { input string start int pos int - width int line int state stateFn items chan item + // Allow for backing up up to three runes. + // This is necessary because TOML contains 3-rune tokens (""" and '''). + prevWidths [3]int + nprev int // how many of prevWidths are in use + // If we emit an eof, we can still back up, but it is not OK to call + // next again. + atEOF bool + // A stack of state functions used to maintain context. // The idea is to reuse parts of the state machine in various places. // For example, values can appear at the top level or within arbitrarily @@ -87,7 +99,7 @@ func (lx *lexer) nextItem() item { func lex(input string) *lexer { lx := &lexer{ - input: input + "\n", + input: input, state: lexTop, line: 1, items: make(chan item, 10), @@ -102,7 +114,7 @@ func (lx *lexer) push(state stateFn) { func (lx *lexer) pop() stateFn { if len(lx.stack) == 0 { - return lx.errorf("BUG in lexer: no states to pop.") + return lx.errorf("BUG in lexer: no states to pop") } last := lx.stack[len(lx.stack)-1] lx.stack = lx.stack[0 : len(lx.stack)-1] @@ -124,16 +136,25 @@ func (lx *lexer) emitTrim(typ itemType) { } func (lx *lexer) next() (r rune) { + if lx.atEOF { + panic("next called after EOF") + } if lx.pos >= len(lx.input) { - lx.width = 0 + lx.atEOF = true return eof } if lx.input[lx.pos] == '\n' { lx.line++ } - r, lx.width = utf8.DecodeRuneInString(lx.input[lx.pos:]) - lx.pos += lx.width + lx.prevWidths[2] = lx.prevWidths[1] + lx.prevWidths[1] = lx.prevWidths[0] + if lx.nprev < 3 { + lx.nprev++ + } + r, w := utf8.DecodeRuneInString(lx.input[lx.pos:]) + lx.prevWidths[0] = w + lx.pos += w return r } @@ -142,9 +163,20 @@ func (lx *lexer) ignore() { lx.start = lx.pos } -// backup steps back one rune. Can be called only once per call of next. +// backup steps back one rune. Can be called only twice between calls to next. func (lx *lexer) backup() { - lx.pos -= lx.width + if lx.atEOF { + lx.atEOF = false + return + } + if lx.nprev < 1 { + panic("backed up too far") + } + w := lx.prevWidths[0] + lx.prevWidths[0] = lx.prevWidths[1] + lx.prevWidths[1] = lx.prevWidths[2] + lx.nprev-- + lx.pos -= w if lx.pos < len(lx.input) && lx.input[lx.pos] == '\n' { lx.line-- } @@ -166,9 +198,22 @@ func (lx *lexer) peek() rune { return r } +// skip ignores all input that matches the given predicate. +func (lx *lexer) skip(pred func(rune) bool) { + for { + r := lx.next() + if pred(r) { + continue + } + lx.backup() + lx.ignore() + return + } +} + // errorf stops all lexing by emitting an error and returning `nil`. // Note that any value that is a character is escaped if it's a special -// character (new lines, tabs, etc.). +// character (newlines, tabs, etc.). func (lx *lexer) errorf(format string, values ...interface{}) stateFn { lx.items <- item{ itemError, @@ -184,7 +229,6 @@ func lexTop(lx *lexer) stateFn { if isWhitespace(r) || isNL(r) { return lexSkip(lx, lexTop) } - switch r { case commentStart: lx.push(lexTop) @@ -193,7 +237,7 @@ func lexTop(lx *lexer) stateFn { return lexTableStart case eof: if lx.pos > lx.start { - return lx.errorf("Unexpected EOF.") + return lx.errorf("unexpected EOF") } lx.emit(itemEOF) return nil @@ -208,12 +252,12 @@ func lexTop(lx *lexer) stateFn { // lexTopEnd is entered whenever a top-level item has been consumed. (A value // or a table.) It must see only whitespace, and will turn back to lexTop -// upon a new line. If it sees EOF, it will quit the lexer successfully. +// upon a newline. If it sees EOF, it will quit the lexer successfully. func lexTopEnd(lx *lexer) stateFn { r := lx.next() switch { case r == commentStart: - // a comment will read to a new line for us. + // a comment will read to a newline for us. lx.push(lexTop) return lexCommentStart case isWhitespace(r): @@ -222,11 +266,11 @@ func lexTopEnd(lx *lexer) stateFn { lx.ignore() return lexTop case r == eof: - lx.ignore() - return lexTop + lx.emit(itemEOF) + return nil } - return lx.errorf("Expected a top-level item to end with a new line, "+ - "comment or EOF, but got %q instead.", r) + return lx.errorf("expected a top-level item to end with a newline, "+ + "comment, or EOF, but got %q instead", r) } // lexTable lexes the beginning of a table. Namely, it makes sure that @@ -253,21 +297,22 @@ func lexTableEnd(lx *lexer) stateFn { func lexArrayTableEnd(lx *lexer) stateFn { if r := lx.next(); r != arrayTableEnd { - return lx.errorf("Expected end of table array name delimiter %q, "+ - "but got %q instead.", arrayTableEnd, r) + return lx.errorf("expected end of table array name delimiter %q, "+ + "but got %q instead", arrayTableEnd, r) } lx.emit(itemArrayTableEnd) return lexTopEnd } func lexTableNameStart(lx *lexer) stateFn { + lx.skip(isWhitespace) switch r := lx.peek(); { case r == tableEnd || r == eof: - return lx.errorf("Unexpected end of table name. (Table names cannot " + - "be empty.)") + return lx.errorf("unexpected end of table name " + + "(table names cannot be empty)") case r == tableSep: - return lx.errorf("Unexpected table separator. (Table names cannot " + - "be empty.)") + return lx.errorf("unexpected table separator " + + "(table names cannot be empty)") case r == stringStart || r == rawStringStart: lx.ignore() lx.push(lexTableNameEnd) @@ -277,24 +322,22 @@ func lexTableNameStart(lx *lexer) stateFn { } } -// lexTableName lexes the name of a table. It assumes that at least one +// lexBareTableName lexes the name of a table. It assumes that at least one // valid character for the table has already been read. func lexBareTableName(lx *lexer) stateFn { - switch r := lx.next(); { - case isBareKeyChar(r): + r := lx.next() + if isBareKeyChar(r) { return lexBareTableName - case r == tableSep || r == tableEnd: - lx.backup() - lx.emitTrim(itemText) - return lexTableNameEnd - default: - return lx.errorf("Bare keys cannot contain %q.", r) } + lx.backup() + lx.emit(itemText) + return lexTableNameEnd } // lexTableNameEnd reads the end of a piece of a table name, optionally // consuming whitespace. func lexTableNameEnd(lx *lexer) stateFn { + lx.skip(isWhitespace) switch r := lx.next(); { case isWhitespace(r): return lexTableNameEnd @@ -304,8 +347,8 @@ func lexTableNameEnd(lx *lexer) stateFn { case r == tableEnd: return lx.pop() default: - return lx.errorf("Expected '.' or ']' to end table name, but got %q "+ - "instead.", r) + return lx.errorf("expected '.' or ']' to end table name, "+ + "but got %q instead", r) } } @@ -315,7 +358,7 @@ func lexKeyStart(lx *lexer) stateFn { r := lx.peek() switch { case r == keySep: - return lx.errorf("Unexpected key separator %q.", keySep) + return lx.errorf("unexpected key separator %q", keySep) case isWhitespace(r) || isNL(r): lx.next() return lexSkip(lx, lexKeyStart) @@ -338,14 +381,15 @@ func lexBareKey(lx *lexer) stateFn { case isBareKeyChar(r): return lexBareKey case isWhitespace(r): - lx.emitTrim(itemText) + lx.backup() + lx.emit(itemText) return lexKeyEnd case r == keySep: lx.backup() - lx.emitTrim(itemText) + lx.emit(itemText) return lexKeyEnd default: - return lx.errorf("Bare keys cannot contain %q.", r) + return lx.errorf("bare keys cannot contain %q", r) } } @@ -358,7 +402,7 @@ func lexKeyEnd(lx *lexer) stateFn { case isWhitespace(r): return lexSkip(lx, lexKeyEnd) default: - return lx.errorf("Expected key separator %q, but got %q instead.", + return lx.errorf("expected key separator %q, but got %q instead", keySep, r) } } @@ -367,20 +411,26 @@ func lexKeyEnd(lx *lexer) stateFn { // lexValue will ignore whitespace. // After a value is lexed, the last state on the next is popped and returned. func lexValue(lx *lexer) stateFn { - // We allow whitespace to precede a value, but NOT new lines. - // In array syntax, the array states are responsible for ignoring new - // lines. + // We allow whitespace to precede a value, but NOT newlines. + // In array syntax, the array states are responsible for ignoring newlines. r := lx.next() - if isWhitespace(r) { + switch { + case isWhitespace(r): return lexSkip(lx, lexValue) + case isDigit(r): + lx.backup() // avoid an extra state and use the same as above + return lexNumberOrDateStart } - - switch { - case r == arrayStart: + switch r { + case arrayStart: lx.ignore() lx.emit(itemArray) return lexArrayValue - case r == stringStart: + case inlineTableStart: + lx.ignore() + lx.emit(itemInlineTableStart) + return lexInlineTableValue + case stringStart: if lx.accept(stringStart) { if lx.accept(stringStart) { lx.ignore() // Ignore """ @@ -390,7 +440,7 @@ func lexValue(lx *lexer) stateFn { } lx.ignore() // ignore the '"' return lexString - case r == rawStringStart: + case rawStringStart: if lx.accept(rawStringStart) { if lx.accept(rawStringStart) { lx.ignore() // Ignore """ @@ -400,23 +450,24 @@ func lexValue(lx *lexer) stateFn { } lx.ignore() // ignore the "'" return lexRawString - case r == 't': - return lexTrue - case r == 'f': - return lexFalse - case r == '-': + case '+', '-': return lexNumberStart - case isDigit(r): - lx.backup() // avoid an extra state and use the same as above - return lexNumberOrDateStart - case r == '.': // special error case, be kind to users - return lx.errorf("Floats must start with a digit, not '.'.") + case '.': // special error case, be kind to users + return lx.errorf("floats must start with a digit, not '.'") + } + if unicode.IsLetter(r) { + // Be permissive here; lexBool will give a nice error if the + // user wrote something like + // x = foo + // (i.e. not 'true' or 'false' but is something else word-like.) + lx.backup() + return lexBool } - return lx.errorf("Expected value but found %q instead.", r) + return lx.errorf("expected value but found %q instead", r) } // lexArrayValue consumes one value in an array. It assumes that '[' or ',' -// have already been consumed. All whitespace and new lines are ignored. +// have already been consumed. All whitespace and newlines are ignored. func lexArrayValue(lx *lexer) stateFn { r := lx.next() switch { @@ -425,10 +476,11 @@ func lexArrayValue(lx *lexer) stateFn { case r == commentStart: lx.push(lexArrayValue) return lexCommentStart - case r == arrayValTerm: - return lx.errorf("Unexpected array value terminator %q.", - arrayValTerm) + case r == comma: + return lx.errorf("unexpected comma") case r == arrayEnd: + // NOTE(caleb): The spec isn't clear about whether you can have + // a trailing comma or not, so we'll allow it. return lexArrayEnd } @@ -437,8 +489,9 @@ func lexArrayValue(lx *lexer) stateFn { return lexValue } -// lexArrayValueEnd consumes the cruft between values of an array. Namely, -// it ignores whitespace and expects either a ',' or a ']'. +// lexArrayValueEnd consumes everything between the end of an array value and +// the next value (or the end of the array): it ignores whitespace and newlines +// and expects either a ',' or a ']'. func lexArrayValueEnd(lx *lexer) stateFn { r := lx.next() switch { @@ -447,31 +500,88 @@ func lexArrayValueEnd(lx *lexer) stateFn { case r == commentStart: lx.push(lexArrayValueEnd) return lexCommentStart - case r == arrayValTerm: + case r == comma: lx.ignore() return lexArrayValue // move on to the next value case r == arrayEnd: return lexArrayEnd } - return lx.errorf("Expected an array value terminator %q or an array "+ - "terminator %q, but got %q instead.", arrayValTerm, arrayEnd, r) + return lx.errorf( + "expected a comma or array terminator %q, but got %q instead", + arrayEnd, r, + ) } -// lexArrayEnd finishes the lexing of an array. It assumes that a ']' has -// just been consumed. +// lexArrayEnd finishes the lexing of an array. +// It assumes that a ']' has just been consumed. func lexArrayEnd(lx *lexer) stateFn { lx.ignore() lx.emit(itemArrayEnd) return lx.pop() } +// lexInlineTableValue consumes one key/value pair in an inline table. +// It assumes that '{' or ',' have already been consumed. Whitespace is ignored. +func lexInlineTableValue(lx *lexer) stateFn { + r := lx.next() + switch { + case isWhitespace(r): + return lexSkip(lx, lexInlineTableValue) + case isNL(r): + return lx.errorf("newlines not allowed within inline tables") + case r == commentStart: + lx.push(lexInlineTableValue) + return lexCommentStart + case r == comma: + return lx.errorf("unexpected comma") + case r == inlineTableEnd: + return lexInlineTableEnd + } + lx.backup() + lx.push(lexInlineTableValueEnd) + return lexKeyStart +} + +// lexInlineTableValueEnd consumes everything between the end of an inline table +// key/value pair and the next pair (or the end of the table): +// it ignores whitespace and expects either a ',' or a '}'. +func lexInlineTableValueEnd(lx *lexer) stateFn { + r := lx.next() + switch { + case isWhitespace(r): + return lexSkip(lx, lexInlineTableValueEnd) + case isNL(r): + return lx.errorf("newlines not allowed within inline tables") + case r == commentStart: + lx.push(lexInlineTableValueEnd) + return lexCommentStart + case r == comma: + lx.ignore() + return lexInlineTableValue + case r == inlineTableEnd: + return lexInlineTableEnd + } + return lx.errorf("expected a comma or an inline table terminator %q, "+ + "but got %q instead", inlineTableEnd, r) +} + +// lexInlineTableEnd finishes the lexing of an inline table. +// It assumes that a '}' has just been consumed. +func lexInlineTableEnd(lx *lexer) stateFn { + lx.ignore() + lx.emit(itemInlineTableEnd) + return lx.pop() +} + // lexString consumes the inner contents of a string. It assumes that the // beginning '"' has already been consumed and ignored. func lexString(lx *lexer) stateFn { r := lx.next() switch { + case r == eof: + return lx.errorf("unexpected EOF") case isNL(r): - return lx.errorf("Strings cannot contain new lines.") + return lx.errorf("strings cannot contain newlines") case r == '\\': lx.push(lexString) return lexStringEscape @@ -488,11 +598,12 @@ func lexString(lx *lexer) stateFn { // lexMultilineString consumes the inner contents of a string. It assumes that // the beginning '"""' has already been consumed and ignored. func lexMultilineString(lx *lexer) stateFn { - r := lx.next() - switch { - case r == '\\': + switch lx.next() { + case eof: + return lx.errorf("unexpected EOF") + case '\\': return lexMultilineStringEscape - case r == stringEnd: + case stringEnd: if lx.accept(stringEnd) { if lx.accept(stringEnd) { lx.backup() @@ -516,8 +627,10 @@ func lexMultilineString(lx *lexer) stateFn { func lexRawString(lx *lexer) stateFn { r := lx.next() switch { + case r == eof: + return lx.errorf("unexpected EOF") case isNL(r): - return lx.errorf("Strings cannot contain new lines.") + return lx.errorf("strings cannot contain newlines") case r == rawStringEnd: lx.backup() lx.emit(itemRawString) @@ -529,12 +642,13 @@ func lexRawString(lx *lexer) stateFn { } // lexMultilineRawString consumes a raw string. Nothing can be escaped in such -// a string. It assumes that the beginning "'" has already been consumed and +// a string. It assumes that the beginning "'''" has already been consumed and // ignored. func lexMultilineRawString(lx *lexer) stateFn { - r := lx.next() - switch { - case r == rawStringEnd: + switch lx.next() { + case eof: + return lx.errorf("unexpected EOF") + case rawStringEnd: if lx.accept(rawStringEnd) { if lx.accept(rawStringEnd) { lx.backup() @@ -559,11 +673,10 @@ func lexMultilineStringEscape(lx *lexer) stateFn { // Handle the special case first: if isNL(lx.next()) { return lexMultilineString - } else { - lx.backup() - lx.push(lexMultilineString) - return lexStringEscape(lx) } + lx.backup() + lx.push(lexMultilineString) + return lexStringEscape(lx) } func lexStringEscape(lx *lexer) stateFn { @@ -588,10 +701,9 @@ func lexStringEscape(lx *lexer) stateFn { case 'U': return lexLongUnicodeEscape } - return lx.errorf("Invalid escape character %q. Only the following "+ + return lx.errorf("invalid escape character %q; only the following "+ "escape characters are allowed: "+ - "\\b, \\t, \\n, \\f, \\r, \\\", \\/, \\\\, "+ - "\\uXXXX and \\UXXXXXXXX.", r) + `\b, \t, \n, \f, \r, \", \\, \uXXXX, and \UXXXXXXXX`, r) } func lexShortUnicodeEscape(lx *lexer) stateFn { @@ -599,8 +711,8 @@ func lexShortUnicodeEscape(lx *lexer) stateFn { for i := 0; i < 4; i++ { r = lx.next() if !isHexadecimal(r) { - return lx.errorf("Expected four hexadecimal digits after '\\u', "+ - "but got '%s' instead.", lx.current()) + return lx.errorf(`expected four hexadecimal digits after '\u', `+ + "but got %q instead", lx.current()) } } return lx.pop() @@ -611,40 +723,43 @@ func lexLongUnicodeEscape(lx *lexer) stateFn { for i := 0; i < 8; i++ { r = lx.next() if !isHexadecimal(r) { - return lx.errorf("Expected eight hexadecimal digits after '\\U', "+ - "but got '%s' instead.", lx.current()) + return lx.errorf(`expected eight hexadecimal digits after '\U', `+ + "but got %q instead", lx.current()) } } return lx.pop() } -// lexNumberOrDateStart consumes either a (positive) integer, float or -// datetime. It assumes that NO negative sign has been consumed. +// lexNumberOrDateStart consumes either an integer, a float, or datetime. func lexNumberOrDateStart(lx *lexer) stateFn { r := lx.next() - if !isDigit(r) { - if r == '.' { - return lx.errorf("Floats must start with a digit, not '.'.") - } else { - return lx.errorf("Expected a digit but got %q.", r) - } + if isDigit(r) { + return lexNumberOrDate } - return lexNumberOrDate + switch r { + case '_': + return lexNumber + case 'e', 'E': + return lexFloat + case '.': + return lx.errorf("floats must start with a digit, not '.'") + } + return lx.errorf("expected a digit but got %q", r) } -// lexNumberOrDate consumes either a (positive) integer, float or datetime. +// lexNumberOrDate consumes either an integer, float or datetime. func lexNumberOrDate(lx *lexer) stateFn { r := lx.next() - switch { - case r == '-': - if lx.pos-lx.start != 5 { - return lx.errorf("All ISO8601 dates must be in full Zulu form.") - } - return lexDateAfterYear - case isDigit(r): + if isDigit(r) { return lexNumberOrDate - case r == '.': - return lexFloatStart + } + switch r { + case '-': + return lexDatetime + case '_': + return lexNumber + case '.', 'e', 'E': + return lexFloat } lx.backup() @@ -652,46 +767,34 @@ func lexNumberOrDate(lx *lexer) stateFn { return lx.pop() } -// lexDateAfterYear consumes a full Zulu Datetime in ISO8601 format. -// It assumes that "YYYY-" has already been consumed. -func lexDateAfterYear(lx *lexer) stateFn { - formats := []rune{ - // digits are '0'. - // everything else is direct equality. - '0', '0', '-', '0', '0', - 'T', - '0', '0', ':', '0', '0', ':', '0', '0', - 'Z', +// lexDatetime consumes a Datetime, to a first approximation. +// The parser validates that it matches one of the accepted formats. +func lexDatetime(lx *lexer) stateFn { + r := lx.next() + if isDigit(r) { + return lexDatetime } - for _, f := range formats { - r := lx.next() - if f == '0' { - if !isDigit(r) { - return lx.errorf("Expected digit in ISO8601 datetime, "+ - "but found %q instead.", r) - } - } else if f != r { - return lx.errorf("Expected %q in ISO8601 datetime, "+ - "but found %q instead.", f, r) - } + switch r { + case '-', 'T', ':', '.', 'Z', '+': + return lexDatetime } + + lx.backup() lx.emit(itemDatetime) return lx.pop() } -// lexNumberStart consumes either an integer or a float. It assumes that -// a negative sign has already been read, but that *no* digits have been -// consumed. lexNumberStart will move to the appropriate integer or float -// states. +// lexNumberStart consumes either an integer or a float. It assumes that a sign +// has already been read, but that *no* digits have been consumed. +// lexNumberStart will move to the appropriate integer or float states. func lexNumberStart(lx *lexer) stateFn { - // we MUST see a digit. Even floats have to start with a digit. + // We MUST see a digit. Even floats have to start with a digit. r := lx.next() if !isDigit(r) { if r == '.' { - return lx.errorf("Floats must start with a digit, not '.'.") - } else { - return lx.errorf("Expected a digit but got %q.", r) + return lx.errorf("floats must start with a digit, not '.'") } + return lx.errorf("expected a digit but got %q", r) } return lexNumber } @@ -699,11 +802,14 @@ func lexNumberStart(lx *lexer) stateFn { // lexNumber consumes an integer or a float after seeing the first digit. func lexNumber(lx *lexer) stateFn { r := lx.next() - switch { - case isDigit(r): + if isDigit(r) { return lexNumber - case r == '.': - return lexFloatStart + } + switch r { + case '_': + return lexNumber + case '.', 'e', 'E': + return lexFloat } lx.backup() @@ -711,60 +817,42 @@ func lexNumber(lx *lexer) stateFn { return lx.pop() } -// lexFloatStart starts the consumption of digits of a float after a '.'. -// Namely, at least one digit is required. -func lexFloatStart(lx *lexer) stateFn { - r := lx.next() - if !isDigit(r) { - return lx.errorf("Floats must have a digit after the '.', but got "+ - "%q instead.", r) - } - return lexFloat -} - -// lexFloat consumes the digits of a float after a '.'. -// Assumes that one digit has been consumed after a '.' already. +// lexFloat consumes the elements of a float. It allows any sequence of +// float-like characters, so floats emitted by the lexer are only a first +// approximation and must be validated by the parser. func lexFloat(lx *lexer) stateFn { r := lx.next() if isDigit(r) { return lexFloat } + switch r { + case '_', '.', '-', '+', 'e', 'E': + return lexFloat + } lx.backup() lx.emit(itemFloat) return lx.pop() } -// lexConst consumes the s[1:] in s. It assumes that s[0] has already been -// consumed. -func lexConst(lx *lexer, s string) stateFn { - for i := range s[1:] { - if r := lx.next(); r != rune(s[i+1]) { - return lx.errorf("Expected %q, but found %q instead.", s[:i+1], - s[:i]+string(r)) +// lexBool consumes a bool string: 'true' or 'false. +func lexBool(lx *lexer) stateFn { + var rs []rune + for { + r := lx.next() + if !unicode.IsLetter(r) { + lx.backup() + break } + rs = append(rs, r) } - return nil -} - -// lexTrue consumes the "rue" in "true". It assumes that 't' has already -// been consumed. -func lexTrue(lx *lexer) stateFn { - if fn := lexConst(lx, "true"); fn != nil { - return fn - } - lx.emit(itemBool) - return lx.pop() -} - -// lexFalse consumes the "alse" in "false". It assumes that 'f' has already -// been consumed. -func lexFalse(lx *lexer) stateFn { - if fn := lexConst(lx, "false"); fn != nil { - return fn + s := string(rs) + switch s { + case "true", "false": + lx.emit(itemBool) + return lx.pop() } - lx.emit(itemBool) - return lx.pop() + return lx.errorf("expected value but found %q instead", s) } // lexCommentStart begins the lexing of a comment. It will emit @@ -776,7 +864,7 @@ func lexCommentStart(lx *lexer) stateFn { } // lexComment lexes an entire comment. It assumes that '#' has been consumed. -// It will consume *up to* the first new line character, and pass control +// It will consume *up to* the first newline character, and pass control // back to the last state on the stack. func lexComment(lx *lexer) stateFn { r := lx.peek() @@ -834,13 +922,7 @@ func (itype itemType) String() string { return "EOF" case itemText: return "Text" - case itemString: - return "String" - case itemRawString: - return "String" - case itemMultilineString: - return "String" - case itemRawMultilineString: + case itemString, itemRawString, itemMultilineString, itemRawMultilineString: return "String" case itemBool: return "Bool" diff --git a/vendor/github.com/BurntSushi/toml/parse.go b/vendor/github.com/BurntSushi/toml/parse.go index 6a82e84f6..50869ef92 100644 --- a/vendor/github.com/BurntSushi/toml/parse.go +++ b/vendor/github.com/BurntSushi/toml/parse.go @@ -2,7 +2,6 @@ package toml import ( "fmt" - "log" "strconv" "strings" "time" @@ -81,7 +80,7 @@ func (p *parser) next() item { } func (p *parser) bug(format string, v ...interface{}) { - log.Panicf("BUG: %s\n\n", fmt.Sprintf(format, v...)) + panic(fmt.Sprintf("BUG: "+format+"\n\n", v...)) } func (p *parser) expect(typ itemType) item { @@ -179,10 +178,18 @@ func (p *parser) value(it item) (interface{}, tomlType) { } p.bug("Expected boolean value, but got '%s'.", it.val) case itemInteger: - num, err := strconv.ParseInt(it.val, 10, 64) + if !numUnderscoresOK(it.val) { + p.panicf("Invalid integer %q: underscores must be surrounded by digits", + it.val) + } + val := strings.Replace(it.val, "_", "", -1) + num, err := strconv.ParseInt(val, 10, 64) if err != nil { - // See comment below for floats describing why we make a - // distinction between a bug and a user error. + // Distinguish integer values. Normally, it'd be a bug if the lexer + // provides an invalid integer, but it's possible that the number is + // out of range of valid values (which the lexer cannot determine). + // So mark the former as a bug but the latter as a legitimate user + // error. if e, ok := err.(*strconv.NumError); ok && e.Err == strconv.ErrRange { @@ -194,29 +201,57 @@ func (p *parser) value(it item) (interface{}, tomlType) { } return num, p.typeOfPrimitive(it) case itemFloat: - num, err := strconv.ParseFloat(it.val, 64) + parts := strings.FieldsFunc(it.val, func(r rune) bool { + switch r { + case '.', 'e', 'E': + return true + } + return false + }) + for _, part := range parts { + if !numUnderscoresOK(part) { + p.panicf("Invalid float %q: underscores must be "+ + "surrounded by digits", it.val) + } + } + if !numPeriodsOK(it.val) { + // As a special case, numbers like '123.' or '1.e2', + // which are valid as far as Go/strconv are concerned, + // must be rejected because TOML says that a fractional + // part consists of '.' followed by 1+ digits. + p.panicf("Invalid float %q: '.' must be followed "+ + "by one or more digits", it.val) + } + val := strings.Replace(it.val, "_", "", -1) + num, err := strconv.ParseFloat(val, 64) if err != nil { - // Distinguish float values. Normally, it'd be a bug if the lexer - // provides an invalid float, but it's possible that the float is - // out of range of valid values (which the lexer cannot determine). - // So mark the former as a bug but the latter as a legitimate user - // error. - // - // This is also true for integers. if e, ok := err.(*strconv.NumError); ok && e.Err == strconv.ErrRange { p.panicf("Float '%s' is out of the range of 64-bit "+ "IEEE-754 floating-point numbers.", it.val) } else { - p.bug("Expected float value, but got '%s'.", it.val) + p.panicf("Invalid float value: %q", it.val) } } return num, p.typeOfPrimitive(it) case itemDatetime: - t, err := time.Parse("2006-01-02T15:04:05Z", it.val) - if err != nil { - p.panicf("Invalid RFC3339 Zulu DateTime: '%s'.", it.val) + var t time.Time + var ok bool + var err error + for _, format := range []string{ + "2006-01-02T15:04:05Z07:00", + "2006-01-02T15:04:05", + "2006-01-02", + } { + t, err = time.ParseInLocation(format, it.val, time.Local) + if err == nil { + ok = true + break + } + } + if !ok { + p.panicf("Invalid TOML Datetime: %q.", it.val) } return t, p.typeOfPrimitive(it) case itemArray: @@ -234,11 +269,75 @@ func (p *parser) value(it item) (interface{}, tomlType) { types = append(types, typ) } return array, p.typeOfArray(types) + case itemInlineTableStart: + var ( + hash = make(map[string]interface{}) + outerContext = p.context + outerKey = p.currentKey + ) + + p.context = append(p.context, p.currentKey) + p.currentKey = "" + for it := p.next(); it.typ != itemInlineTableEnd; it = p.next() { + if it.typ != itemKeyStart { + p.bug("Expected key start but instead found %q, around line %d", + it.val, p.approxLine) + } + if it.typ == itemCommentStart { + p.expect(itemText) + continue + } + + // retrieve key + k := p.next() + p.approxLine = k.line + kname := p.keyString(k) + + // retrieve value + p.currentKey = kname + val, typ := p.value(p.next()) + // make sure we keep metadata up to date + p.setType(kname, typ) + p.ordered = append(p.ordered, p.context.add(p.currentKey)) + hash[kname] = val + } + p.context = outerContext + p.currentKey = outerKey + return hash, tomlHash } p.bug("Unexpected value type: %s", it.typ) panic("unreachable") } +// numUnderscoresOK checks whether each underscore in s is surrounded by +// characters that are not underscores. +func numUnderscoresOK(s string) bool { + accept := false + for _, r := range s { + if r == '_' { + if !accept { + return false + } + accept = false + continue + } + accept = true + } + return accept +} + +// numPeriodsOK checks whether every period in s is followed by a digit. +func numPeriodsOK(s string) bool { + period := false + for _, r := range s { + if period && !isDigit(r) { + return false + } + period = r == '.' + } + return !period +} + // establishContext sets the current context of the parser, // where the context is either a hash or an array of hashes. Which one is // set depends on the value of the `array` parameter. diff --git a/vendor/github.com/BurntSushi/toml/type_fields.go b/vendor/github.com/BurntSushi/toml/type_fields.go index 6da608af4..608997c22 100644 --- a/vendor/github.com/BurntSushi/toml/type_fields.go +++ b/vendor/github.com/BurntSushi/toml/type_fields.go @@ -95,8 +95,8 @@ func typeFields(t reflect.Type) []field { if sf.PkgPath != "" && !sf.Anonymous { // unexported continue } - name, _ := getOptions(sf.Tag.Get("toml")) - if name == "-" { + opts := getOptions(sf.Tag) + if opts.skip { continue } index := make([]int, len(f.index)+1) @@ -110,8 +110,9 @@ func typeFields(t reflect.Type) []field { } // Record found field and index sequence. - if name != "" || !sf.Anonymous || ft.Kind() != reflect.Struct { - tagged := name != "" + if opts.name != "" || !sf.Anonymous || ft.Kind() != reflect.Struct { + tagged := opts.name != "" + name := opts.name if name == "" { name = sf.Name } diff --git a/vendor/github.com/containers/buildah/add.go b/vendor/github.com/containers/buildah/add.go index 589e090a8..11bfb6a12 100644 --- a/vendor/github.com/containers/buildah/add.go +++ b/vendor/github.com/containers/buildah/add.go @@ -14,6 +14,7 @@ import ( "github.com/containers/buildah/pkg/chrootuser" "github.com/containers/buildah/util" "github.com/containers/storage/pkg/archive" + "github.com/containers/storage/pkg/fileutils" "github.com/containers/storage/pkg/idtools" "github.com/containers/storage/pkg/system" "github.com/opencontainers/runtime-spec/specs-go" @@ -89,7 +90,10 @@ func addURL(destination, srcurl string, owner idtools.IDPair, hasher io.Writer) // filesystem, optionally extracting contents of local files that look like // non-empty archives. func (b *Builder) Add(destination string, extract bool, options AddAndCopyOptions, source ...string) error { - excludes := dockerIgnoreHelper(options.Excludes, options.ContextDir) + excludes, err := dockerIgnoreMatcher(options.Excludes, options.ContextDir) + if err != nil { + return err + } mountPoint, err := b.Mount(b.MountLabel) if err != nil { return err @@ -100,7 +104,7 @@ func (b *Builder) Add(destination string, extract bool, options AddAndCopyOption } }() // Find out which user (and group) the destination should belong to. - user, err := b.user(mountPoint, options.Chown) + user, _, err := b.user(mountPoint, options.Chown) if err != nil { return err } @@ -153,12 +157,12 @@ func (b *Builder) Add(destination string, extract bool, options AddAndCopyOption } // user returns the user (and group) information which the destination should belong to. -func (b *Builder) user(mountPoint string, userspec string) (specs.User, error) { +func (b *Builder) user(mountPoint string, userspec string) (specs.User, string, error) { if userspec == "" { userspec = b.User() } - uid, gid, err := chrootuser.GetUser(mountPoint, userspec) + uid, gid, homeDir, err := chrootuser.GetUser(mountPoint, userspec) u := specs.User{ UID: uid, GID: gid, @@ -175,45 +179,48 @@ func (b *Builder) user(mountPoint string, userspec string) (specs.User, error) { } } - return u, err + return u, homeDir, err } -// dockerIgnore struct keep info from .dockerignore -type dockerIgnore struct { - ExcludePath string - IsExcluded bool -} - -// dockerIgnoreHelper returns the lines from .dockerignore file without the comments -// and reverses the order -func dockerIgnoreHelper(lines []string, contextDir string) []dockerIgnore { - var excludes []dockerIgnore - // the last match of a file in the .dockerignmatches determines whether it is included or excluded - // reverse the order - for i := len(lines) - 1; i >= 0; i-- { - exclude := lines[i] - // ignore the comment in .dockerignore - if strings.HasPrefix(exclude, "#") || len(exclude) == 0 { +// dockerIgnoreMatcher returns a matcher based on the contents of the .dockerignore file under contextDir +func dockerIgnoreMatcher(lines []string, contextDir string) (*fileutils.PatternMatcher, error) { + // if there's no context dir, there's no .dockerignore file to consult + if contextDir == "" { + return nil, nil + } + patterns := []string{".dockerignore"} + for _, ignoreSpec := range lines { + ignoreSpec = strings.TrimSpace(ignoreSpec) + // ignore comments passed back from .dockerignore + if ignoreSpec == "" || ignoreSpec[0] == '#' { continue } - excludeFlag := true - if strings.HasPrefix(exclude, "!") { - exclude = strings.TrimPrefix(exclude, "!") - excludeFlag = false + // if the spec starts with '!' it means the pattern + // should be included. make a note so that we can move + // it to the front of the updated pattern + includeFlag := "" + if strings.HasPrefix(ignoreSpec, "!") { + includeFlag = "!" + ignoreSpec = ignoreSpec[1:] } - excludes = append(excludes, dockerIgnore{ExcludePath: filepath.Join(contextDir, exclude), IsExcluded: excludeFlag}) + if ignoreSpec == "" { + continue + } + patterns = append(patterns, includeFlag+filepath.Join(contextDir, ignoreSpec)) } - if len(excludes) != 0 { - excludes = append(excludes, dockerIgnore{ExcludePath: filepath.Join(contextDir, ".dockerignore"), IsExcluded: true}) + // if there are no patterns, save time by not constructing the object + if len(patterns) == 0 { + return nil, nil } - return excludes -} - -func addHelper(excludes []dockerIgnore, extract bool, dest string, destfi os.FileInfo, hostOwner idtools.IDPair, options AddAndCopyOptions, copyFileWithTar, copyWithTar, untarPath func(src, dest string) error, source ...string) error { - dirsInDockerignore, err := getDirsInDockerignore(options.ContextDir, excludes) + // return a matcher object + matcher, err := fileutils.NewPatternMatcher(patterns) if err != nil { - return errors.Wrapf(err, "error checking directories in .dockerignore") + return nil, errors.Wrapf(err, "error creating file matcher using patterns %v", patterns) } + return matcher, nil +} + +func addHelper(excludes *fileutils.PatternMatcher, extract bool, dest string, destfi os.FileInfo, hostOwner idtools.IDPair, options AddAndCopyOptions, copyFileWithTar, copyWithTar, untarPath func(src, dest string) error, source ...string) error { for _, src := range source { if strings.HasPrefix(src, "http://") || strings.HasPrefix(src, "https://") { // We assume that source is a file, and we're copying @@ -242,7 +249,7 @@ func addHelper(excludes []dockerIgnore, extract bool, dest string, destfi os.Fil if len(glob) == 0 { return errors.Wrapf(syscall.ENOENT, "no files found matching %q", src) } - outer: + for _, gsrc := range glob { esrc, err := filepath.EvalSymlinks(gsrc) if err != nil { @@ -261,7 +268,7 @@ func addHelper(excludes []dockerIgnore, extract bool, dest string, destfi os.Fil return errors.Wrapf(err, "error creating directory %q", dest) } logrus.Debugf("copying %q to %q", esrc+string(os.PathSeparator)+"*", dest+string(os.PathSeparator)+"*") - if len(excludes) == 0 { + if excludes == nil { if err = copyWithTar(esrc, dest); err != nil { return errors.Wrapf(err, "error copying %q to %q", esrc, dest) } @@ -271,23 +278,12 @@ func addHelper(excludes []dockerIgnore, extract bool, dest string, destfi os.Fil if err != nil { return err } - for _, exclude := range excludes { - match, err := filepath.Match(filepath.Clean(exclude.ExcludePath), filepath.Clean(path)) - if err != nil { - return err - } - prefix, exist := dirsInDockerignore[exclude.ExcludePath] - hasPrefix := false - if exist { - hasPrefix = filepath.HasPrefix(path, prefix) - } - if !(match || hasPrefix) { - continue - } - if (hasPrefix && exclude.IsExcluded) || (match && exclude.IsExcluded) { - return nil - } - break + skip, err := excludes.Matches(path) + if err != nil { + return errors.Wrapf(err, "error checking if %s is an excluded path", path) + } + if skip { + return nil } // combine the filename with the dest directory fpath, err := filepath.Rel(esrc, path) @@ -297,8 +293,8 @@ func addHelper(excludes []dockerIgnore, extract bool, dest string, destfi os.Fil mtime := info.ModTime() atime := mtime times := []syscall.Timespec{ - {Sec: atime.Unix(), Nsec: atime.UnixNano() % 1000000000}, - {Sec: mtime.Unix(), Nsec: mtime.UnixNano() % 1000000000}, + syscall.NsecToTimespec(atime.Unix()), + syscall.NsecToTimespec(mtime.Unix()), } if info.IsDir() { return addHelperDirectory(esrc, path, filepath.Join(dest, fpath), info, hostOwner, times) @@ -320,20 +316,6 @@ func addHelper(excludes []dockerIgnore, extract bool, dest string, destfi os.Fil continue } - for _, exclude := range excludes { - match, err := filepath.Match(filepath.Clean(exclude.ExcludePath), esrc) - if err != nil { - return err - } - if !match { - continue - } - if exclude.IsExcluded { - continue outer - } - break - } - if !extract || !archive.IsArchivePath(esrc) { // This source is a file, and either it's not an // archive, or we don't care whether or not it's an @@ -349,6 +331,7 @@ func addHelper(excludes []dockerIgnore, extract bool, dest string, destfi os.Fil } continue } + // We're extracting an archive into the destination directory. logrus.Debugf("extracting contents of %q into %q", esrc, dest) if err = untarPath(esrc, dest); err != nil { @@ -381,7 +364,15 @@ func addHelperSymlink(src, dest string, info os.FileInfo, hostOwner idtools.IDPa return errors.Wrapf(err, "error reading contents of symbolic link at %q", src) } if err = os.Symlink(linkContents, dest); err != nil { - return errors.Wrapf(err, "error creating symbolic link to %q at %q", linkContents, dest) + if !os.IsExist(err) { + return errors.Wrapf(err, "error creating symbolic link to %q at %q", linkContents, dest) + } + if err = os.RemoveAll(dest); err != nil { + return errors.Wrapf(err, "error clearing symbolic link target %q", dest) + } + if err = os.Symlink(linkContents, dest); err != nil { + return errors.Wrapf(err, "error creating symbolic link to %q at %q (second try)", linkContents, dest) + } } if err = idtools.SafeLchown(dest, hostOwner.UID, hostOwner.GID); err != nil { return errors.Wrapf(err, "error setting owner of symbolic link %q to %d:%d", dest, hostOwner.UID, hostOwner.GID) @@ -392,35 +383,3 @@ func addHelperSymlink(src, dest string, info os.FileInfo, hostOwner idtools.IDPa logrus.Debugf("Symlink(%s, %s)", linkContents, dest) return nil } - -func getDirsInDockerignore(srcAbsPath string, excludes []dockerIgnore) (map[string]string, error) { - visitedDir := make(map[string]string) - if len(excludes) == 0 { - return visitedDir, nil - } - err := filepath.Walk(srcAbsPath, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if info.IsDir() { - for _, exclude := range excludes { - match, err := filepath.Match(filepath.Clean(exclude.ExcludePath), filepath.Clean(path)) - if err != nil { - return err - } - if !match { - continue - } - if _, exist := visitedDir[exclude.ExcludePath]; exist { - continue - } - visitedDir[exclude.ExcludePath] = path - } - } - return nil - }) - if err != nil { - return visitedDir, err - } - return visitedDir, nil -} diff --git a/vendor/github.com/containers/buildah/buildah.go b/vendor/github.com/containers/buildah/buildah.go index 33b7afccd..56c8f088f 100644 --- a/vendor/github.com/containers/buildah/buildah.go +++ b/vendor/github.com/containers/buildah/buildah.go @@ -26,7 +26,7 @@ const ( Package = "buildah" // Version for the Package. Bump version in contrib/rpm/buildah.spec // too. - Version = "1.9.0-dev" + Version = "1.8.3" // The value we use to identify what type of information, currently a // serialized Builder structure, we are using as per-container state. // This should only be changed when we make incompatible changes to diff --git a/vendor/github.com/containers/buildah/chroot/run.go b/vendor/github.com/containers/buildah/chroot/run.go index c65926c8e..d6e5a61ea 100644 --- a/vendor/github.com/containers/buildah/chroot/run.go +++ b/vendor/github.com/containers/buildah/chroot/run.go @@ -84,9 +84,18 @@ type runUsingChrootExecSubprocOptions struct { // RunUsingChroot runs a chrooted process, using some of the settings from the // passed-in spec, and using the specified bundlePath to hold temporary files, // directories, and mountpoints. -func RunUsingChroot(spec *specs.Spec, bundlePath string, stdin io.Reader, stdout, stderr io.Writer) (err error) { +func RunUsingChroot(spec *specs.Spec, bundlePath, homeDir string, stdin io.Reader, stdout, stderr io.Writer) (err error) { var confwg sync.WaitGroup - + var homeFound bool + for _, env := range spec.Process.Env { + if strings.HasPrefix(env, "HOME=") { + homeFound = true + break + } + } + if !homeFound { + spec.Process.Env = append(spec.Process.Env, fmt.Sprintf("HOME=%s", homeDir)) + } runtime.LockOSThread() defer runtime.UnlockOSThread() diff --git a/vendor/github.com/containers/buildah/config.go b/vendor/github.com/containers/buildah/config.go index 05b0abb23..234f93259 100644 --- a/vendor/github.com/containers/buildah/config.go +++ b/vendor/github.com/containers/buildah/config.go @@ -3,7 +3,6 @@ package buildah import ( "context" "encoding/json" - "os" "runtime" "strings" "time" @@ -269,21 +268,11 @@ func (b *Builder) Env() []string { // built using an image built from this container. func (b *Builder) SetEnv(k string, v string) { reset := func(s *[]string) { - getenv := func(name string) string { - for i := range *s { - val := strings.SplitN((*s)[i], "=", 2) - if len(val) == 2 && val[0] == name { - return val[1] - } - } - return name - } n := []string{} for i := range *s { if !strings.HasPrefix((*s)[i], k+"=") { n = append(n, (*s)[i]) } - v = os.Expand(v, getenv) } n = append(n, k+"="+v) *s = n diff --git a/vendor/github.com/containers/buildah/imagebuildah/build.go b/vendor/github.com/containers/buildah/imagebuildah/build.go index b8b9db0f3..3665251cd 100644 --- a/vendor/github.com/containers/buildah/imagebuildah/build.go +++ b/vendor/github.com/containers/buildah/imagebuildah/build.go @@ -487,6 +487,7 @@ func (s *StageExecutor) Copy(excludes []string, copies ...imagebuilder.Copy) err // Check the file and see if part of it is a symlink. // Convert it to the target if so. To be ultrasafe // do the same for the mountpoint. + hadFinalPathSeparator := len(copy.Dest) > 0 && copy.Dest[len(copy.Dest)-1] == os.PathSeparator secureMountPoint, err := securejoin.SecureJoin("", s.mountPoint) finalPath, err := securejoin.SecureJoin(secureMountPoint, copy.Dest) if err != nil { @@ -496,6 +497,11 @@ func (s *StageExecutor) Copy(excludes []string, copies ...imagebuilder.Copy) err return errors.Wrapf(err, "error resolving copy destination %s", copy.Dest) } copy.Dest = strings.TrimPrefix(finalPath, secureMountPoint) + if len(copy.Dest) == 0 || copy.Dest[len(copy.Dest)-1] != os.PathSeparator { + if hadFinalPathSeparator { + copy.Dest += string(os.PathSeparator) + } + } if copy.Download { logrus.Debugf("ADD %#v, %#v", excludes, copy) @@ -507,29 +513,32 @@ func (s *StageExecutor) Copy(excludes []string, copies ...imagebuilder.Copy) err } sources := []string{} for _, src := range copy.Src { + contextDir := s.executor.contextDir + copyExcludes := excludes if strings.HasPrefix(src, "http://") || strings.HasPrefix(src, "https://") { sources = append(sources, src) } else if len(copy.From) > 0 { if other, ok := s.executor.stages[copy.From]; ok && other.index < s.index { sources = append(sources, filepath.Join(other.mountPoint, src)) + contextDir = other.mountPoint } else if builder, ok := s.executor.containerMap[copy.From]; ok { sources = append(sources, filepath.Join(builder.MountPoint, src)) + contextDir = builder.MountPoint } else { return errors.Errorf("the stage %q has not been built", copy.From) } } else { sources = append(sources, filepath.Join(s.executor.contextDir, src)) + copyExcludes = append(s.executor.excludes, excludes...) + } + options := buildah.AddAndCopyOptions{ + Chown: copy.Chown, + ContextDir: contextDir, + Excludes: copyExcludes, + } + if err := s.builder.Add(copy.Dest, copy.Download, options, sources...); err != nil { + return err } - } - - options := buildah.AddAndCopyOptions{ - Chown: copy.Chown, - ContextDir: s.executor.contextDir, - Excludes: s.executor.excludes, - } - - if err := s.builder.Add(copy.Dest, copy.Download, options, sources...); err != nil { - return err } } return nil @@ -590,7 +599,11 @@ func (s *StageExecutor) Run(run imagebuilder.Run, config docker.Config) error { args := run.Args if run.Shell { - args = append([]string{"/bin/sh", "-c"}, args...) + if len(config.Shell) > 0 && s.builder.Format == buildah.Dockerv2ImageManifest { + args = append(config.Shell, args...) + } else { + args = append([]string{"/bin/sh", "-c"}, args...) + } } if err := s.volumeCacheSave(); err != nil { return err diff --git a/vendor/github.com/containers/buildah/imagebuildah/util.go b/vendor/github.com/containers/buildah/imagebuildah/util.go index f982fcebf..3962d1a9d 100644 --- a/vendor/github.com/containers/buildah/imagebuildah/util.go +++ b/vendor/github.com/containers/buildah/imagebuildah/util.go @@ -17,7 +17,7 @@ import ( ) func cloneToDirectory(url, dir string) error { - if !strings.HasPrefix(url, "git://") { + if !strings.HasPrefix(url, "git://") && !strings.HasSuffix(url, ".git") { url = "git://" + url } logrus.Debugf("cloning %q to %q", url, dir) @@ -72,7 +72,7 @@ func TempDirForURL(dir, prefix, url string) (name string, subdir string, err err if err != nil { return "", "", errors.Wrapf(err, "error creating temporary directory for %q", url) } - if strings.HasPrefix(url, "git://") { + if strings.HasPrefix(url, "git://") || strings.HasSuffix(url, ".git") { err = cloneToDirectory(url, name) if err != nil { if err2 := os.Remove(name); err2 != nil { diff --git a/vendor/github.com/containers/buildah/pkg/chrootuser/user.go b/vendor/github.com/containers/buildah/pkg/chrootuser/user.go index c83dcc230..26a67c35a 100644 --- a/vendor/github.com/containers/buildah/pkg/chrootuser/user.go +++ b/vendor/github.com/containers/buildah/pkg/chrootuser/user.go @@ -18,7 +18,7 @@ var ( // it will use the /etc/passwd and /etc/group files inside of the rootdir // to return this information. // userspec format [user | user:group | uid | uid:gid | user:gid | uid:group ] -func GetUser(rootdir, userspec string) (uint32, uint32, error) { +func GetUser(rootdir, userspec string) (uint32, uint32, string, error) { var gid64 uint64 var gerr error = user.UnknownGroupError("error looking up group") @@ -26,7 +26,7 @@ func GetUser(rootdir, userspec string) (uint32, uint32, error) { userspec = spec[0] groupspec := "" if userspec == "" { - return 0, 0, nil + return 0, 0, "/", nil } if len(spec) > 1 { groupspec = spec[1] @@ -65,15 +65,21 @@ func GetUser(rootdir, userspec string) (uint32, uint32, error) { } } + homedir, err := lookupHomedirInContainer(rootdir, uid64) + if err != nil { + homedir = "/" + } + if uerr == nil && gerr == nil { - return uint32(uid64), uint32(gid64), nil + return uint32(uid64), uint32(gid64), homedir, nil } - err := errors.Wrapf(uerr, "error determining run uid") + err = errors.Wrapf(uerr, "error determining run uid") if uerr == nil { err = errors.Wrapf(gerr, "error determining run gid") } - return 0, 0, err + + return 0, 0, homedir, err } // GetGroup returns the gid by looking it up in the /etc/group file diff --git a/vendor/github.com/containers/buildah/pkg/chrootuser/user_basic.go b/vendor/github.com/containers/buildah/pkg/chrootuser/user_basic.go index 79b0b24b5..6c997c4c9 100644 --- a/vendor/github.com/containers/buildah/pkg/chrootuser/user_basic.go +++ b/vendor/github.com/containers/buildah/pkg/chrootuser/user_basic.go @@ -25,3 +25,7 @@ func lookupAdditionalGroupsForUIDInContainer(rootdir string, userid uint64) (gid func lookupUIDInContainer(rootdir string, uid uint64) (string, uint64, error) { return "", 0, errors.New("UID lookup not supported") } + +func lookupHomedirInContainer(rootdir string, uid uint64) (string, error) { + return "", errors.New("Home directory lookup not supported") +} diff --git a/vendor/github.com/containers/buildah/pkg/chrootuser/user_linux.go b/vendor/github.com/containers/buildah/pkg/chrootuser/user_linux.go index 583eca569..ea20fca80 100644 --- a/vendor/github.com/containers/buildah/pkg/chrootuser/user_linux.go +++ b/vendor/github.com/containers/buildah/pkg/chrootuser/user_linux.go @@ -84,6 +84,7 @@ type lookupPasswdEntry struct { name string uid uint64 gid uint64 + home string } type lookupGroupEntry struct { name string @@ -135,6 +136,7 @@ func parseNextPasswd(rc *bufio.Reader) *lookupPasswdEntry { name: fields[0], uid: uid, gid: gid, + home: fields[5], } } @@ -291,3 +293,29 @@ func lookupUIDInContainer(rootdir string, uid uint64) (string, uint64, error) { return "", 0, user.UnknownUserError(fmt.Sprintf("error looking up uid %q", uid)) } + +func lookupHomedirInContainer(rootdir string, uid uint64) (string, error) { + cmd, f, err := openChrootedFile(rootdir, "/etc/passwd") + if err != nil { + return "", err + } + defer func() { + _ = cmd.Wait() + }() + rc := bufio.NewReader(f) + defer f.Close() + + lookupUser.Lock() + defer lookupUser.Unlock() + + pwd := parseNextPasswd(rc) + for pwd != nil { + if pwd.uid != uid { + pwd = parseNextPasswd(rc) + continue + } + return pwd.home, nil + } + + return "", user.UnknownUserError(fmt.Sprintf("error looking up uid %q for homedir", uid)) +} diff --git a/vendor/github.com/containers/buildah/pkg/secrets/secrets.go b/vendor/github.com/containers/buildah/pkg/secrets/secrets.go index 97b681125..70bd6a4b7 100644 --- a/vendor/github.com/containers/buildah/pkg/secrets/secrets.go +++ b/vendor/github.com/containers/buildah/pkg/secrets/secrets.go @@ -117,7 +117,12 @@ func getMounts(filePath string) []string { } var mounts []string for scanner.Scan() { - mounts = append(mounts, scanner.Text()) + if strings.HasPrefix(strings.TrimSpace(scanner.Text()), "/") { + mounts = append(mounts, scanner.Text()) + } else { + logrus.Debugf("skipping unrecognized mount in %v: %q", + filePath, scanner.Text()) + } } return mounts } @@ -190,58 +195,79 @@ func addSecretsFromMountsFile(filePath, mountLabel, containerWorkingDir, mountPr var mounts []rspec.Mount defaultMountsPaths := getMounts(filePath) for _, path := range defaultMountsPaths { - hostDir, ctrDir, err := getMountsMap(path) + hostDirOrFile, ctrDirOrFile, err := getMountsMap(path) if err != nil { return nil, err } - // skip if the hostDir path doesn't exist - if _, err = os.Stat(hostDir); err != nil { + // skip if the hostDirOrFile path doesn't exist + fileInfo, err := os.Stat(hostDirOrFile) + if err != nil { if os.IsNotExist(err) { - logrus.Warnf("Path %q from %q doesn't exist, skipping", hostDir, filePath) + logrus.Warnf("Path %q from %q doesn't exist, skipping", hostDirOrFile, filePath) continue } - return nil, errors.Wrapf(err, "failed to stat %q", hostDir) + return nil, errors.Wrapf(err, "failed to stat %q", hostDirOrFile) } - ctrDirOnHost := filepath.Join(containerWorkingDir, ctrDir) + ctrDirOrFileOnHost := filepath.Join(containerWorkingDir, ctrDirOrFile) - // In the event of a restart, don't want to copy secrets over again as they already would exist in ctrDirOnHost - _, err = os.Stat(ctrDirOnHost) + // In the event of a restart, don't want to copy secrets over again as they already would exist in ctrDirOrFileOnHost + _, err = os.Stat(ctrDirOrFileOnHost) if os.IsNotExist(err) { - if err = os.MkdirAll(ctrDirOnHost, 0755); err != nil { - return nil, errors.Wrapf(err, "making container directory %q failed", ctrDirOnHost) - } - hostDir, err = resolveSymbolicLink(hostDir) + + hostDirOrFile, err = resolveSymbolicLink(hostDirOrFile) if err != nil { return nil, err } - data, err := getHostSecretData(hostDir) - if err != nil { - return nil, errors.Wrapf(err, "getting host secret data failed") - } - for _, s := range data { - if err := s.saveTo(ctrDirOnHost); err != nil { - return nil, errors.Wrapf(err, "error saving data to container filesystem on host %q", ctrDirOnHost) + switch mode := fileInfo.Mode(); { + case mode.IsDir(): + if err = os.MkdirAll(ctrDirOrFileOnHost, 0755); err != nil { + return nil, errors.Wrapf(err, "making container directory %q failed", ctrDirOrFileOnHost) + } + data, err := getHostSecretData(hostDirOrFile) + if err != nil { + return nil, errors.Wrapf(err, "getting host secret data failed") + } + for _, s := range data { + if err := s.saveTo(ctrDirOrFileOnHost); err != nil { + return nil, errors.Wrapf(err, "error saving data to container filesystem on host %q", ctrDirOrFileOnHost) + } + } + case mode.IsRegular(): + data, err := readFile("", hostDirOrFile) + if err != nil { + return nil, errors.Wrapf(err, "error reading file %q", hostDirOrFile) + + } + for _, s := range data { + if err := os.MkdirAll(filepath.Dir(ctrDirOrFileOnHost), 0700); err != nil { + return nil, err + } + if err := ioutil.WriteFile(ctrDirOrFileOnHost, s.data, 0700); err != nil { + return nil, errors.Wrapf(err, "error saving data to container filesystem on host %q", ctrDirOrFileOnHost) + } } + default: + return nil, errors.Errorf("unsupported file type for: %q", hostDirOrFile) } - err = label.Relabel(ctrDirOnHost, mountLabel, false) + err = label.Relabel(ctrDirOrFileOnHost, mountLabel, false) if err != nil { return nil, errors.Wrap(err, "error applying correct labels") } if uid != 0 || gid != 0 { - if err := rchown(ctrDirOnHost, uid, gid); err != nil { + if err := rchown(ctrDirOrFileOnHost, uid, gid); err != nil { return nil, err } } } else if err != nil { - return nil, errors.Wrapf(err, "error getting status of %q", ctrDirOnHost) + return nil, errors.Wrapf(err, "error getting status of %q", ctrDirOrFileOnHost) } m := rspec.Mount{ - Source: filepath.Join(mountPrefix, ctrDir), - Destination: ctrDir, + Source: filepath.Join(mountPrefix, ctrDirOrFile), + Destination: ctrDirOrFile, Type: "bind", Options: []string{"bind", "rprivate"}, } diff --git a/vendor/github.com/containers/buildah/pkg/unshare/unshare.go b/vendor/github.com/containers/buildah/pkg/unshare/unshare.go index 33232740e..5a68d6b8d 100644 --- a/vendor/github.com/containers/buildah/pkg/unshare/unshare.go +++ b/vendor/github.com/containers/buildah/pkg/unshare/unshare.go @@ -64,6 +64,7 @@ func (c *Cmd) Start() error { if os.Geteuid() != 0 { c.Env = append(c.Env, "_CONTAINERS_USERNS_CONFIGURED=done") c.Env = append(c.Env, fmt.Sprintf("_CONTAINERS_ROOTLESS_UID=%d", os.Geteuid())) + c.Env = append(c.Env, fmt.Sprintf("_CONTAINERS_ROOTLESS_GID=%d", os.Getegid())) } // Create the pipe for reading the child's PID. @@ -183,6 +184,7 @@ func (c *Cmd) Start() error { for _, m := range c.GidMappings { fmt.Fprintf(g, "%d %d %d\n", m.ContainerID, m.HostID, m.Size) } + gidmapSet := false // Set the GID map. if c.UseNewgidmap { cmd := exec.Command("newgidmap", append([]string{pidString}, strings.Fields(strings.Replace(g.String(), "\n", " ", -1))...)...) @@ -190,11 +192,16 @@ func (c *Cmd) Start() error { cmd.Stdout = g cmd.Stderr = g err := cmd.Run() - if err != nil { + if err == nil { + gidmapSet = true + } else { fmt.Fprintf(continueWrite, "error running newgidmap: %v: %s", err, g.String()) - return errors.Wrapf(err, "error running newgidmap: %s", g.String()) + fmt.Fprintf(continueWrite, "falling back to single mapping\n") + g.Reset() + g.Write([]byte(fmt.Sprintf("0 %d 1\n", os.Getegid()))) } - } else { + } + if !gidmapSet { gidmap, err := os.OpenFile(fmt.Sprintf("/proc/%s/gid_map", pidString), os.O_TRUNC|os.O_WRONLY, 0) if err != nil { fmt.Fprintf(continueWrite, "error opening /proc/%s/gid_map: %v", pidString, err) @@ -214,6 +221,7 @@ func (c *Cmd) Start() error { for _, m := range c.UidMappings { fmt.Fprintf(u, "%d %d %d\n", m.ContainerID, m.HostID, m.Size) } + uidmapSet := false // Set the GID map. if c.UseNewuidmap { cmd := exec.Command("newuidmap", append([]string{pidString}, strings.Fields(strings.Replace(u.String(), "\n", " ", -1))...)...) @@ -221,11 +229,16 @@ func (c *Cmd) Start() error { cmd.Stdout = u cmd.Stderr = u err := cmd.Run() - if err != nil { + if err == nil { + uidmapSet = true + } else { fmt.Fprintf(continueWrite, "error running newuidmap: %v: %s", err, u.String()) - return errors.Wrapf(err, "error running newuidmap: %s", u.String()) + fmt.Fprintf(continueWrite, "falling back to single mapping\n") + u.Reset() + u.Write([]byte(fmt.Sprintf("0 %d 1\n", os.Geteuid()))) } - } else { + } + if !uidmapSet { uidmap, err := os.OpenFile(fmt.Sprintf("/proc/%s/uid_map", pidString), os.O_TRUNC|os.O_WRONLY, 0) if err != nil { fmt.Fprintf(continueWrite, "error opening /proc/%s/uid_map: %v", pidString, err) @@ -354,7 +367,9 @@ func MaybeReexecUsingUserNamespace(evenForRoot bool) { // range in /etc/subuid and /etc/subgid file is a starting host // ID and a range size. uidmap, gidmap, err = GetSubIDMappings(me.Username, me.Username) - bailOnError(err, "error reading allowed ID mappings") + if err != nil { + logrus.Warnf("error reading allowed ID mappings: %v", err) + } if len(uidmap) == 0 { logrus.Warnf("Found no UID ranges set aside for user %q in /etc/subuid.", me.Username) } diff --git a/vendor/github.com/containers/buildah/run_linux.go b/vendor/github.com/containers/buildah/run_linux.go index 81ce2b944..16c0550aa 100644 --- a/vendor/github.com/containers/buildah/run_linux.go +++ b/vendor/github.com/containers/buildah/run_linux.go @@ -131,7 +131,8 @@ func (b *Builder) Run(command []string, options RunOptions) error { return err } - if err := b.configureUIDGID(g, mountPoint, options); err != nil { + homeDir, err := b.configureUIDGID(g, mountPoint, options) + if err != nil { return err } @@ -210,7 +211,7 @@ func (b *Builder) Run(command []string, options RunOptions) error { } err = b.runUsingRuntimeSubproc(isolation, options, configureNetwork, configureNetworks, moreCreateArgs, spec, mountPoint, path, Package+"-"+filepath.Base(path)) case IsolationChroot: - err = chroot.RunUsingChroot(spec, path, options.Stdin, options.Stdout, options.Stderr) + err = chroot.RunUsingChroot(spec, path, homeDir, options.Stdin, options.Stdout, options.Stderr) case IsolationOCIRootless: moreCreateArgs := []string{"--no-new-keyring"} if options.NoPivot { @@ -1454,7 +1455,18 @@ func setupNamespaces(g *generate.Generator, namespaceOptions NamespaceOptions, i } if configureNetwork { for name, val := range util.DefaultNetworkSysctl { - g.AddLinuxSysctl(name, val) + // Check that the sysctl we are adding is actually supported + // by the kernel + p := filepath.Join("/proc/sys", strings.Replace(name, ".", "/", -1)) + _, err := os.Stat(p) + if err != nil && !os.IsNotExist(err) { + return false, nil, false, errors.Wrapf(err, "cannot stat %s", p) + } + if err == nil { + g.AddLinuxSysctl(name, val) + } else { + logrus.Warnf("ignoring sysctl %s since %s doesn't exist", name, p) + } } } return configureNetwork, configureNetworks, configureUTS, nil @@ -1775,14 +1787,14 @@ func getDNSIP(dnsServers []string) (dns []net.IP, err error) { return dns, nil } -func (b *Builder) configureUIDGID(g *generate.Generator, mountPoint string, options RunOptions) error { +func (b *Builder) configureUIDGID(g *generate.Generator, mountPoint string, options RunOptions) (string, error) { // Set the user UID/GID/supplemental group list/capabilities lists. - user, err := b.user(mountPoint, options.User) + user, homeDir, err := b.user(mountPoint, options.User) if err != nil { - return err + return "", err } if err := setupCapabilities(g, b.AddCapabilities, b.DropCapabilities, options.AddCapabilities, options.DropCapabilities); err != nil { - return err + return "", err } g.SetProcessUID(user.UID) g.SetProcessGID(user.GID) @@ -1797,7 +1809,7 @@ func (b *Builder) configureUIDGID(g *generate.Generator, mountPoint string, opti g.Config.Process.Capabilities.Bounding = bounding } - return nil + return homeDir, nil } func (b *Builder) configureEnvironment(g *generate.Generator, options RunOptions) { diff --git a/vendor/github.com/containers/storage/containers.go b/vendor/github.com/containers/storage/containers.go index bbac78b60..e69552361 100644 --- a/vendor/github.com/containers/storage/containers.go +++ b/vendor/github.com/containers/storage/containers.go @@ -572,6 +572,10 @@ func (r *containerStore) Lock() { r.lockfile.Lock() } +func (r *containerStore) RecursiveLock() { + r.lockfile.RecursiveLock() +} + func (r *containerStore) RLock() { r.lockfile.RLock() } diff --git a/vendor/github.com/containers/storage/drivers/aufs/aufs.go b/vendor/github.com/containers/storage/drivers/aufs/aufs.go index e821bc0c5..353d1707a 100644 --- a/vendor/github.com/containers/storage/drivers/aufs/aufs.go +++ b/vendor/github.com/containers/storage/drivers/aufs/aufs.go @@ -255,6 +255,9 @@ func (a *Driver) AdditionalImageStores() []string { // CreateFromTemplate creates a layer with the same contents and parent as another layer. func (a *Driver) CreateFromTemplate(id, template string, templateIDMappings *idtools.IDMappings, parent string, parentIDMappings *idtools.IDMappings, opts *graphdriver.CreateOpts, readWrite bool) error { + if opts == nil { + opts = &graphdriver.CreateOpts{} + } return graphdriver.NaiveCreateFromTemplate(a, id, template, templateIDMappings, parent, parentIDMappings, opts, readWrite) } diff --git a/vendor/github.com/containers/storage/drivers/chown.go b/vendor/github.com/containers/storage/drivers/chown.go index 4d4011ee0..f2f1ec386 100644 --- a/vendor/github.com/containers/storage/drivers/chown.go +++ b/vendor/github.com/containers/storage/drivers/chown.go @@ -55,6 +55,9 @@ func chownByMapsMain() { if err != nil { return fmt.Errorf("error walking to %q: %v", path, err) } + if path == "." { + return nil + } return platformLChown(path, info, toHost, toContainer) } if err := filepath.Walk(".", chown); err != nil { diff --git a/vendor/github.com/containers/storage/images.go b/vendor/github.com/containers/storage/images.go index 38b5a3ef3..6f487504a 100644 --- a/vendor/github.com/containers/storage/images.go +++ b/vendor/github.com/containers/storage/images.go @@ -82,6 +82,9 @@ type Image struct { // is set before using it. Created time.Time `json:"created,omitempty"` + // ReadOnly is true if this image resides in a read-only layer store. + ReadOnly bool `json:"-"` + Flags map[string]interface{} `json:"flags,omitempty"` } @@ -159,6 +162,7 @@ func copyImage(i *Image) *Image { BigDataSizes: copyStringInt64Map(i.BigDataSizes), BigDataDigests: copyStringDigestMap(i.BigDataDigests), Created: i.Created, + ReadOnly: i.ReadOnly, Flags: copyStringInterfaceMap(i.Flags), } } @@ -269,6 +273,7 @@ func (r *imageStore) Load() error { list := digests[digest] digests[digest] = append(list, image) } + image.ReadOnly = !r.IsReadWrite() } } if shouldSave && (!r.IsReadWrite() || !r.Locked()) { @@ -739,6 +744,10 @@ func (r *imageStore) Lock() { r.lockfile.Lock() } +func (r *imageStore) RecursiveLock() { + r.lockfile.RecursiveLock() +} + func (r *imageStore) RLock() { r.lockfile.RLock() } diff --git a/vendor/github.com/containers/storage/layers.go b/vendor/github.com/containers/storage/layers.go index a35dd476b..fb79238cd 100644 --- a/vendor/github.com/containers/storage/layers.go +++ b/vendor/github.com/containers/storage/layers.go @@ -103,6 +103,9 @@ type Layer struct { // for use inside of a user namespace where UID mapping is being used. UIDMap []idtools.IDMap `json:"uidmap,omitempty"` GIDMap []idtools.IDMap `json:"gidmap,omitempty"` + + // ReadOnly is true if this layer resides in a read-only layer store. + ReadOnly bool `json:"-"` } type layerMountPoint struct { @@ -259,6 +262,7 @@ func copyLayer(l *Layer) *Layer { UncompressedDigest: l.UncompressedDigest, UncompressedSize: l.UncompressedSize, CompressionType: l.CompressionType, + ReadOnly: l.ReadOnly, Flags: copyStringInterfaceMap(l.Flags), UIDMap: copyIDMap(l.UIDMap), GIDMap: copyIDMap(l.GIDMap), @@ -318,6 +322,7 @@ func (r *layerStore) Load() error { if layer.MountLabel != "" { label.ReserveLabel(layer.MountLabel) } + layer.ReadOnly = !r.IsReadWrite() } err = nil } @@ -1304,6 +1309,10 @@ func (r *layerStore) Lock() { r.lockfile.Lock() } +func (r *layerStore) RecursiveLock() { + r.lockfile.RecursiveLock() +} + func (r *layerStore) RLock() { r.lockfile.RLock() } diff --git a/vendor/github.com/containers/storage/layers_ffjson.go b/vendor/github.com/containers/storage/layers_ffjson.go index 09b5d0f33..125b5d8c9 100644 --- a/vendor/github.com/containers/storage/layers_ffjson.go +++ b/vendor/github.com/containers/storage/layers_ffjson.go @@ -1,5 +1,5 @@ // Code generated by ffjson <https://github.com/pquerna/ffjson>. DO NOT EDIT. -// source: ./layers.go +// source: layers.go package storage diff --git a/vendor/github.com/containers/storage/lockfile.go b/vendor/github.com/containers/storage/lockfile.go index ed8753337..c4f1b5549 100644 --- a/vendor/github.com/containers/storage/lockfile.go +++ b/vendor/github.com/containers/storage/lockfile.go @@ -15,6 +15,10 @@ type Locker interface { // Acquire a writer lock. Lock() + // Acquire a writer lock recursively, allowing for recursive acquisitions + // within the same process space. + RecursiveLock() + // Unlock the lock. Unlock() diff --git a/vendor/github.com/containers/storage/lockfile_linux.go b/vendor/github.com/containers/storage/lockfile_linux.go deleted file mode 100644 index 903387c66..000000000 --- a/vendor/github.com/containers/storage/lockfile_linux.go +++ /dev/null @@ -1,20 +0,0 @@ -// +build linux solaris - -package storage - -import ( - "time" - - "golang.org/x/sys/unix" -) - -// TouchedSince indicates if the lock file has been touched since the specified time -func (l *lockfile) TouchedSince(when time.Time) bool { - st := unix.Stat_t{} - err := unix.Fstat(int(l.fd), &st) - if err != nil { - return true - } - touched := time.Unix(st.Mtim.Unix()) - return when.Before(touched) -} diff --git a/vendor/github.com/containers/storage/lockfile_otherunix.go b/vendor/github.com/containers/storage/lockfile_otherunix.go deleted file mode 100644 index 041d54c05..000000000 --- a/vendor/github.com/containers/storage/lockfile_otherunix.go +++ /dev/null @@ -1,19 +0,0 @@ -// +build darwin freebsd - -package storage - -import ( - "time" - - "golang.org/x/sys/unix" -) - -func (l *lockfile) TouchedSince(when time.Time) bool { - st := unix.Stat_t{} - err := unix.Fstat(int(l.fd), &st) - if err != nil { - return true - } - touched := time.Unix(st.Mtimespec.Unix()) - return when.Before(touched) -} diff --git a/vendor/github.com/containers/storage/lockfile_unix.go b/vendor/github.com/containers/storage/lockfile_unix.go index 8e0f22cb5..00215e928 100644 --- a/vendor/github.com/containers/storage/lockfile_unix.go +++ b/vendor/github.com/containers/storage/lockfile_unix.go @@ -9,6 +9,7 @@ import ( "time" "github.com/containers/storage/pkg/stringid" + "github.com/containers/storage/pkg/system" "github.com/pkg/errors" "golang.org/x/sys/unix" ) @@ -25,6 +26,7 @@ type lockfile struct { locktype int16 locked bool ro bool + recursive bool } // openLock opens the file at path and returns the corresponding file @@ -75,7 +77,7 @@ func createLockerForPath(path string, ro bool) (Locker, error) { // lock locks the lockfile via FCTNL(2) based on the specified type and // command. -func (l *lockfile) lock(l_type int16) { +func (l *lockfile) lock(l_type int16, recursive bool) { lk := unix.Flock_t{ Type: l_type, Whence: int16(os.SEEK_SET), @@ -86,7 +88,13 @@ func (l *lockfile) lock(l_type int16) { case unix.F_RDLCK: l.rwMutex.RLock() case unix.F_WRLCK: - l.rwMutex.Lock() + if recursive { + // NOTE: that's okay as recursive is only set in RecursiveLock(), so + // there's no need to protect against hypothetical RDLCK cases. + l.rwMutex.RLock() + } else { + l.rwMutex.Lock() + } default: panic(fmt.Sprintf("attempted to acquire a file lock of unrecognized type %d", l_type)) } @@ -110,6 +118,7 @@ func (l *lockfile) lock(l_type int16) { } l.locktype = l_type l.locked = true + l.recursive = recursive l.counter++ } @@ -119,13 +128,24 @@ func (l *lockfile) Lock() { if l.ro { l.RLock() } else { - l.lock(unix.F_WRLCK) + l.lock(unix.F_WRLCK, false) + } +} + +// RecursiveLock locks the lockfile as a writer but allows for recursive +// acquisitions within the same process space. Note that RLock() will be called +// if it's a lockTypReader lock. +func (l *lockfile) RecursiveLock() { + if l.ro { + l.RLock() + } else { + l.lock(unix.F_WRLCK, true) } } // LockRead locks the lockfile as a reader. func (l *lockfile) RLock() { - l.lock(unix.F_RDLCK) + l.lock(unix.F_RDLCK, false) } // Unlock unlocks the lockfile. @@ -161,7 +181,7 @@ func (l *lockfile) Unlock() { // Close the file descriptor on the last unlock. unix.Close(int(l.fd)) } - if l.locktype == unix.F_RDLCK { + if l.locktype == unix.F_RDLCK || l.recursive { l.rwMutex.RUnlock() } else { l.rwMutex.Unlock() @@ -232,3 +252,14 @@ func (l *lockfile) Modified() (bool, error) { func (l *lockfile) IsReadWrite() bool { return !l.ro } + +// TouchedSince indicates if the lock file has been touched since the specified time +func (l *lockfile) TouchedSince(when time.Time) bool { + st, err := system.Fstat(int(l.fd)) + if err != nil { + return true + } + mtim := st.Mtim() + touched := time.Unix(mtim.Unix()) + return when.Before(touched) +} diff --git a/vendor/github.com/containers/storage/lockfile_windows.go b/vendor/github.com/containers/storage/lockfile_windows.go index c02069495..caf7c184a 100644 --- a/vendor/github.com/containers/storage/lockfile_windows.go +++ b/vendor/github.com/containers/storage/lockfile_windows.go @@ -36,6 +36,12 @@ func (l *lockfile) Lock() { l.locked = true } +func (l *lockfile) RecursiveLock() { + // We don't support Windows but a recursive writer-lock in one process-space + // is really a writer lock, so just panic. + panic("not supported") +} + func (l *lockfile) RLock() { l.mu.Lock() l.locked = true diff --git a/vendor/github.com/containers/storage/pkg/chrootarchive/archive.go b/vendor/github.com/containers/storage/pkg/chrootarchive/archive.go index a36ff1cb1..33ba6a128 100644 --- a/vendor/github.com/containers/storage/pkg/chrootarchive/archive.go +++ b/vendor/github.com/containers/storage/pkg/chrootarchive/archive.go @@ -1,7 +1,7 @@ package chrootarchive import ( - "archive/tar" + stdtar "archive/tar" "fmt" "io" "io/ioutil" @@ -34,18 +34,34 @@ func NewArchiverWithChown(tarIDMappings *idtools.IDMappings, chownOpts *idtools. // The archive may be compressed with one of the following algorithms: // identity (uncompressed), gzip, bzip2, xz. func Untar(tarArchive io.Reader, dest string, options *archive.TarOptions) error { - return untarHandler(tarArchive, dest, options, true) + return untarHandler(tarArchive, dest, options, true, dest) +} + +// UntarWithRoot is the same as `Untar`, but allows you to pass in a root directory +// The root directory is the directory that will be chrooted to. +// `dest` must be a path within `root`, if it is not an error will be returned. +// +// `root` should set to a directory which is not controlled by any potentially +// malicious process. +// +// This should be used to prevent a potential attacker from manipulating `dest` +// such that it would provide access to files outside of `dest` through things +// like symlinks. Normally `ResolveSymlinksInScope` would handle this, however +// sanitizing symlinks in this manner is inherrently racey: +// ref: CVE-2018-15664 +func UntarWithRoot(tarArchive io.Reader, dest string, options *archive.TarOptions, root string) error { + return untarHandler(tarArchive, dest, options, true, root) } // UntarUncompressed reads a stream of bytes from `archive`, parses it as a tar archive, // and unpacks it into the directory at `dest`. // The archive must be an uncompressed stream. func UntarUncompressed(tarArchive io.Reader, dest string, options *archive.TarOptions) error { - return untarHandler(tarArchive, dest, options, false) + return untarHandler(tarArchive, dest, options, false, dest) } // Handler for teasing out the automatic decompression -func untarHandler(tarArchive io.Reader, dest string, options *archive.TarOptions, decompress bool) error { +func untarHandler(tarArchive io.Reader, dest string, options *archive.TarOptions, decompress bool, root string) error { if tarArchive == nil { return fmt.Errorf("Empty archive") } @@ -77,7 +93,15 @@ func untarHandler(tarArchive io.Reader, dest string, options *archive.TarOptions r = decompressedArchive } - return invokeUnpack(r, dest, options) + return invokeUnpack(r, dest, options, root) +} + +// Tar tars the requested path while chrooted to the specified root. +func Tar(srcPath string, options *archive.TarOptions, root string) (io.ReadCloser, error) { + if options == nil { + options = &archive.TarOptions{} + } + return invokePack(srcPath, options, root) } // CopyFileWithTarAndChown returns a function which copies a single file from outside @@ -99,7 +123,7 @@ func CopyFileWithTarAndChown(chownOpts *idtools.IDPair, hasher io.Writer, uidmap var hashWorker sync.WaitGroup hashWorker.Add(1) go func() { - t := tar.NewReader(contentReader) + t := stdtar.NewReader(contentReader) _, err := t.Next() if err != nil { hashError = err diff --git a/vendor/github.com/containers/storage/pkg/chrootarchive/archive_unix.go b/vendor/github.com/containers/storage/pkg/chrootarchive/archive_unix.go index e04ed787c..ca9fb10d7 100644 --- a/vendor/github.com/containers/storage/pkg/chrootarchive/archive_unix.go +++ b/vendor/github.com/containers/storage/pkg/chrootarchive/archive_unix.go @@ -10,10 +10,13 @@ import ( "io" "io/ioutil" "os" + "path/filepath" "runtime" + "strings" "github.com/containers/storage/pkg/archive" "github.com/containers/storage/pkg/reexec" + "github.com/pkg/errors" ) // untar is the entry-point for storage-untar on re-exec. This is not used on @@ -23,18 +26,28 @@ func untar() { runtime.LockOSThread() flag.Parse() - var options *archive.TarOptions + var options archive.TarOptions //read the options from the pipe "ExtraFiles" if err := json.NewDecoder(os.NewFile(3, "options")).Decode(&options); err != nil { fatal(err) } - if err := chroot(flag.Arg(0)); err != nil { + dst := flag.Arg(0) + var root string + if len(flag.Args()) > 1 { + root = flag.Arg(1) + } + + if root == "" { + root = dst + } + + if err := chroot(root); err != nil { fatal(err) } - if err := archive.Unpack(os.Stdin, "/", options); err != nil { + if err := archive.Unpack(os.Stdin, dst, &options); err != nil { fatal(err) } // fully consume stdin in case it is zero padded @@ -45,7 +58,10 @@ func untar() { os.Exit(0) } -func invokeUnpack(decompressedArchive io.Reader, dest string, options *archive.TarOptions) error { +func invokeUnpack(decompressedArchive io.Reader, dest string, options *archive.TarOptions, root string) error { + if root == "" { + return errors.New("must specify a root to chroot to") + } // We can't pass a potentially large exclude list directly via cmd line // because we easily overrun the kernel's max argument/environment size @@ -57,7 +73,21 @@ func invokeUnpack(decompressedArchive io.Reader, dest string, options *archive.T return fmt.Errorf("Untar pipe failure: %v", err) } - cmd := reexec.Command("storage-untar", dest) + if root != "" { + relDest, err := filepath.Rel(root, dest) + if err != nil { + return err + } + if relDest == "." { + relDest = "/" + } + if relDest[0] != '/' { + relDest = "/" + relDest + } + dest = relDest + } + + cmd := reexec.Command("storage-untar", dest, root) cmd.Stdin = decompressedArchive cmd.ExtraFiles = append(cmd.ExtraFiles, r) @@ -68,6 +98,7 @@ func invokeUnpack(decompressedArchive io.Reader, dest string, options *archive.T if err := cmd.Start(); err != nil { return fmt.Errorf("Untar error on re-exec cmd: %v", err) } + //write the options to the pipe for the untar exec to read if err := json.NewEncoder(w).Encode(options); err != nil { return fmt.Errorf("Untar json encode to pipe failed: %v", err) @@ -84,3 +115,92 @@ func invokeUnpack(decompressedArchive io.Reader, dest string, options *archive.T } return nil } + +func tar() { + runtime.LockOSThread() + flag.Parse() + + src := flag.Arg(0) + var root string + if len(flag.Args()) > 1 { + root = flag.Arg(1) + } + + if root == "" { + root = src + } + + if err := realChroot(root); err != nil { + fatal(err) + } + + var options archive.TarOptions + if err := json.NewDecoder(os.Stdin).Decode(&options); err != nil { + fatal(err) + } + + rdr, err := archive.TarWithOptions(src, &options) + if err != nil { + fatal(err) + } + defer rdr.Close() + + if _, err := io.Copy(os.Stdout, rdr); err != nil { + fatal(err) + } + + os.Exit(0) +} + +func invokePack(srcPath string, options *archive.TarOptions, root string) (io.ReadCloser, error) { + if root == "" { + return nil, errors.New("root path must not be empty") + } + + relSrc, err := filepath.Rel(root, srcPath) + if err != nil { + return nil, err + } + if relSrc == "." { + relSrc = "/" + } + if relSrc[0] != '/' { + relSrc = "/" + relSrc + } + + // make sure we didn't trim a trailing slash with the call to `Rel` + if strings.HasSuffix(srcPath, "/") && !strings.HasSuffix(relSrc, "/") { + relSrc += "/" + } + + cmd := reexec.Command("storage-tar", relSrc, root) + + errBuff := bytes.NewBuffer(nil) + cmd.Stderr = errBuff + + tarR, tarW := io.Pipe() + cmd.Stdout = tarW + + stdin, err := cmd.StdinPipe() + if err != nil { + return nil, errors.Wrap(err, "error getting options pipe for tar process") + } + + if err := cmd.Start(); err != nil { + return nil, errors.Wrap(err, "tar error on re-exec cmd") + } + + go func() { + err := cmd.Wait() + err = errors.Wrapf(err, "error processing tar file: %s", errBuff) + tarW.CloseWithError(err) + }() + + if err := json.NewEncoder(stdin).Encode(options); err != nil { + stdin.Close() + return nil, errors.Wrap(err, "tar json encode to pipe failed") + } + stdin.Close() + + return tarR, nil +} diff --git a/vendor/github.com/containers/storage/pkg/chrootarchive/archive_windows.go b/vendor/github.com/containers/storage/pkg/chrootarchive/archive_windows.go index 93fde4220..8a5c680b1 100644 --- a/vendor/github.com/containers/storage/pkg/chrootarchive/archive_windows.go +++ b/vendor/github.com/containers/storage/pkg/chrootarchive/archive_windows.go @@ -14,9 +14,16 @@ func chroot(path string) error { func invokeUnpack(decompressedArchive io.ReadCloser, dest string, - options *archive.TarOptions) error { + options *archive.TarOptions, root string) error { // Windows is different to Linux here because Windows does not support // chroot. Hence there is no point sandboxing a chrooted process to // do the unpack. We call inline instead within the daemon process. return archive.Unpack(decompressedArchive, longpath.AddPrefix(dest), options) } + +func invokePack(srcPath string, options *archive.TarOptions, root string) (io.ReadCloser, error) { + // Windows is different to Linux here because Windows does not support + // chroot. Hence there is no point sandboxing a chrooted process to + // do the pack. We call inline instead within the daemon process. + return archive.TarWithOptions(srcPath, options) +} diff --git a/vendor/github.com/containers/storage/pkg/chrootarchive/chroot_unix.go b/vendor/github.com/containers/storage/pkg/chrootarchive/chroot_unix.go index f9b5dece8..83278ee50 100644 --- a/vendor/github.com/containers/storage/pkg/chrootarchive/chroot_unix.go +++ b/vendor/github.com/containers/storage/pkg/chrootarchive/chroot_unix.go @@ -4,9 +4,13 @@ package chrootarchive import "golang.org/x/sys/unix" -func chroot(path string) error { +func realChroot(path string) error { if err := unix.Chroot(path); err != nil { return err } return unix.Chdir("/") } + +func chroot(path string) error { + return realChroot(path) +} diff --git a/vendor/github.com/containers/storage/pkg/chrootarchive/init_unix.go b/vendor/github.com/containers/storage/pkg/chrootarchive/init_unix.go index 21cd87992..ea08135e4 100644 --- a/vendor/github.com/containers/storage/pkg/chrootarchive/init_unix.go +++ b/vendor/github.com/containers/storage/pkg/chrootarchive/init_unix.go @@ -14,6 +14,7 @@ import ( func init() { reexec.Register("storage-applyLayer", applyLayer) reexec.Register("storage-untar", untar) + reexec.Register("storage-tar", tar) } func fatal(err error) { diff --git a/vendor/github.com/containers/storage/pkg/system/stat_unix.go b/vendor/github.com/containers/storage/pkg/system/stat_unix.go index 91c7d121c..f9a1b4877 100644 --- a/vendor/github.com/containers/storage/pkg/system/stat_unix.go +++ b/vendor/github.com/containers/storage/pkg/system/stat_unix.go @@ -58,3 +58,15 @@ func Stat(path string) (*StatT, error) { } return fromStatT(s) } + +// Fstat takes an open file descriptor and returns +// a system.StatT type pertaining to that file. +// +// Throws an error if the file descriptor is invalid +func Fstat(fd int) (*StatT, error) { + s := &syscall.Stat_t{} + if err := syscall.Fstat(fd, s); err != nil { + return nil, err + } + return fromStatT(s) +} diff --git a/version/version.go b/version/version.go index a917931b7..c3917c016 100644 --- a/version/version.go +++ b/version/version.go @@ -4,7 +4,7 @@ package version // NOTE: remember to bump the version at the top // of the top-level README.md file when this is // bumped. -const Version = "1.3.2-dev" +const Version = "1.4.1-dev" // RemoteAPIVersion is the version for the remote // client API. It is used to determine compatibility |