diff options
48 files changed, 1099 insertions, 353 deletions
diff --git a/.cirrus.yml b/.cirrus.yml index 199c2a533..3192d15ae 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -25,6 +25,16 @@ env: CIRRUS_CLONE_DEPTH: 200 #### + #### Cache-image names to test with + ### + FEDORA_CACHE_IMAGE_NAME: "fedora-29-libpod-9afa57a9" + PRIOR_FEDORA_CACHE_IMAGE_NAME: "fedora-28-libpod-9afa57a9" + UBUNTU_CACHE_IMAGE_NAME: "ubuntu-18-libpod-9afa57a9" + RHEL_CACHE_IMAGE_NAME: "rhel-8-notready" + PRIOR_RHEL_CACHE_IMAGE_NAME: "rhel-7-notready" + CENTOS_CACHE_IMAGE_NAME: "centos-7-notready" + + #### #### Variables for composing new cache-images (used in PR testing) from #### base-images (pre-existing in GCE) #### @@ -38,6 +48,8 @@ env: PACKER_BUILDS: "ubuntu-18,fedora-29,fedora-28" # TODO: fah-29,rhel-7,centos-7 # Version of packer to use PACKER_VER: "1.3.1" + # Special image w/ nested-libvirt + tools for creating new cache and base images + IMAGE_BUILDER_CACHE_IMAGE_NAME: "image-builder-image-1541772081" # Google-maintained base-image names UBUNTU_BASE_IMAGE: "ubuntu-1804-bionic-v20181203a" CENTOS_BASE_IMAGE: "centos-7-v20181113" @@ -113,7 +125,7 @@ build_each_commit_task: memory: "4Gb" disk: 40 matrix: - image_name: "fedora-29-libpod-9afa57a9" + image_name: "${FEDORA_CACHE_IMAGE_NAME}" timeout_in: 30m @@ -139,14 +151,14 @@ testing_task: # 'matrix' combinations. matrix: # Images are generated separately, from build_images_task (below) - image_name: fedora-29-libpod-9afa57a9 - image_name: fedora-28-libpod-9afa57a9 - image_name: ubuntu-18-libpod-9afa57a9 + image_name: "${FEDORA_CACHE_IMAGE_NAME}" + image_name: "${PRIOR_FEDORA_CACHE_IMAGE_NAME}" + image_name: "${UBUNTU_CACHE_IMAGE_NAME}" # TODO: tests fail - # image_name: "rhel-7-something-something" - # image_name: "centos-7-something-something" - # image_name: "fah-29-libpod-5070733157859328" + # image_name: "${RHEL_CACHE_IMAGE_NAME} + # image_name: "${PRIOR_RHEL_CACHE_IMAGE_NAME} + # image_name: "${CENTOS_CACHE_IMAGE_NAME}" timeout_in: 120m @@ -176,12 +188,13 @@ optional_testing_task: gce_instance: image_project: "libpod-218412" matrix: - image_name: fedora-29-libpod-9afa57a9 - image_name: fedora-28-libpod-9afa57a9 - image_name: ubuntu-18-libpod-9afa57a9 + image_name: "${FEDORA_CACHE_IMAGE_NAME}" + image_name: "${PRIOR_FEDORA_CACHE_IMAGE_NAME}" + image_name: "${UBUNTU_CACHE_IMAGE_NAME}" # TODO: Make these work (also build_images_task below) - #image_name: "rhel-server-ec2-7-5-165-1-libpod-fce09afe" - #image_name: "centos-7-v20180911-libpod-fce09afe" + # image_name: "${RHEL_CACHE_IMAGE_NAME} + # image_name: "${PRIOR_RHEL_CACHE_IMAGE_NAME} + # image_name: "${CENTOS_CACHE_IMAGE_NAME}" timeout_in: 60m @@ -214,7 +227,7 @@ cache_images_task: cpu: 4 memory: "4Gb" disk: 200 - image_name: "image-builder-image-1541772081" # Simply CentOS 7 + packer dependencies + image_name: "${IMAGE_BUILDER_CACHE_IMAGE_NAME}" # Additional permissions for building GCE images, within a GCE VM scopes: - compute @@ -9,14 +9,22 @@ in the [API.md](https://github.com/containers/libpod/blob/master/API.md) file in [func Commit(name: string, image_name: string, changes: []string, author: string, message: string, pause: bool, manifestType: string) string](#Commit) +[func ContainerArtifacts(name: string, artifactName: string) string](#ContainerArtifacts) + [func ContainerCheckpoint(name: string, keep: bool, leaveRunning: bool, tcpEstablished: bool) string](#ContainerCheckpoint) +[func ContainerConfig(name: string) string](#ContainerConfig) + [func ContainerExists(name: string) int](#ContainerExists) +[func ContainerInspectData(name: string) string](#ContainerInspectData) + [func ContainerRestore(name: string, keep: bool, tcpEstablished: bool) string](#ContainerRestore) [func ContainerRunlabel(runlabel: Runlabel) ](#ContainerRunlabel) +[func ContainerStateData(name: string) string](#ContainerStateData) + [func CreateContainer(create: Create) string](#CreateContainer) [func CreateImage() NotImplemented](#CreateImage) @@ -239,12 +247,24 @@ attributes: _CMD, ENTRYPOINT, ENV, EXPOSE, LABEL, ONBUILD, STOPSIGNAL, USER, VOL container while it is being committed, pass a _true_ bool for the pause argument. If the container cannot be found by the ID or name provided, a (ContainerNotFound)[#ContainerNotFound] error will be returned; otherwise, the resulting image's ID will be returned as a string. +### <a name="ContainerArtifacts"></a>func ContainerArtifacts +<div style="background-color: #E8E8E8; padding: 15px; margin: 10px; border-radius: 10px;"> + +method ContainerArtifacts(name: [string](https://godoc.org/builtin#string), artifactName: [string](https://godoc.org/builtin#string)) [string](https://godoc.org/builtin#string)</div> +ContainerArtifacts returns a container's artifacts in string form. This call is for +development of Podman only and generally should not be used. ### <a name="ContainerCheckpoint"></a>func ContainerCheckpoint <div style="background-color: #E8E8E8; padding: 15px; margin: 10px; border-radius: 10px;"> method ContainerCheckpoint(name: [string](https://godoc.org/builtin#string), keep: [bool](https://godoc.org/builtin#bool), leaveRunning: [bool](https://godoc.org/builtin#bool), tcpEstablished: [bool](https://godoc.org/builtin#bool)) [string](https://godoc.org/builtin#string)</div> ContainerCheckPoint performs a checkpopint on a container by its name or full/partial container ID. On successful checkpoint, the id of the checkpointed container is returned. +### <a name="ContainerConfig"></a>func ContainerConfig +<div style="background-color: #E8E8E8; padding: 15px; margin: 10px; border-radius: 10px;"> + +method ContainerConfig(name: [string](https://godoc.org/builtin#string)) [string](https://godoc.org/builtin#string)</div> +ContainerConfig returns a container's config in string form. This call is for +development of Podman only and generally should not be used. ### <a name="ContainerExists"></a>func ContainerExists <div style="background-color: #E8E8E8; padding: 15px; margin: 10px; border-radius: 10px;"> @@ -258,6 +278,12 @@ $ varlink call -m unix:/run/podman/io.podman/io.podman.ContainerExists '{"name": "exists": 0 } ~~~ +### <a name="ContainerInspectData"></a>func ContainerInspectData +<div style="background-color: #E8E8E8; padding: 15px; margin: 10px; border-radius: 10px;"> + +method ContainerInspectData(name: [string](https://godoc.org/builtin#string)) [string](https://godoc.org/builtin#string)</div> +ContainerInspectData returns a container's inspect data in string form. This call is for +development of Podman only and generally should not be used. ### <a name="ContainerRestore"></a>func ContainerRestore <div style="background-color: #E8E8E8; padding: 15px; margin: 10px; border-radius: 10px;"> @@ -270,6 +296,12 @@ of the container's ID. method ContainerRunlabel(runlabel: [Runlabel](#Runlabel)) </div> ContainerRunlabel runs executes a command as described by a given container image label. +### <a name="ContainerStateData"></a>func ContainerStateData +<div style="background-color: #E8E8E8; padding: 15px; margin: 10px; border-radius: 10px;"> + +method ContainerStateData(name: [string](https://godoc.org/builtin#string)) [string](https://godoc.org/builtin#string)</div> +ContainerStateData returns a container's state config in string form. This call is for +development of Podman only and generally should not be used. ### <a name="CreateContainer"></a>func CreateContainer <div style="background-color: #E8E8E8; padding: 15px; margin: 10px; border-radius: 10px;"> @@ -1373,6 +1405,8 @@ virtualSize [int](https://godoc.org/builtin#int) containers [int](https://godoc.org/builtin#int) labels [map[string]](#map[string]) + +isParent [bool](https://godoc.org/builtin#bool) ### <a name="ImageSearch"></a>type ImageSearch ImageSearch is the returned structure for SearchImage. It is returned @@ -1460,7 +1494,7 @@ graph_status [InfoGraphStatus](#InfoGraphStatus) run_root [string](https://godoc.org/builtin#string) ### <a name="ListContainerData"></a>type ListContainerData -ListContainer is the returned struct for an individual container +ListContainerData is the returned struct for an individual container id [string](https://godoc.org/builtin#string) diff --git a/cmd/podman/commands.go b/cmd/podman/commands.go new file mode 100644 index 000000000..98eb92bab --- /dev/null +++ b/cmd/podman/commands.go @@ -0,0 +1,151 @@ +// +build !remoteclient + +package main + +import "github.com/urfave/cli" + +func getAppCommands() []cli.Command { + return []cli.Command{ + attachCommand, + commitCommand, + buildCommand, + createCommand, + diffCommand, + execCommand, + exportCommand, + importCommand, + killCommand, + kubeCommand, + loadCommand, + loginCommand, + logoutCommand, + logsCommand, + mountCommand, + pauseCommand, + psCommand, + podCommand, + portCommand, + pushCommand, + playCommand, + restartCommand, + rmCommand, + runCommand, + saveCommand, + searchCommand, + startCommand, + statsCommand, + stopCommand, + topCommand, + umountCommand, + unpauseCommand, + versionCommand, + volumeCommand, + waitCommand, + } +} + +func getImageSubCommands() []cli.Command { + return []cli.Command{ + buildCommand, + importCommand, + loadCommand, + pullCommand, + saveCommand, + trustCommand, + signCommand, + } +} + +func getContainerSubCommands() []cli.Command { + return []cli.Command{ + attachCommand, + checkpointCommand, + cleanupCommand, + containerExistsCommand, + commitCommand, + createCommand, + diffCommand, + execCommand, + exportCommand, + killCommand, + logsCommand, + psCommand, + mountCommand, + pauseCommand, + portCommand, + pruneContainersCommand, + refreshCommand, + restartCommand, + restoreCommand, + rmCommand, + runCommand, + runlabelCommand, + startCommand, + statsCommand, + stopCommand, + topCommand, + umountCommand, + unpauseCommand, + // updateCommand, + waitCommand, + } +} +func getMainAppFlags() []cli.Flag { + return []cli.Flag{ + cli.StringFlag{ + Name: "cgroup-manager", + Usage: "cgroup manager to use (cgroupfs or systemd, default systemd)", + }, + cli.StringFlag{ + Name: "cni-config-dir", + Usage: "path of the configuration directory for CNI networks", + }, + cli.StringFlag{ + Name: "conmon", + Usage: "path of the conmon binary", + }, + cli.StringFlag{ + Name: "default-mounts-file", + Usage: "path to default mounts file", + Hidden: true, + }, + cli.StringSliceFlag{ + Name: "hooks-dir", + Usage: "set the OCI hooks directory path (may be set multiple times)", + }, + cli.IntFlag{ + Name: "max-workers", + Usage: "the maximum number of workers for parallel operations", + Hidden: true, + }, + cli.StringFlag{ + Name: "namespace", + Usage: "set the libpod namespace, used to create separate views of the containers and pods on the system", + Value: "", + }, + cli.StringFlag{ + Name: "root", + Usage: "path to the root directory in which data, including images, is stored", + }, + cli.StringFlag{ + Name: "runroot", + Usage: "path to the 'run directory' where all state information is stored", + }, + cli.StringFlag{ + Name: "runtime", + Usage: "path to the OCI-compatible binary used to run containers, default is /usr/bin/runc", + }, + cli.StringFlag{ + Name: "storage-driver, s", + Usage: "select which storage driver is used to manage storage of images and containers (default is overlay)", + }, + cli.StringSliceFlag{ + Name: "storage-opt", + Usage: "used to pass an option to the storage driver", + }, + cli.BoolFlag{ + Name: "syslog", + Usage: "output logging information to syslog as well as the console", + }, + } +} diff --git a/cmd/podman/commands_remoteclient.go b/cmd/podman/commands_remoteclient.go new file mode 100644 index 000000000..6701e14a1 --- /dev/null +++ b/cmd/podman/commands_remoteclient.go @@ -0,0 +1,21 @@ +// +build remoteclient + +package main + +import "github.com/urfave/cli" + +func getAppCommands() []cli.Command { + return []cli.Command{} +} + +func getImageSubCommands() []cli.Command { + return []cli.Command{} +} + +func getContainerSubCommands() []cli.Command { + return []cli.Command{} +} + +func getMainAppFlags() []cli.Flag { + return []cli.Flag{} +} diff --git a/cmd/podman/container.go b/cmd/podman/container.go index 4bb6f287a..acbcbb644 100644 --- a/cmd/podman/container.go +++ b/cmd/podman/container.go @@ -1,52 +1,29 @@ package main import ( + "sort" + "github.com/urfave/cli" ) var ( - subCommands = []cli.Command{ - attachCommand, - checkpointCommand, - cleanupCommand, - containerExistsCommand, - commitCommand, - createCommand, - diffCommand, - execCommand, - exportCommand, + containerSubCommands = []cli.Command{ inspectCommand, - killCommand, - logsCommand, - psCommand, - mountCommand, - pauseCommand, - portCommand, - pruneContainersCommand, - refreshCommand, - restartCommand, - restoreCommand, - rmCommand, - runCommand, - runlabelCommand, - startCommand, - statsCommand, - stopCommand, - topCommand, - umountCommand, - unpauseCommand, - // updateCommand, - waitCommand, } - containerDescription = "Manage containers" containerCommand = cli.Command{ Name: "container", Usage: "Manage Containers", Description: containerDescription, ArgsUsage: "", - Subcommands: subCommands, + Subcommands: getContainerSubCommandsSorted(), UseShortOptionHandling: true, OnUsageError: usageErrorHandler, } ) + +func getContainerSubCommandsSorted() []cli.Command { + containerSubCommands = append(containerSubCommands, getContainerSubCommands()...) + sort.Sort(commandSortedAlpha{containerSubCommands}) + return containerSubCommands +} diff --git a/cmd/podman/image.go b/cmd/podman/image.go index 557fc1056..a51a90b0e 100644 --- a/cmd/podman/image.go +++ b/cmd/podman/image.go @@ -1,36 +1,36 @@ package main import ( + "sort" + "github.com/urfave/cli" ) var ( imageSubCommands = []cli.Command{ - buildCommand, historyCommand, - importCommand, imageExistsCommand, inspectCommand, - loadCommand, lsImagesCommand, pruneImagesCommand, pullCommand, - pushCommand, rmImageCommand, - saveCommand, tagCommand, - trustCommand, - signCommand, } - imageDescription = "Manage images" imageCommand = cli.Command{ Name: "image", Usage: "Manage images", Description: imageDescription, ArgsUsage: "", - Subcommands: imageSubCommands, + Subcommands: getImageSubCommandsSorted(), UseShortOptionHandling: true, OnUsageError: usageErrorHandler, } ) + +func getImageSubCommandsSorted() []cli.Command { + imageSubCommands = append(imageSubCommands, getImageSubCommands()...) + sort.Sort(commandSortedAlpha{imageSubCommands}) + return imageSubCommands +} diff --git a/cmd/podman/inspect.go b/cmd/podman/inspect.go index 2f1e97c6c..3ef740463 100644 --- a/cmd/podman/inspect.go +++ b/cmd/podman/inspect.go @@ -2,12 +2,13 @@ package main import ( "context" + "encoding/json" "strings" "github.com/containers/libpod/cmd/podman/formats" - "github.com/containers/libpod/cmd/podman/libpodruntime" "github.com/containers/libpod/cmd/podman/shared" - "github.com/containers/libpod/libpod" + "github.com/containers/libpod/libpod/adapter" + cc "github.com/containers/libpod/pkg/spec" "github.com/containers/libpod/pkg/util" "github.com/pkg/errors" "github.com/urfave/cli" @@ -63,7 +64,7 @@ func inspectCmd(c *cli.Context) error { return err } - runtime, err := libpodruntime.GetRuntime(c) + runtime, err := adapter.GetRuntime(c) if err != nil { return errors.Wrapf(err, "error creating libpod runtime") } @@ -104,7 +105,7 @@ func inspectCmd(c *cli.Context) error { } // func iterateInput iterates the images|containers the user has requested and returns the inspect data and error -func iterateInput(ctx context.Context, c *cli.Context, args []string, runtime *libpod.Runtime, inspectType string) ([]interface{}, error) { +func iterateInput(ctx context.Context, c *cli.Context, args []string, runtime *adapter.LocalRuntime, inspectType string) ([]interface{}, error) { var ( data interface{} inspectedItems []interface{} @@ -124,13 +125,18 @@ func iterateInput(ctx context.Context, c *cli.Context, args []string, runtime *l inspectError = errors.Wrapf(err, "error getting libpod container inspect data %s", ctr.ID()) break } - data, err = shared.GetCtrInspectInfo(ctr, libpodInspectData) + artifact, err := getArtifact(ctr) + if inspectError != nil { + inspectError = err + break + } + data, err = shared.GetCtrInspectInfo(ctr.Config(), libpodInspectData, artifact) if err != nil { inspectError = errors.Wrapf(err, "error parsing container data %q", ctr.ID()) break } case inspectTypeImage: - image, err := runtime.ImageRuntime().NewFromLocal(input) + image, err := runtime.NewImageFromLocal(input) if err != nil { inspectError = errors.Wrapf(err, "error getting image %q", input) break @@ -143,7 +149,7 @@ func iterateInput(ctx context.Context, c *cli.Context, args []string, runtime *l case inspectAll: ctr, err := runtime.LookupContainer(input) if err != nil { - image, err := runtime.ImageRuntime().NewFromLocal(input) + image, err := runtime.NewImageFromLocal(input) if err != nil { inspectError = errors.Wrapf(err, "error getting image %q", input) break @@ -159,7 +165,12 @@ func iterateInput(ctx context.Context, c *cli.Context, args []string, runtime *l inspectError = errors.Wrapf(err, "error getting libpod container inspect data %s", ctr.ID()) break } - data, err = shared.GetCtrInspectInfo(ctr, libpodInspectData) + artifact, inspectError := getArtifact(ctr) + if inspectError != nil { + inspectError = err + break + } + data, err = shared.GetCtrInspectInfo(ctr.Config(), libpodInspectData, artifact) if err != nil { inspectError = errors.Wrapf(err, "error parsing container data %s", ctr.ID()) break @@ -172,3 +183,15 @@ func iterateInput(ctx context.Context, c *cli.Context, args []string, runtime *l } return inspectedItems, inspectError } + +func getArtifact(ctr *adapter.Container) (*cc.CreateConfig, error) { + var createArtifact cc.CreateConfig + artifact, err := ctr.GetArtifact("create-config") + if err != nil { + return nil, err + } + if err := json.Unmarshal(artifact, &createArtifact); err != nil { + return nil, err + } + return &createArtifact, nil +} diff --git a/cmd/podman/main.go b/cmd/podman/main.go index ce60bbfb7..20486e80d 100644 --- a/cmd/podman/main.go +++ b/cmd/podman/main.go @@ -6,6 +6,7 @@ import ( "os" "os/exec" "runtime/pprof" + "sort" "syscall" "github.com/containers/libpod/libpod" @@ -47,6 +48,28 @@ var cmdsNotRequiringRootless = map[string]bool{ "top": true, } +type commandSorted []cli.Command + +func (a commandSorted) Len() int { return len(a) } +func (a commandSorted) Swap(i, j int) { a[i], a[j] = a[j], a[i] } + +type commandSortedAlpha struct{ commandSorted } + +func (a commandSortedAlpha) Less(i, j int) bool { + return a.commandSorted[i].Name < a.commandSorted[j].Name +} + +type flagSorted []cli.Flag + +func (a flagSorted) Len() int { return len(a) } +func (a flagSorted) Swap(i, j int) { a[i], a[j] = a[j], a[i] } + +type flagSortedAlpha struct{ flagSorted } + +func (a flagSortedAlpha) Less(i, j int) bool { + return a.flagSorted[i].GetName() < a.flagSorted[j].GetName() +} + func main() { debug := false cpuProfile := false @@ -64,52 +87,20 @@ func main() { app.Version = version.Version app.Commands = []cli.Command{ - attachCommand, - commitCommand, containerCommand, - buildCommand, - createCommand, - diffCommand, - execCommand, - exportCommand, historyCommand, imageCommand, imagesCommand, - importCommand, infoCommand, inspectCommand, - killCommand, - kubeCommand, - loadCommand, - loginCommand, - logoutCommand, - logsCommand, - mountCommand, - pauseCommand, - psCommand, - podCommand, - portCommand, pullCommand, - pushCommand, - playCommand, - restartCommand, - rmCommand, rmiCommand, - runCommand, - saveCommand, - searchCommand, - startCommand, - statsCommand, - stopCommand, tagCommand, - topCommand, - umountCommand, - unpauseCommand, - versionCommand, - volumeCommand, - waitCommand, } + app.Commands = append(app.Commands, getAppCommands()...) + sort.Sort(commandSortedAlpha{app.Commands}) + if varlinkCommand != nil { app.Commands = append(app.Commands, *varlinkCommand) } @@ -192,79 +183,28 @@ func main() { } app.Flags = []cli.Flag{ cli.StringFlag{ - Name: "cgroup-manager", - Usage: "cgroup manager to use (cgroupfs or systemd, default systemd)", - }, - cli.StringFlag{ - Name: "cni-config-dir", - Usage: "path of the configuration directory for CNI networks", - }, - cli.StringFlag{ Name: "config, c", Usage: "path of a libpod config file detailing container server configuration options", Hidden: true, }, cli.StringFlag{ - Name: "conmon", - Usage: "path of the conmon binary", - }, - cli.StringFlag{ Name: "cpu-profile", Usage: "path for the cpu profiling results", }, cli.StringFlag{ - Name: "default-mounts-file", - Usage: "path to default mounts file", - Hidden: true, - }, - cli.StringSliceFlag{ - Name: "hooks-dir", - Usage: "set the OCI hooks directory path (may be set multiple times)", - }, - cli.IntFlag{ - Name: "max-workers", - Usage: "the maximum number of workers for parallel operations", - Hidden: true, - }, - cli.StringFlag{ Name: "log-level", Usage: "log messages above specified level: debug, info, warn, error (default), fatal or panic", Value: "error", }, cli.StringFlag{ - Name: "namespace", - Usage: "set the libpod namespace, used to create separate views of the containers and pods on the system", - Value: "", - }, - cli.StringFlag{ - Name: "root", - Usage: "path to the root directory in which data, including images, is stored", - }, - cli.StringFlag{ Name: "tmpdir", Usage: "path to the tmp directory", }, - cli.StringFlag{ - Name: "runroot", - Usage: "path to the 'run directory' where all state information is stored", - }, - cli.StringFlag{ - Name: "runtime", - Usage: "path to the OCI-compatible binary used to run containers, default is /usr/bin/runc", - }, - cli.StringFlag{ - Name: "storage-driver, s", - Usage: "select which storage driver is used to manage storage of images and containers (default is overlay)", - }, - cli.StringSliceFlag{ - Name: "storage-opt", - Usage: "used to pass an option to the storage driver", - }, - cli.BoolFlag{ - Name: "syslog", - Usage: "output logging information to syslog as well as the console", - }, } + + app.Flags = append(app.Flags, getMainAppFlags()...) + sort.Sort(flagSortedAlpha{app.Flags}) + // Check if /etc/containers/registries.conf exists when running in // in a local environment. CheckForRegistries() diff --git a/cmd/podman/shared/container.go b/cmd/podman/shared/container.go index a904ef75a..9040c4a5c 100644 --- a/cmd/podman/shared/container.go +++ b/cmd/podman/shared/container.go @@ -2,7 +2,6 @@ package shared import ( "context" - "encoding/json" "fmt" "github.com/google/shlex" "io" @@ -446,8 +445,7 @@ func getStrFromSquareBrackets(cmd string) string { // GetCtrInspectInfo takes container inspect data and collects all its info into a ContainerData // structure for inspection related methods -func GetCtrInspectInfo(ctr *libpod.Container, ctrInspectData *inspect.ContainerInspectData) (*inspect.ContainerData, error) { - config := ctr.Config() +func GetCtrInspectInfo(config *libpod.ContainerConfig, ctrInspectData *inspect.ContainerInspectData, createArtifact *cc.CreateConfig) (*inspect.ContainerData, error) { spec := config.Spec cpus, mems, period, quota, realtimePeriod, realtimeRuntime, shares := getCPUInfo(spec) @@ -456,16 +454,6 @@ func GetCtrInspectInfo(ctr *libpod.Container, ctrInspectData *inspect.ContainerI pidsLimit := getPidsInfo(spec) cgroup := getCgroup(spec) - var createArtifact cc.CreateConfig - artifact, err := ctr.GetArtifact("create-config") - if err == nil { - if err := json.Unmarshal(artifact, &createArtifact); err != nil { - return nil, err - } - } else { - logrus.Errorf("couldn't get some inspect information, error getting artifact %q: %v", ctr.ID(), err) - } - data := &inspect.ContainerData{ ctrInspectData, &inspect.HostConfig{ @@ -493,7 +481,7 @@ func GetCtrInspectInfo(ctr *libpod.Container, ctrInspectData *inspect.ContainerI PidsLimit: pidsLimit, Privileged: config.Privileged, ReadonlyRootfs: spec.Root.Readonly, - Runtime: ctr.RuntimeName(), + Runtime: config.OCIRuntime, NetworkMode: string(createArtifact.NetMode), IpcMode: string(createArtifact.IpcMode), Cgroup: cgroup, diff --git a/cmd/podman/varlink/io.podman.varlink b/cmd/podman/varlink/io.podman.varlink index c6f1d3f1b..79300f9bc 100644 --- a/cmd/podman/varlink/io.podman.varlink +++ b/cmd/podman/varlink/io.podman.varlink @@ -1035,6 +1035,22 @@ method GenerateKubeService() -> (notimplemented: NotImplemented) # like that created by GenerateKube. See also [GenerateKube](GenerateKube). method ReplayKube() -> (notimplemented: NotImplemented) +# ContainerConfig returns a container's config in string form. This call is for +# development of Podman only and generally should not be used. +method ContainerConfig(name: string) -> (config: string) + +# ContainerArtifacts returns a container's artifacts in string form. This call is for +# development of Podman only and generally should not be used. +method ContainerArtifacts(name: string, artifactName: string) -> (config: string) + +# ContainerInspectData returns a container's inspect data in string form. This call is for +# development of Podman only and generally should not be used. +method ContainerInspectData(name: string) -> (config: string) + +# ContainerStateData returns a container's state config in string form. This call is for +# development of Podman only and generally should not be used. +method ContainerStateData(name: string) -> (config: string) + # ImageNotFound means the image could not be found by the provided name or ID in local storage. error ImageNotFound (name: string) diff --git a/hack/get_ci_vm.sh b/hack/get_ci_vm.sh index e1d8e42b9..b058b4273 100755 --- a/hack/get_ci_vm.sh +++ b/hack/get_ci_vm.sh @@ -54,15 +54,19 @@ delvm() { showrun $CLEANUP_CMD # prompts for Yes/No } -show_usage(){ +image_hints() { + egrep '[[:space:]]+[[:alnum:]].+_CACHE_IMAGE_NAME:[[:space:]+"[[:print:]]+"' \ + "$LIBPODROOT/.cirrus.yml" | cut -d: -f 2 | tr -d '"[:blank:]' | \ + grep -v 'notready' | grep -v 'image-builder' | sort -u +} + +show_usage() { echo -e "\n${RED}ERROR: $1${NOR}" echo -e "${YEL}Usage: $(basename $0) [-s | -p] <image_name>${NOR}\n" if [[ -r ".cirrus.yml" ]] then - egrep 'image_name' ".cirrus.yml" | grep -v '#' | cut -d: -f 2 | \ - tr -d [:blank:] | sort -u > "$TEMPFILE" echo -e "${YEL}Some possible image_name values (from .cirrus.yml):${NOR}" - cat $TEMPFILE + image_hints echo "" fi exit 1 diff --git a/libpod/adapter/containers_remote.go b/libpod/adapter/containers_remote.go new file mode 100644 index 000000000..9623304e5 --- /dev/null +++ b/libpod/adapter/containers_remote.go @@ -0,0 +1,50 @@ +// +build remoteclient + +package adapter + +import ( + "encoding/json" + + iopodman "github.com/containers/libpod/cmd/podman/varlink" + "github.com/containers/libpod/libpod" + "github.com/containers/libpod/pkg/inspect" +) + +// Inspect returns an inspect struct from varlink +func (c *Container) Inspect(size bool) (*inspect.ContainerInspectData, error) { + reply, err := iopodman.ContainerInspectData().Call(c.Runtime.Conn, c.ID()) + if err != nil { + return nil, err + } + data := inspect.ContainerInspectData{} + if err := json.Unmarshal([]byte(reply), &data); err != nil { + return nil, err + } + return &data, err +} + +// ID returns the ID of the container +func (c *Container) ID() string { + return c.config.ID +} + +// GetArtifact returns a container's artifacts +func (c *Container) GetArtifact(name string) ([]byte, error) { + var data []byte + reply, err := iopodman.ContainerArtifacts().Call(c.Runtime.Conn, c.ID(), name) + if err != nil { + return nil, err + } + if err := json.Unmarshal([]byte(reply), &data); err != nil { + return nil, err + } + return data, err +} + +// Config returns a container's Config ... same as ctr.Config() +func (c *Container) Config() *libpod.ContainerConfig { + if c.config != nil { + return c.config + } + return c.Runtime.Config(c.ID()) +} diff --git a/libpod/adapter/images_remote.go b/libpod/adapter/images_remote.go index 77b0629a7..e7b38dccc 100644 --- a/libpod/adapter/images_remote.go +++ b/libpod/adapter/images_remote.go @@ -3,15 +3,22 @@ package adapter import ( - "github.com/containers/libpod/libpod" + "context" + "encoding/json" + + iopodman "github.com/containers/libpod/cmd/podman/varlink" + "github.com/containers/libpod/pkg/inspect" ) -// Images returns information for the host system and its components -func (r RemoteRuntime) Images() ([]libpod.InfoData, error) { - conn, err := r.Connect() +// Inspect returns returns an ImageData struct from over a varlink connection +func (i *ContainerImage) Inspect(ctx context.Context) (*inspect.ImageData, error) { + reply, err := iopodman.InspectImage().Call(i.Runtime.Conn, i.ID()) if err != nil { return nil, err } - _ = conn - return nil, nil + data := inspect.ImageData{} + if err := json.Unmarshal([]byte(reply), &data); err != nil { + return nil, err + } + return &data, nil } diff --git a/libpod/adapter/runtime_remote.go b/libpod/adapter/runtime_remote.go index 8ef8fe167..7189348bc 100644 --- a/libpod/adapter/runtime_remote.go +++ b/libpod/adapter/runtime_remote.go @@ -4,15 +4,18 @@ package adapter import ( "context" + "encoding/json" "fmt" "io" "strings" "time" "github.com/containers/image/types" - iopodman "github.com/containers/libpod/cmd/podman/varlink" + "github.com/containers/libpod/cmd/podman/varlink" + "github.com/containers/libpod/libpod" "github.com/containers/libpod/libpod/image" - digest "github.com/opencontainers/go-digest" + "github.com/opencontainers/go-digest" + "github.com/sirupsen/logrus" "github.com/urfave/cli" "github.com/varlink/go/varlink" ) @@ -80,21 +83,9 @@ type Container struct { // remoteContainer .... type remoteContainer struct { - ID string - Image string - ImageID string - Command []string - Created time.Time - RunningFor string - Status string - //Ports []ocicni.PortMapping - RootFsSize int64 - RWSize int64 - Names string - Labels []map[string]string - // Mounts []string - // ContainerRunning bool - //Namespaces []LinuxNameSpace + Runtime *LocalRuntime + config *libpod.ContainerConfig + state *libpod.ContainerState } // GetImages returns a slice of containerimages over a varlink connection @@ -272,39 +263,60 @@ func (ci *ContainerImage) History(ctx context.Context) ([]*image.History, error) // LookupContainer gets basic information about container over a varlink // connection and then translates it to a *Container -func (r *RemoteRuntime) LookupContainer(idOrName string) (*Container, error) { - container, err := iopodman.GetContainer().Call(r.Conn, idOrName) +func (r *LocalRuntime) LookupContainer(idOrName string) (*Container, error) { + state, err := r.ContainerState(idOrName) if err != nil { return nil, err } - return listContainerDataToContainer(container) + config := r.Config(idOrName) + if err != nil { + return nil, err + } + + rc := remoteContainer{ + r, + config, + state, + } + + c := Container{ + rc, + } + return &c, nil +} + +func (r *LocalRuntime) GetLatestContainer() (*Container, error) { + return nil, libpod.ErrNotImplemented } -// listContainerDataToContainer takes a varlink listcontainerData struct and makes -// an "adapted" Container -func listContainerDataToContainer(listData iopodman.ListContainerData) (*Container, error) { - created, err := splitStringDate(listData.Createdat) +// ContainerState returns the "state" of the container. +func (r *LocalRuntime) ContainerState(name string) (*libpod.ContainerState, error) { //no-lint + reply, err := iopodman.ContainerStateData().Call(r.Conn, name) if err != nil { return nil, err } - rc := remoteContainer{ - // TODO commented out attributes will be populated when podman-remote ps - // is implemented. They are not needed yet for basic container operations. - ID: listData.Id, - Image: listData.Image, - ImageID: listData.Imageid, - Command: listData.Command, - Created: created, - RunningFor: listData.Runningfor, - Status: listData.Status, - //ports: - RootFsSize: listData.Rootfssize, - RWSize: listData.Rwsize, - Names: listData.Names, - //Labels: - //Mounts - //ContainerRunning: - //namespaces: + data := libpod.ContainerState{} + if err := json.Unmarshal([]byte(reply), &data); err != nil { + return nil, err } - return &Container{rc}, nil + return &data, err + +} + +// Config returns a container config +func (r *LocalRuntime) Config(name string) *libpod.ContainerConfig { + // TODO the Spec being returned is not populated. Matt and I could not figure out why. Will defer + // further looking into it for after devconf. + // The libpod function for this has no errors so we are kind of in a tough + // spot here. Logging the errors for now. + reply, err := iopodman.ContainerConfig().Call(r.Conn, name) + if err != nil { + logrus.Error("call to container.config failed") + } + data := libpod.ContainerConfig{} + if err := json.Unmarshal([]byte(reply), &data); err != nil { + logrus.Error("failed to unmarshal container inspect data") + } + return &data + } diff --git a/libpod/boltdb_state.go b/libpod/boltdb_state.go index e7a07a9a8..5bc15dd7f 100644 --- a/libpod/boltdb_state.go +++ b/libpod/boltdb_state.go @@ -205,7 +205,7 @@ func (s *BoltState) Refresh() error { return errors.Wrapf(ErrInternal, "container %s missing state in DB", string(id)) } - state := new(containerState) + state := new(ContainerState) if err := json.Unmarshal(stateBytes, state); err != nil { return errors.Wrapf(err, "error unmarshalling state for container %s", string(id)) @@ -325,7 +325,7 @@ func (s *BoltState) Container(id string) (*Container, error) { ctr := new(Container) ctr.config = new(ContainerConfig) - ctr.state = new(containerState) + ctr.state = new(ContainerState) db, err := s.getDBCon() if err != nil { @@ -361,7 +361,7 @@ func (s *BoltState) LookupContainer(idOrName string) (*Container, error) { ctr := new(Container) ctr.config = new(ContainerConfig) - ctr.state = new(containerState) + ctr.state = new(ContainerState) db, err := s.getDBCon() if err != nil { @@ -542,7 +542,7 @@ func (s *BoltState) UpdateContainer(ctr *Container) error { return errors.Wrapf(ErrNSMismatch, "container %s is in namespace %q, does not match our namespace %q", ctr.ID(), ctr.config.Namespace, s.namespace) } - newState := new(containerState) + newState := new(ContainerState) netNSPath := "" ctrID := []byte(ctr.ID()) @@ -754,7 +754,7 @@ func (s *BoltState) AllContainers() ([]*Container, error) { ctr := new(Container) ctr.config = new(ContainerConfig) - ctr.state = new(containerState) + ctr.state = new(ContainerState) if err := s.getContainerFromDB(id, ctr, ctrBucket); err != nil { // If the error is a namespace mismatch, we can @@ -1140,7 +1140,7 @@ func (s *BoltState) PodContainers(pod *Pod) ([]*Container, error) { err = podCtrs.ForEach(func(id, val []byte) error { newCtr := new(Container) newCtr.config = new(ContainerConfig) - newCtr.state = new(containerState) + newCtr.state = new(ContainerState) ctrs = append(ctrs, newCtr) return s.getContainerFromDB(id, newCtr, ctrBkt) diff --git a/libpod/boltdb_state_linux.go b/libpod/boltdb_state_linux.go index d91f311e5..09a9be606 100644 --- a/libpod/boltdb_state_linux.go +++ b/libpod/boltdb_state_linux.go @@ -8,7 +8,7 @@ import ( // replaceNetNS handle network namespace transitions after updating a // container's state. -func replaceNetNS(netNSPath string, ctr *Container, newState *containerState) error { +func replaceNetNS(netNSPath string, ctr *Container, newState *ContainerState) error { if netNSPath != "" { // Check if the container's old state has a good netns if ctr.state.NetNS != nil && netNSPath == ctr.state.NetNS.Path() { diff --git a/libpod/boltdb_state_unsupported.go b/libpod/boltdb_state_unsupported.go index 64610d304..244dc51a0 100644 --- a/libpod/boltdb_state_unsupported.go +++ b/libpod/boltdb_state_unsupported.go @@ -3,7 +3,7 @@ package libpod // replaceNetNS is exclusive to the Linux platform and is a no-op elsewhere -func replaceNetNS(netNSPath string, ctr *Container, newState *containerState) error { +func replaceNetNS(netNSPath string, ctr *Container, newState *ContainerState) error { return nil } diff --git a/libpod/common_test.go b/libpod/common_test.go index 4af68a040..df730098e 100644 --- a/libpod/common_test.go +++ b/libpod/common_test.go @@ -48,7 +48,7 @@ func getTestContainer(id, name string, manager lock.Manager) (*Container, error) }, }, }, - state: &containerState{ + state: &ContainerState{ State: ContainerStateRunning, ConfigPath: "/does/not/exist/specs/" + id, RunDir: "/does/not/exist/tmp/", @@ -166,10 +166,10 @@ func testContainersEqual(t *testing.T, a, b *Container, allowedEmpty bool) { aConfig := new(ContainerConfig) bConfig := new(ContainerConfig) - aState := new(containerState) - bState := new(containerState) + aState := new(ContainerState) + bState := new(ContainerState) - blankState := new(containerState) + blankState := new(ContainerState) assert.Equal(t, a.valid, b.valid) diff --git a/libpod/container.go b/libpod/container.go index b5f6a29ba..b0589be3b 100644 --- a/libpod/container.go +++ b/libpod/container.go @@ -116,7 +116,7 @@ func (ns LinuxNS) String() string { type Container struct { config *ContainerConfig - state *containerState + state *ContainerState // Batched indicates that a container has been locked as part of a // Batch() operation @@ -136,10 +136,10 @@ type Container struct { requestedIP net.IP } -// containerState contains the current state of the container +// ContainerState contains the current state of the container // It is stored on disk in a tmpfs and recreated on reboot // easyjson:json -type containerState struct { +type ContainerState struct { // The current state of the running container State ContainerStatus `json:"state"` // The path to the JSON OCI runtime spec for this container @@ -1063,3 +1063,18 @@ func networkDisabled(c *Container) (bool, error) { } return false, nil } + +// ContainerState returns containerstate struct +func (c *Container) ContainerState() (*ContainerState, error) { + if !c.batched { + c.lock.Lock() + defer c.lock.Unlock() + + if err := c.syncContainer(); err != nil { + return nil, err + } + } + returnConfig := new(ContainerState) + deepcopier.Copy(c.state).To(returnConfig) + return c.state, nil +} diff --git a/libpod/container_internal.go b/libpod/container_internal.go index ce8791f08..39c1501da 100644 --- a/libpod/container_internal.go +++ b/libpod/container_internal.go @@ -388,7 +388,7 @@ func (c *Container) teardownStorage() error { // Reset resets state fields to default values // It is performed before a refresh and clears the state after a reboot // It does not save the results - assumes the database will do that for us -func resetState(state *containerState) error { +func resetState(state *ContainerState) error { state.PID = 0 state.Mountpoint = "" state.Mounted = false diff --git a/libpod/container_internal_test.go b/libpod/container_internal_test.go index 124f1d20e..f1e2b70a7 100644 --- a/libpod/container_internal_test.go +++ b/libpod/container_internal_test.go @@ -37,7 +37,7 @@ func TestPostDeleteHooks(t *testing.T) { }, StaticDir: dir, // not the bundle, but good enough for this test }, - state: &containerState{ + state: &ContainerState{ ExtensionStageHooks: map[string][]rspec.Hook{ "poststop": { rspec.Hook{ diff --git a/libpod/networking_linux.go b/libpod/networking_linux.go index a343bee6a..f9caf26d1 100644 --- a/libpod/networking_linux.go +++ b/libpod/networking_linux.go @@ -121,6 +121,19 @@ func (r *Runtime) createNetNS(ctr *Container) (n ns.NetNS, q []*cnitypes.Result, return ctrNS, networkStatus, err } +type slirp4netnsCmdArg struct { + Proto string `json:"proto,omitempty"` + HostAddr string `json:"host_addr"` + HostPort int32 `json:"host_port"` + GuestAddr string `json:"guest_addr"` + GuestPort int32 `json:"guest_port"` +} + +type slirp4netnsCmd struct { + Execute string `json:"execute"` + Args slirp4netnsCmdArg `json:"arguments"` +} + // Configure the network namespace for a rootless container func (r *Runtime) setupRootlessNetNS(ctr *Container) (err error) { defer ctr.rootlessSlirpSyncR.Close() @@ -139,7 +152,15 @@ func (r *Runtime) setupRootlessNetNS(ctr *Container) (err error) { defer syncR.Close() defer syncW.Close() - cmd := exec.Command(path, "-c", "-e", "3", "-r", "4", fmt.Sprintf("%d", ctr.state.PID), "tap0") + havePortMapping := len(ctr.Config().PortMappings) > 0 + apiSocket := filepath.Join(r.ociRuntime.tmpDir, fmt.Sprintf("%s.net", ctr.config.ID)) + var cmd *exec.Cmd + if havePortMapping { + // if we need ports to be mapped from the host, create a API socket to use for communicating with slirp4netns. + cmd = exec.Command(path, "-c", "-e", "3", "-r", "4", "--api-socket", apiSocket, fmt.Sprintf("%d", ctr.state.PID), "tap0") + } else { + cmd = exec.Command(path, "-c", "-e", "3", "-r", "4", fmt.Sprintf("%d", ctr.state.PID), "tap0") + } cmd.SysProcAttr = &syscall.SysProcAttr{ Setpgid: true, @@ -162,19 +183,99 @@ func (r *Runtime) setupRootlessNetNS(ctr *Container) (err error) { if os.IsTimeout(err) { // Check if the process is still running. var status syscall.WaitStatus - _, err := syscall.Wait4(cmd.Process.Pid, &status, syscall.WNOHANG, nil) + pid, err := syscall.Wait4(cmd.Process.Pid, &status, syscall.WNOHANG, nil) if err != nil { return errors.Wrapf(err, "failed to read slirp4netns process status") } + if pid != cmd.Process.Pid { + continue + } if status.Exited() || status.Signaled() { return errors.New("slirp4netns failed") } - continue } return errors.Wrapf(err, "failed to read from slirp4netns sync pipe") } } + + if havePortMapping { + const pidWaitTimeout = 60 * time.Second + chWait := make(chan error) + go func() { + interval := 25 * time.Millisecond + for i := time.Duration(0); i < pidWaitTimeout; i += interval { + // Check if the process is still running. + var status syscall.WaitStatus + pid, err := syscall.Wait4(cmd.Process.Pid, &status, syscall.WNOHANG, nil) + if err != nil { + break + } + if pid != cmd.Process.Pid { + continue + } + if status.Exited() || status.Signaled() { + chWait <- fmt.Errorf("slirp4netns exited with status %d", status.ExitStatus()) + } + time.Sleep(interval) + } + }() + defer close(chWait) + + // wait that API socket file appears before trying to use it. + if _, err := WaitForFile(apiSocket, chWait, pidWaitTimeout*time.Millisecond); err != nil { + return errors.Wrapf(err, "waiting for slirp4nets to create the api socket file %s", apiSocket) + } + + // for each port we want to add we need to open a connection to the slirp4netns control socket + // and send the add_hostfwd command. + for _, i := range ctr.config.PortMappings { + conn, err := net.Dial("unix", apiSocket) + if err != nil { + return errors.Wrapf(err, "cannot open connection to %s", apiSocket) + } + defer conn.Close() + hostIP := i.HostIP + if hostIP == "" { + hostIP = "0.0.0.0" + } + cmd := slirp4netnsCmd{ + Execute: "add_hostfwd", + Args: slirp4netnsCmdArg{ + Proto: i.Protocol, + HostAddr: hostIP, + HostPort: i.HostPort, + GuestPort: i.ContainerPort, + }, + } + // create the JSON payload and send it. Mark the end of request shutting down writes + // to the socket, as requested by slirp4netns. + data, err := json.Marshal(&cmd) + if err != nil { + return errors.Wrapf(err, "cannot marshal JSON for slirp4netns") + } + if _, err := conn.Write([]byte(fmt.Sprintf("%s\n", data))); err != nil { + return errors.Wrapf(err, "cannot write to control socket %s", apiSocket) + } + if err := conn.(*net.UnixConn).CloseWrite(); err != nil { + return errors.Wrapf(err, "cannot shutdown the socket %s", apiSocket) + } + buf := make([]byte, 2048) + len, err := conn.Read(buf) + if err != nil { + return errors.Wrapf(err, "cannot read from control socket %s", apiSocket) + } + // if there is no 'error' key in the received JSON data, then the operation was + // successful. + var y map[string]interface{} + if err := json.Unmarshal(buf[0:len], &y); err != nil { + return errors.Wrapf(err, "error parsing error status from slirp4netns") + } + if e, found := y["error"]; found { + return errors.Errorf("error from slirp4netns while setting up port redirection: %v", e) + } + } + } return nil } diff --git a/libpod/oci.go b/libpod/oci.go index b0c19d5e8..e55bd57dc 100644 --- a/libpod/oci.go +++ b/libpod/oci.go @@ -323,7 +323,7 @@ func (r *OCIRuntime) createOCIContainer(ctr *Container, cgroupParent string, res cmd.Env = append(cmd.Env, fmt.Sprintf("HOME=%s", os.Getenv("HOME"))) cmd.Env = append(cmd.Env, fmt.Sprintf("XDG_RUNTIME_DIR=%s", runtimeDir)) - if r.reservePorts { + if r.reservePorts && !ctr.config.NetMode.IsSlirp4netns() { ports, err := bindPorts(ctr.config.PortMappings) if err != nil { return err diff --git a/libpod/runtime_ctr.go b/libpod/runtime_ctr.go index 68599fe6d..6d5ce5a7e 100644 --- a/libpod/runtime_ctr.go +++ b/libpod/runtime_ctr.go @@ -48,7 +48,7 @@ func (r *Runtime) newContainer(ctx context.Context, rSpec *spec.Spec, options .. ctr := new(Container) ctr.config = new(ContainerConfig) - ctr.state = new(containerState) + ctr.state = new(ContainerState) ctr.config.ID = stringid.GenerateNonCryptoID() diff --git a/pkg/spec/createconfig.go b/pkg/spec/createconfig.go index 632d60b55..344f4afb9 100644 --- a/pkg/spec/createconfig.go +++ b/pkg/spec/createconfig.go @@ -422,11 +422,7 @@ func (c *CreateConfig) GetContainerCreateOptions(runtime *libpod.Runtime, pod *l } options = append(options, libpod.WithNetNSFrom(connectedCtr)) } else if !c.NetMode.IsHost() && !c.NetMode.IsNone() { - isRootless := rootless.IsRootless() postConfigureNetNS := c.NetMode.IsSlirp4netns() || (len(c.IDMappings.UIDMap) > 0 || len(c.IDMappings.GIDMap) > 0) && !c.UsernsMode.IsHost() - if isRootless && len(portBindings) > 0 { - return nil, errors.New("port bindings are not yet supported by rootless containers") - } options = append(options, libpod.WithNetNS(portBindings, postConfigureNetNS, string(c.NetMode), networks)) } diff --git a/pkg/varlinkapi/containers.go b/pkg/varlinkapi/containers.go index 07d981786..a01e3cc2b 100644 --- a/pkg/varlinkapi/containers.go +++ b/pkg/varlinkapi/containers.go @@ -12,6 +12,7 @@ import ( "github.com/containers/libpod/cmd/podman/shared" "github.com/containers/libpod/cmd/podman/varlink" "github.com/containers/libpod/libpod" + cc "github.com/containers/libpod/pkg/spec" "github.com/containers/storage/pkg/archive" "github.com/pkg/errors" ) @@ -68,7 +69,12 @@ func (i *LibpodAPI) InspectContainer(call iopodman.VarlinkCall, name string) err if err != nil { return call.ReplyErrorOccurred(err.Error()) } - data, err := shared.GetCtrInspectInfo(ctr, inspectInfo) + artifact, err := getArtifact(ctr) + if err != nil { + return call.ReplyErrorOccurred(err.Error()) + } + + data, err := shared.GetCtrInspectInfo(ctr.Config(), inspectInfo, artifact) if err != nil { return call.ReplyErrorOccurred(err.Error()) } @@ -462,3 +468,81 @@ func (i *LibpodAPI) ContainerRestore(call iopodman.VarlinkCall, name string, kee } return call.ReplyContainerRestore(ctr.ID()) } + +func getArtifact(ctr *libpod.Container) (*cc.CreateConfig, error) { + var createArtifact cc.CreateConfig + artifact, err := ctr.GetArtifact("create-config") + if err != nil { + return nil, err + } + if err := json.Unmarshal(artifact, &createArtifact); err != nil { + return nil, err + } + return &createArtifact, nil +} + +// ContainerConfig returns just the container.config struct +func (i *LibpodAPI) ContainerConfig(call iopodman.VarlinkCall, name string) error { + ctr, err := i.Runtime.LookupContainer(name) + if err != nil { + return call.ReplyErrorOccurred(err.Error()) + } + config := ctr.Config() + b, err := json.Marshal(config) + if err != nil { + return call.ReplyErrorOccurred("unable to serialize container config") + } + return call.ReplyContainerConfig(string(b)) +} + +// ContainerArtifacts returns an untouched container's artifact in string format +func (i *LibpodAPI) ContainerArtifacts(call iopodman.VarlinkCall, name, artifactName string) error { + ctr, err := i.Runtime.LookupContainer(name) + if err != nil { + return call.ReplyErrorOccurred(err.Error()) + } + artifacts, err := ctr.GetArtifact(artifactName) + if err != nil { + return call.ReplyErrorOccurred("unable to get container artifacts") + } + b, err := json.Marshal(artifacts) + if err != nil { + return call.ReplyErrorOccurred("unable to serialize container artifacts") + } + return call.ReplyContainerArtifacts(string(b)) +} + +// ContainerInspectData returns the inspect data of a container in string format +func (i *LibpodAPI) ContainerInspectData(call iopodman.VarlinkCall, name string) error { + ctr, err := i.Runtime.LookupContainer(name) + if err != nil { + return call.ReplyErrorOccurred(err.Error()) + } + data, err := ctr.Inspect(true) + if err != nil { + return call.ReplyErrorOccurred("unable to inspect container") + } + b, err := json.Marshal(data) + if err != nil { + return call.ReplyErrorOccurred("unable to serialize container inspect data") + } + return call.ReplyContainerInspectData(string(b)) + +} + +// ContainerStateData returns a container's state data in string format +func (i *LibpodAPI) ContainerStateData(call iopodman.VarlinkCall, name string) error { + ctr, err := i.Runtime.LookupContainer(name) + if err != nil { + return call.ReplyErrorOccurred(err.Error()) + } + data, err := ctr.ContainerState() + if err != nil { + return call.ReplyErrorOccurred("unable to obtain container state") + } + b, err := json.Marshal(data) + if err != nil { + return call.ReplyErrorOccurred("unable to serialize container inspect data") + } + return call.ReplyContainerStateData(string(b)) +} diff --git a/test/e2e/common_test.go b/test/e2e/common_test.go index e00a59b7f..308a6bf29 100644 --- a/test/e2e/common_test.go +++ b/test/e2e/common_test.go @@ -1,17 +1,19 @@ package integration import ( + "encoding/json" "fmt" - . "github.com/containers/libpod/test/utils" "os" "os/exec" + "path/filepath" + "strings" + "testing" + "github.com/containers/libpod/pkg/inspect" + . "github.com/containers/libpod/test/utils" "github.com/containers/storage/pkg/reexec" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "path/filepath" - "strings" - "testing" ) var ( @@ -172,28 +174,6 @@ func PodmanTestCreateUtil(tempDir string, remote bool) *PodmanTestIntegration { return p } -//MakeOptions assembles all the podman main options -func (p *PodmanTestIntegration) makeOptions(args []string) []string { - podmanOptions := strings.Split(fmt.Sprintf("--root %s --runroot %s --runtime %s --conmon %s --cni-config-dir %s --cgroup-manager %s", - p.CrioRoot, p.RunRoot, p.RunCBinary, p.ConmonBinary, p.CNIConfigDir, p.CgroupManager), " ") - if os.Getenv("HOOK_OPTION") != "" { - podmanOptions = append(podmanOptions, os.Getenv("HOOK_OPTION")) - } - podmanOptions = append(podmanOptions, strings.Split(p.StorageOptions, " ")...) - podmanOptions = append(podmanOptions, args...) - return podmanOptions -} - -// RestoreArtifact puts the cached image into our test store -func (p *PodmanTestIntegration) RestoreArtifact(image string) error { - fmt.Printf("Restoring %s...\n", image) - dest := strings.Split(image, "/") - destName := fmt.Sprintf("/tmp/%s.tar", strings.Replace(strings.Join(strings.Split(dest[len(dest)-1], "/"), ""), ":", "-", -1)) - restore := p.Podman([]string{"load", "-q", "-i", destName}) - restore.Wait(90) - return nil -} - // RestoreAllArtifacts unpacks all cached images func (p *PodmanTestIntegration) RestoreAllArtifacts() error { if os.Getenv("NO_TEST_CACHE") != "" { @@ -227,3 +207,12 @@ func (p *PodmanTestIntegration) CreateArtifact(image string) error { } return nil } + +// InspectImageJSON takes the session output of an inspect +// image and returns json +func (s *PodmanSessionIntegration) InspectImageJSON() []inspect.ImageData { + var i []inspect.ImageData + err := json.Unmarshal(s.Out.Contents(), &i) + Expect(err).To(BeNil()) + return i +} diff --git a/test/e2e/inspect_test.go b/test/e2e/inspect_test.go index 38c50da14..e5c471bf9 100644 --- a/test/e2e/inspect_test.go +++ b/test/e2e/inspect_test.go @@ -1,5 +1,3 @@ -// +build !remoteclient - package integration import ( @@ -45,6 +43,7 @@ var _ = Describe("Podman inspect", func() { }) It("podman inspect bogus container", func() { + SkipIfRemote() session := podmanTest.Podman([]string{"inspect", "foobar4321"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Not(Equal(0))) @@ -68,6 +67,7 @@ var _ = Describe("Podman inspect", func() { }) It("podman inspect container with size", func() { + SkipIfRemote() _, ec, _ := podmanTest.RunLsContainer("") Expect(ec).To(Equal(0)) @@ -79,6 +79,7 @@ var _ = Describe("Podman inspect", func() { }) It("podman inspect container and image", func() { + SkipIfRemote() ls, ec, _ := podmanTest.RunLsContainer("") Expect(ec).To(Equal(0)) cid := ls.OutputToString() @@ -90,6 +91,7 @@ var _ = Describe("Podman inspect", func() { }) It("podman inspect -l with additional input should fail", func() { + SkipIfRemote() result := podmanTest.Podman([]string{"inspect", "-l", "1234foobar"}) result.WaitWithDefaultTimeout() Expect(result.ExitCode()).To(Equal(125)) diff --git a/test/e2e/libpod_suite_remoteclient_test.go b/test/e2e/libpod_suite_remoteclient_test.go index cad4401f8..4b769a574 100644 --- a/test/e2e/libpod_suite_remoteclient_test.go +++ b/test/e2e/libpod_suite_remoteclient_test.go @@ -55,10 +55,10 @@ func (p *PodmanTestIntegration) RunLsContainer(name string) (*PodmanSessionInteg // InspectImageJSON takes the session output of an inspect // image and returns json -func (s *PodmanSessionIntegration) InspectImageJSON() []inspect.ImageData { - // TODO - return nil -} +//func (s *PodmanSessionIntegration) InspectImageJSON() []inspect.ImageData { +// // TODO +// return nil +//} func (p *PodmanTestIntegration) setDefaultRegistriesConfigEnv() { defaultFile := filepath.Join(INTEGRATION_ROOT, "test/registries.conf") @@ -143,7 +143,7 @@ func (p *PodmanTestIntegration) StartVarlink() { os.MkdirAll("/run/podman", 0755) } args := []string{"varlink", "--timeout", "0", "unix:/run/podman/io.podman"} - podmanOptions := p.MakeOptions(args) + podmanOptions := getVarlinkOptions(p, args) command := exec.Command(p.PodmanBinary, podmanOptions...) fmt.Printf("Running: %s %s\n", p.PodmanBinary, strings.Join(podmanOptions, " ")) command.Start() @@ -155,3 +155,34 @@ func (p *PodmanTestIntegration) StopVarlink() { varlinkSession.Kill() varlinkSession.Wait() } + +//MakeOptions assembles all the podman main options +func (p *PodmanTestIntegration) makeOptions(args []string) []string { + return args +} + +//MakeOptions assembles all the podman main options +func getVarlinkOptions(p *PodmanTestIntegration, args []string) []string { + podmanOptions := strings.Split(fmt.Sprintf("--root %s --runroot %s --runtime %s --conmon %s --cni-config-dir %s --cgroup-manager %s", + p.CrioRoot, p.RunRoot, p.RunCBinary, p.ConmonBinary, p.CNIConfigDir, p.CgroupManager), " ") + if os.Getenv("HOOK_OPTION") != "" { + podmanOptions = append(podmanOptions, os.Getenv("HOOK_OPTION")) + } + podmanOptions = append(podmanOptions, strings.Split(p.StorageOptions, " ")...) + podmanOptions = append(podmanOptions, args...) + return podmanOptions +} + +// RestoreArtifact puts the cached image into our test store +func (p *PodmanTestIntegration) RestoreArtifact(image string) error { + fmt.Printf("Restoring %s...\n", image) + dest := strings.Split(image, "/") + destName := fmt.Sprintf("/tmp/%s.tar", strings.Replace(strings.Join(strings.Split(dest[len(dest)-1], "/"), ""), ":", "-", -1)) + args := []string{"load", "-q", "-i", destName} + podmanOptions := getVarlinkOptions(p, args) + command := exec.Command(p.PodmanBinary, podmanOptions...) + fmt.Printf("Running: %s %s\n", p.PodmanBinary, strings.Join(podmanOptions, " ")) + command.Start() + command.Wait() + return nil +} diff --git a/test/e2e/libpod_suite_test.go b/test/e2e/libpod_suite_test.go index a7ce14239..1f218cbdf 100644 --- a/test/e2e/libpod_suite_test.go +++ b/test/e2e/libpod_suite_test.go @@ -118,15 +118,6 @@ func (s *PodmanSessionIntegration) InspectPodToJSON() libpod.PodInspect { return i } -// InspectImageJSON takes the session output of an inspect -// image and returns json -func (s *PodmanSessionIntegration) InspectImageJSON() []inspect.ImageData { - var i []inspect.ImageData - err := json.Unmarshal(s.Out.Contents(), &i) - Expect(err).To(BeNil()) - return i -} - // CreatePod creates a pod with no infra container // it optionally takes a pod name func (p *PodmanTestIntegration) CreatePod(name string) (*PodmanSessionIntegration, int, string) { @@ -213,5 +204,24 @@ func PodmanTestCreate(tempDir string) *PodmanTestIntegration { return PodmanTestCreateUtil(tempDir, false) } -//func (p *PodmanTestIntegration) StartVarlink() {} -//func (p *PodmanTestIntegration) StopVarlink() {} +//MakeOptions assembles all the podman main options +func (p *PodmanTestIntegration) makeOptions(args []string) []string { + podmanOptions := strings.Split(fmt.Sprintf("--root %s --runroot %s --runtime %s --conmon %s --cni-config-dir %s --cgroup-manager %s", + p.CrioRoot, p.RunRoot, p.RunCBinary, p.ConmonBinary, p.CNIConfigDir, p.CgroupManager), " ") + if os.Getenv("HOOK_OPTION") != "" { + podmanOptions = append(podmanOptions, os.Getenv("HOOK_OPTION")) + } + podmanOptions = append(podmanOptions, strings.Split(p.StorageOptions, " ")...) + podmanOptions = append(podmanOptions, args...) + return podmanOptions +} + +// RestoreArtifact puts the cached image into our test store +func (p *PodmanTestIntegration) RestoreArtifact(image string) error { + fmt.Printf("Restoring %s...\n", image) + dest := strings.Split(image, "/") + destName := fmt.Sprintf("/tmp/%s.tar", strings.Replace(strings.Join(strings.Split(dest[len(dest)-1], "/"), ""), ":", "-", -1)) + restore := p.Podman([]string{"load", "-q", "-i", destName}) + restore.Wait(90) + return nil +} diff --git a/vendor.conf b/vendor.conf index 61b65f6eb..fe7adf6c9 100644 --- a/vendor.conf +++ b/vendor.conf @@ -16,7 +16,7 @@ github.com/containerd/continuity 004b46473808b3e7a4a3049c20e4376c91eb966d github.com/containernetworking/cni v0.7.0-alpha1 github.com/containernetworking/plugins v0.7.4 github.com/containers/image v1.3 -github.com/containers/storage v1.6 +github.com/containers/storage v1.7 github.com/containers/psgo v1.1 github.com/coreos/go-systemd v14 github.com/cri-o/ocicni 2d2983e40c242322a56c22a903785e7f83eb378c diff --git a/vendor/github.com/containers/storage/containers_ffjson.go b/vendor/github.com/containers/storage/containers_ffjson.go index aef6becfe..40b912bb3 100644 --- a/vendor/github.com/containers/storage/containers_ffjson.go +++ b/vendor/github.com/containers/storage/containers_ffjson.go @@ -1,5 +1,5 @@ // Code generated by ffjson <https://github.com/pquerna/ffjson>. DO NOT EDIT. -// source: containers.go +// source: ./containers.go package storage diff --git a/vendor/github.com/containers/storage/drivers/aufs/aufs.go b/vendor/github.com/containers/storage/drivers/aufs/aufs.go index ca69816be..e821bc0c5 100644 --- a/vendor/github.com/containers/storage/drivers/aufs/aufs.go +++ b/vendor/github.com/containers/storage/drivers/aufs/aufs.go @@ -253,6 +253,11 @@ func (a *Driver) AdditionalImageStores() []string { return nil } +// CreateFromTemplate creates a layer with the same contents and parent as another layer. +func (a *Driver) CreateFromTemplate(id, template string, templateIDMappings *idtools.IDMappings, parent string, parentIDMappings *idtools.IDMappings, opts *graphdriver.CreateOpts, readWrite bool) error { + return graphdriver.NaiveCreateFromTemplate(a, id, template, templateIDMappings, parent, parentIDMappings, opts, readWrite) +} + // CreateReadWrite creates a layer that is writable for use as a container // file system. func (a *Driver) CreateReadWrite(id, parent string, opts *graphdriver.CreateOpts) error { diff --git a/vendor/github.com/containers/storage/drivers/btrfs/btrfs.go b/vendor/github.com/containers/storage/drivers/btrfs/btrfs.go index 567cda9d3..30254d9fb 100644 --- a/vendor/github.com/containers/storage/drivers/btrfs/btrfs.go +++ b/vendor/github.com/containers/storage/drivers/btrfs/btrfs.go @@ -490,6 +490,11 @@ func (d *Driver) quotasDirID(id string) string { return path.Join(d.quotasDir(), id) } +// CreateFromTemplate creates a layer with the same contents and parent as another layer. +func (d *Driver) CreateFromTemplate(id, template string, templateIDMappings *idtools.IDMappings, parent string, parentIDMappings *idtools.IDMappings, opts *graphdriver.CreateOpts, readWrite bool) error { + return d.Create(id, template, opts) +} + // CreateReadWrite creates a layer that is writable for use as a container // file system. func (d *Driver) CreateReadWrite(id, parent string, opts *graphdriver.CreateOpts) error { diff --git a/vendor/github.com/containers/storage/drivers/devmapper/driver.go b/vendor/github.com/containers/storage/drivers/devmapper/driver.go index 39a4fbe2c..13677c93a 100644 --- a/vendor/github.com/containers/storage/drivers/devmapper/driver.go +++ b/vendor/github.com/containers/storage/drivers/devmapper/driver.go @@ -123,6 +123,11 @@ func (d *Driver) Cleanup() error { return err } +// CreateFromTemplate creates a layer with the same contents and parent as another layer. +func (d *Driver) CreateFromTemplate(id, template string, templateIDMappings *idtools.IDMappings, parent string, parentIDMappings *idtools.IDMappings, opts *graphdriver.CreateOpts, readWrite bool) error { + return d.Create(id, template, opts) +} + // CreateReadWrite creates a layer that is writable for use as a container // file system. func (d *Driver) CreateReadWrite(id, parent string, opts *graphdriver.CreateOpts) error { diff --git a/vendor/github.com/containers/storage/drivers/driver.go b/vendor/github.com/containers/storage/drivers/driver.go index 476b55160..dda172574 100644 --- a/vendor/github.com/containers/storage/drivers/driver.go +++ b/vendor/github.com/containers/storage/drivers/driver.go @@ -72,6 +72,9 @@ type ProtoDriver interface { // specified id and parent and options passed in opts. Parent // may be "" and opts may be nil. Create(id, parent string, opts *CreateOpts) error + // CreateFromTemplate creates a new filesystem layer with the specified id + // and parent, with contents identical to the specified template layer. + CreateFromTemplate(id, template string, templateIDMappings *idtools.IDMappings, parent string, parentIDMappings *idtools.IDMappings, opts *CreateOpts, readWrite bool) error // Remove attempts to remove the filesystem layer with this id. Remove(id string) error // Get returns the mountpoint for the layered filesystem referred diff --git a/vendor/github.com/containers/storage/drivers/overlay/check.go b/vendor/github.com/containers/storage/drivers/overlay/check.go index 590d517fa..a566e4afd 100644 --- a/vendor/github.com/containers/storage/drivers/overlay/check.go +++ b/vendor/github.com/containers/storage/drivers/overlay/check.go @@ -10,6 +10,8 @@ import ( "path/filepath" "syscall" + "github.com/containers/storage/pkg/ioutils" + "github.com/containers/storage/pkg/mount" "github.com/containers/storage/pkg/system" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -57,10 +59,11 @@ func doesSupportNativeDiff(d, mountOpts string) error { } opts := fmt.Sprintf("lowerdir=%s:%s,upperdir=%s,workdir=%s", path.Join(td, "l2"), path.Join(td, "l1"), path.Join(td, "l3"), path.Join(td, "work")) - if mountOpts != "" { - opts = fmt.Sprintf("%s,%s", opts, mountOpts) + flags, data := mount.ParseOptions(mountOpts) + if data != "" { + opts = fmt.Sprintf("%s,%s", opts, data) } - if err := unix.Mount("overlay", filepath.Join(td, "merged"), "overlay", 0, opts); err != nil { + if err := unix.Mount("overlay", filepath.Join(td, "merged"), "overlay", uintptr(flags), opts); err != nil { return errors.Wrap(err, "failed to mount overlay") } defer func() { @@ -103,3 +106,60 @@ func doesSupportNativeDiff(d, mountOpts string) error { return nil } + +// doesMetacopy checks if the filesystem is going to optimize changes to +// metadata by using nodes marked with an "overlay.metacopy" attribute to avoid +// copying up a file from a lower layer unless/until its contents are being +// modified +func doesMetacopy(d, mountOpts string) (bool, error) { + td, err := ioutil.TempDir(d, "metacopy-check") + if err != nil { + return false, err + } + defer func() { + if err := os.RemoveAll(td); err != nil { + logrus.Warnf("Failed to remove check directory %v: %v", td, err) + } + }() + + // Make directories l1, l2, work, merged + if err := os.MkdirAll(filepath.Join(td, "l1"), 0755); err != nil { + return false, err + } + if err := ioutils.AtomicWriteFile(filepath.Join(td, "l1", "f"), []byte{0xff}, 0700); err != nil { + return false, err + } + if err := os.MkdirAll(filepath.Join(td, "l2"), 0755); err != nil { + return false, err + } + if err := os.Mkdir(filepath.Join(td, "work"), 0755); err != nil { + return false, err + } + if err := os.Mkdir(filepath.Join(td, "merged"), 0755); err != nil { + return false, err + } + // Mount using the mandatory options and configured options + opts := fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", path.Join(td, "l1"), path.Join(td, "l2"), path.Join(td, "work")) + flags, data := mount.ParseOptions(mountOpts) + if data != "" { + opts = fmt.Sprintf("%s,%s", opts, data) + } + if err := unix.Mount("overlay", filepath.Join(td, "merged"), "overlay", uintptr(flags), opts); err != nil { + return false, errors.Wrap(err, "failed to mount overlay for metacopy check") + } + defer func() { + if err := unix.Unmount(filepath.Join(td, "merged"), 0); err != nil { + logrus.Warnf("Failed to unmount check directory %v: %v", filepath.Join(td, "merged"), err) + } + }() + // Make a change that only impacts the inode, and check if the pulled-up copy is marked + // as a metadata-only copy + if err := os.Chmod(filepath.Join(td, "merged", "f"), 0600); err != nil { + return false, errors.Wrap(err, "error changing permissions on file for metacopy check") + } + metacopy, err := system.Lgetxattr(filepath.Join(td, "l2", "f"), "trusted.overlay.metacopy") + if err != nil { + return false, errors.Wrap(err, "metacopy flag was not set on file in upper layer") + } + return metacopy != nil, nil +} diff --git a/vendor/github.com/containers/storage/drivers/overlay/overlay.go b/vendor/github.com/containers/storage/drivers/overlay/overlay.go index 06d58db38..3e8daf23c 100644 --- a/vendor/github.com/containers/storage/drivers/overlay/overlay.go +++ b/vendor/github.com/containers/storage/drivers/overlay/overlay.go @@ -85,13 +85,12 @@ const ( ) type overlayOptions struct { - overrideKernelCheck bool - imageStores []string - quota quota.Quota - mountProgram string - ostreeRepo string - skipMountHome bool - mountOptions string + imageStores []string + quota quota.Quota + mountProgram string + ostreeRepo string + skipMountHome bool + mountOptions string } // Driver contains information about the home directory and the list of active mounts that are created using this driver. @@ -105,6 +104,7 @@ type Driver struct { options overlayOptions naiveDiff graphdriver.DiffDriver supportsDType bool + usingMetacopy bool locker *locker.Locker convert map[string]bool } @@ -174,6 +174,18 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap } } + usingMetacopy, err := doesMetacopy(home, opts.mountOptions) + if err == nil { + if usingMetacopy { + logrus.Debugf("overlay test mount indicated that metacopy is being used") + } else { + logrus.Debugf("overlay test mount indicated that metacopy is not being used") + } + } else { + logrus.Warnf("overlay test mount did not indicate whether or not metacopy is being used: %v", err) + return nil, err + } + if !opts.skipMountHome { if err := mount.MakePrivate(home); err != nil { return nil, err @@ -193,6 +205,7 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap gidMaps: gidMaps, ctr: graphdriver.NewRefCounter(graphdriver.NewFsChecker(graphdriver.FsMagicOverlay)), supportsDType: supportsDType, + usingMetacopy: usingMetacopy, locker: locker.New(), options: *opts, convert: make(map[string]bool), @@ -212,7 +225,7 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap return nil, fmt.Errorf("Storage option overlay.size only supported for backingFS XFS. Found %v", backingFs) } - logrus.Debugf("backingFs=%s, projectQuotaSupported=%v, useNativeDiff=%v", backingFs, projectQuotaSupported, !d.useNaiveDiff()) + logrus.Debugf("backingFs=%s, projectQuotaSupported=%v, useNativeDiff=%v, usingMetacopy=%v", backingFs, projectQuotaSupported, !d.useNaiveDiff(), d.usingMetacopy) return d, nil } @@ -226,12 +239,6 @@ func parseOptions(options []string) (*overlayOptions, error) { } key = strings.ToLower(key) switch key { - case ".override_kernel_check", "overlay.override_kernel_check", "overlay2.override_kernel_check": - logrus.Debugf("overlay: override_kernelcheck=%s", val) - o.overrideKernelCheck, err = strconv.ParseBool(val) - if err != nil { - return nil, err - } case ".mountopt", "overlay.mountopt", "overlay2.mountopt": o.mountOptions = val case ".size", "overlay.size", "overlay2.size": @@ -375,6 +382,7 @@ func (d *Driver) Status() [][2]string { {"Backing Filesystem", backingFs}, {"Supports d_type", strconv.FormatBool(d.supportsDType)}, {"Native Overlay Diff", strconv.FormatBool(!d.useNaiveDiff())}, + {"Using metacopy", strconv.FormatBool(d.usingMetacopy)}, } } @@ -410,6 +418,14 @@ func (d *Driver) Cleanup() error { return mount.Unmount(d.home) } +// CreateFromTemplate creates a layer with the same contents and parent as another layer. +func (d *Driver) CreateFromTemplate(id, template string, templateIDMappings *idtools.IDMappings, parent string, parentIDMappings *idtools.IDMappings, opts *graphdriver.CreateOpts, readWrite bool) error { + if readWrite { + return d.CreateReadWrite(id, template, opts) + } + return d.Create(id, template, opts) +} + // CreateReadWrite creates a layer that is writable for use as a container // file system. func (d *Driver) CreateReadWrite(id, parent string, opts *graphdriver.CreateOpts) error { diff --git a/vendor/github.com/containers/storage/drivers/template.go b/vendor/github.com/containers/storage/drivers/template.go new file mode 100644 index 000000000..dfcbffb83 --- /dev/null +++ b/vendor/github.com/containers/storage/drivers/template.go @@ -0,0 +1,45 @@ +package graphdriver + +import ( + "github.com/sirupsen/logrus" + + "github.com/containers/storage/pkg/idtools" +) + +// TemplateDriver is just barely enough of a driver that we can implement a +// naive version of CreateFromTemplate on top of it. +type TemplateDriver interface { + DiffDriver + CreateReadWrite(id, parent string, opts *CreateOpts) error + Create(id, parent string, opts *CreateOpts) error + Remove(id string) error +} + +// CreateFromTemplate creates a layer with the same contents and parent as +// another layer. Internally, it may even depend on that other layer +// continuing to exist, as if it were actually a child of the child layer. +func NaiveCreateFromTemplate(d TemplateDriver, id, template string, templateIDMappings *idtools.IDMappings, parent string, parentIDMappings *idtools.IDMappings, opts *CreateOpts, readWrite bool) error { + var err error + if readWrite { + err = d.CreateReadWrite(id, parent, opts) + } else { + err = d.Create(id, parent, opts) + } + if err != nil { + return err + } + diff, err := d.Diff(template, templateIDMappings, parent, parentIDMappings, opts.MountLabel) + if err != nil { + if err2 := d.Remove(id); err2 != nil { + logrus.Errorf("error removing layer %q: %v", id, err2) + } + return err + } + if _, err = d.ApplyDiff(id, templateIDMappings, parent, opts.MountLabel, diff); err != nil { + if err2 := d.Remove(id); err2 != nil { + logrus.Errorf("error removing layer %q: %v", id, err2) + } + return err + } + return nil +} diff --git a/vendor/github.com/containers/storage/drivers/vfs/driver.go b/vendor/github.com/containers/storage/drivers/vfs/driver.go index f7f3c75ba..5941ccc17 100644 --- a/vendor/github.com/containers/storage/drivers/vfs/driver.go +++ b/vendor/github.com/containers/storage/drivers/vfs/driver.go @@ -99,6 +99,14 @@ func (d *Driver) Cleanup() error { return nil } +// CreateFromTemplate creates a layer with the same contents and parent as another layer. +func (d *Driver) CreateFromTemplate(id, template string, templateIDMappings *idtools.IDMappings, parent string, parentIDMappings *idtools.IDMappings, opts *graphdriver.CreateOpts, readWrite bool) error { + if readWrite { + return d.CreateReadWrite(id, template, opts) + } + return d.Create(id, template, opts) +} + // CreateReadWrite creates a layer that is writable for use as a container // file system. func (d *Driver) CreateReadWrite(id, parent string, opts *graphdriver.CreateOpts) error { diff --git a/vendor/github.com/containers/storage/drivers/windows/windows.go b/vendor/github.com/containers/storage/drivers/windows/windows.go index c6d86a4ab..c7df1c1fe 100644 --- a/vendor/github.com/containers/storage/drivers/windows/windows.go +++ b/vendor/github.com/containers/storage/drivers/windows/windows.go @@ -185,6 +185,11 @@ func (d *Driver) Exists(id string) bool { return result } +// CreateFromTemplate creates a layer with the same contents and parent as another layer. +func (d *Driver) CreateFromTemplate(id, template string, templateIDMappings *idtools.IDMappings, parent string, parentIDMappings *idtools.IDMappings, opts *graphdriver.CreateOpts, readWrite bool) error { + return graphdriver.NaiveCreateFromTemplate(d, id, template, templateIDMappings, parent, parentIDMappings, opts, readWrite) +} + // CreateReadWrite creates a layer that is writable for use as a container // file system. func (d *Driver) CreateReadWrite(id, parent string, opts *graphdriver.CreateOpts) error { diff --git a/vendor/github.com/containers/storage/drivers/zfs/zfs.go b/vendor/github.com/containers/storage/drivers/zfs/zfs.go index 69b143c54..eaa9e8bc5 100644 --- a/vendor/github.com/containers/storage/drivers/zfs/zfs.go +++ b/vendor/github.com/containers/storage/drivers/zfs/zfs.go @@ -259,6 +259,11 @@ func (d *Driver) mountPath(id string) string { return path.Join(d.options.mountPath, "graph", getMountpoint(id)) } +// CreateFromTemplate creates a layer with the same contents and parent as another layer. +func (d *Driver) CreateFromTemplate(id, template string, templateIDMappings *idtools.IDMappings, parent string, parentIDMappings *idtools.IDMappings, opts *graphdriver.CreateOpts, readWrite bool) error { + return d.Create(id, template, opts) +} + // CreateReadWrite creates a layer that is writable for use as a container // file system. func (d *Driver) CreateReadWrite(id, parent string, opts *graphdriver.CreateOpts) error { diff --git a/vendor/github.com/containers/storage/images.go b/vendor/github.com/containers/storage/images.go index b10501b08..d99842534 100644 --- a/vendor/github.com/containers/storage/images.go +++ b/vendor/github.com/containers/storage/images.go @@ -42,7 +42,9 @@ type Image struct { // MappedTopLayers are the IDs of alternate versions of the top layer // which have the same contents and parent, and which differ from - // TopLayer only in which ID mappings they use. + // TopLayer only in which ID mappings they use. When the image is + // to be removed, they should be removed before the TopLayer, as the + // graph driver may depend on that. MappedTopLayers []string `json:"mapped-layers,omitempty"` // Metadata is data we keep for the convenience of the caller. It is not diff --git a/vendor/github.com/containers/storage/images_ffjson.go b/vendor/github.com/containers/storage/images_ffjson.go index 6b40ebd59..539acfe93 100644 --- a/vendor/github.com/containers/storage/images_ffjson.go +++ b/vendor/github.com/containers/storage/images_ffjson.go @@ -1,5 +1,5 @@ // Code generated by ffjson <https://github.com/pquerna/ffjson>. DO NOT EDIT. -// source: images.go +// source: ./images.go package storage diff --git a/vendor/github.com/containers/storage/layers.go b/vendor/github.com/containers/storage/layers.go index 299d2f818..cdc3cbba9 100644 --- a/vendor/github.com/containers/storage/layers.go +++ b/vendor/github.com/containers/storage/layers.go @@ -551,9 +551,20 @@ func (r *layerStore) Put(id string, parentLayer *Layer, names []string, mountLab } } parent := "" - var parentMappings *idtools.IDMappings if parentLayer != nil { parent = parentLayer.ID + } + var parentMappings, templateIDMappings, oldMappings *idtools.IDMappings + if moreOptions.TemplateLayer != "" { + templateLayer, ok := r.lookup(moreOptions.TemplateLayer) + if !ok { + return nil, -1, ErrLayerUnknown + } + templateIDMappings = idtools.NewIDMappingsFromMaps(templateLayer.UIDMap, templateLayer.GIDMap) + } else { + templateIDMappings = &idtools.IDMappings{} + } + if parentLayer != nil { parentMappings = idtools.NewIDMappingsFromMaps(parentLayer.UIDMap, parentLayer.GIDMap) } else { parentMappings = &idtools.IDMappings{} @@ -566,23 +577,34 @@ func (r *layerStore) Put(id string, parentLayer *Layer, names []string, mountLab MountLabel: mountLabel, StorageOpt: options, } - if writeable { - if err = r.driver.CreateReadWrite(id, parent, &opts); err != nil { + if moreOptions.TemplateLayer != "" { + if err = r.driver.CreateFromTemplate(id, moreOptions.TemplateLayer, templateIDMappings, parent, parentMappings, &opts, writeable); err != nil { if id != "" { - return nil, -1, errors.Wrapf(err, "error creating read-write layer with ID %q", id) + return nil, -1, errors.Wrapf(err, "error creating copy of template layer %q with ID %q", moreOptions.TemplateLayer, id) } - return nil, -1, errors.Wrapf(err, "error creating read-write layer") + return nil, -1, errors.Wrapf(err, "error creating copy of template layer %q", moreOptions.TemplateLayer) } + oldMappings = templateIDMappings } else { - if err = r.driver.Create(id, parent, &opts); err != nil { - if id != "" { - return nil, -1, errors.Wrapf(err, "error creating layer with ID %q", id) + if writeable { + if err = r.driver.CreateReadWrite(id, parent, &opts); err != nil { + if id != "" { + return nil, -1, errors.Wrapf(err, "error creating read-write layer with ID %q", id) + } + return nil, -1, errors.Wrapf(err, "error creating read-write layer") + } + } else { + if err = r.driver.Create(id, parent, &opts); err != nil { + if id != "" { + return nil, -1, errors.Wrapf(err, "error creating layer with ID %q", id) + } + return nil, -1, errors.Wrapf(err, "error creating layer") } - return nil, -1, errors.Wrapf(err, "error creating layer") } + oldMappings = parentMappings } - if !reflect.DeepEqual(parentMappings.UIDs(), idMappings.UIDs()) || !reflect.DeepEqual(parentMappings.GIDs(), idMappings.GIDs()) { - if err = r.driver.UpdateLayerIDMap(id, parentMappings, idMappings, mountLabel); err != nil { + if !reflect.DeepEqual(oldMappings.UIDs(), idMappings.UIDs()) || !reflect.DeepEqual(oldMappings.GIDs(), idMappings.GIDs()) { + if err = r.driver.UpdateLayerIDMap(id, oldMappings, idMappings, mountLabel); err != nil { // We don't have a record of this layer, but at least // try to clean it up underneath us. r.driver.Remove(id) diff --git a/vendor/github.com/containers/storage/layers_ffjson.go b/vendor/github.com/containers/storage/layers_ffjson.go index 125b5d8c9..09b5d0f33 100644 --- a/vendor/github.com/containers/storage/layers_ffjson.go +++ b/vendor/github.com/containers/storage/layers_ffjson.go @@ -1,5 +1,5 @@ // Code generated by ffjson <https://github.com/pquerna/ffjson>. DO NOT EDIT. -// source: layers.go +// source: ./layers.go package storage diff --git a/vendor/github.com/containers/storage/pkg/archive/example_changes.go b/vendor/github.com/containers/storage/pkg/archive/example_changes.go new file mode 100644 index 000000000..70f9c5564 --- /dev/null +++ b/vendor/github.com/containers/storage/pkg/archive/example_changes.go @@ -0,0 +1,97 @@ +// +build ignore + +// Simple tool to create an archive stream from an old and new directory +// +// By default it will stream the comparison of two temporary directories with junk files +package main + +import ( + "flag" + "fmt" + "io" + "io/ioutil" + "os" + "path" + + "github.com/containers/storage/pkg/archive" + "github.com/sirupsen/logrus" +) + +var ( + flDebug = flag.Bool("D", false, "debugging output") + flNewDir = flag.String("newdir", "", "") + flOldDir = flag.String("olddir", "", "") + log = logrus.New() +) + +func main() { + flag.Usage = func() { + fmt.Println("Produce a tar from comparing two directory paths. By default a demo tar is created of around 200 files (including hardlinks)") + fmt.Printf("%s [OPTIONS]\n", os.Args[0]) + flag.PrintDefaults() + } + flag.Parse() + log.Out = os.Stderr + if (len(os.Getenv("DEBUG")) > 0) || *flDebug { + logrus.SetLevel(logrus.DebugLevel) + } + var newDir, oldDir string + + if len(*flNewDir) == 0 { + var err error + newDir, err = ioutil.TempDir("", "storage-test-newDir") + if err != nil { + log.Fatal(err) + } + defer os.RemoveAll(newDir) + if _, err := prepareUntarSourceDirectory(100, newDir, true); err != nil { + log.Fatal(err) + } + } else { + newDir = *flNewDir + } + + if len(*flOldDir) == 0 { + oldDir, err := ioutil.TempDir("", "storage-test-oldDir") + if err != nil { + log.Fatal(err) + } + defer os.RemoveAll(oldDir) + } else { + oldDir = *flOldDir + } + + changes, err := archive.ChangesDirs(newDir, oldDir) + if err != nil { + log.Fatal(err) + } + + a, err := archive.ExportChanges(newDir, changes) + if err != nil { + log.Fatal(err) + } + defer a.Close() + + i, err := io.Copy(os.Stdout, a) + if err != nil && err != io.EOF { + log.Fatal(err) + } + fmt.Fprintf(os.Stderr, "wrote archive of %d bytes", i) +} + +func prepareUntarSourceDirectory(numberOfFiles int, targetPath string, makeLinks bool) (int, error) { + fileData := []byte("fooo") + for n := 0; n < numberOfFiles; n++ { + fileName := fmt.Sprintf("file-%d", n) + if err := ioutil.WriteFile(path.Join(targetPath, fileName), fileData, 0700); err != nil { + return 0, err + } + if makeLinks { + if err := os.Link(path.Join(targetPath, fileName), path.Join(targetPath, fileName+"-link")); err != nil { + return 0, err + } + } + } + totalSize := numberOfFiles * len(fileData) + return totalSize, nil +} diff --git a/vendor/github.com/containers/storage/store.go b/vendor/github.com/containers/storage/store.go index 5877c3b06..3fe305cc1 100644 --- a/vendor/github.com/containers/storage/store.go +++ b/vendor/github.com/containers/storage/store.go @@ -482,6 +482,10 @@ type LayerOptions struct { // inherit settings from its parent layer or, if it has no parent // layer, the Store object. IDMappingOptions + // TemplateLayer is the ID of a layer whose contents will be used to + // initialize this layer. If set, it should be a child of the layer + // which we want to use as the parent of the new layer. + TemplateLayer string } // ImageOptions is used for passing options to a Store's CreateImage() method. @@ -973,7 +977,7 @@ func (s *store) CreateImage(id string, names []string, layer, metadata string, o return ristore.Create(id, names, layer, metadata, creationDate, options.Digest) } -func (s *store) imageTopLayerForMapping(image *Image, ristore ROImageStore, readWrite bool, rlstore LayerStore, lstores []ROLayerStore, options IDMappingOptions) (*Layer, error) { +func (s *store) imageTopLayerForMapping(image *Image, ristore ROImageStore, createMappedLayer bool, rlstore LayerStore, lstores []ROLayerStore, options IDMappingOptions) (*Layer, error) { layerMatchesMappingOptions := func(layer *Layer, options IDMappingOptions) bool { // If the driver supports shifting and the layer has no mappings, we can use it. if s.graphDriver.SupportsShifting() && len(layer.UIDMap) == 0 && len(layer.GIDMap) == 0 { @@ -994,7 +998,6 @@ func (s *store) imageTopLayerForMapping(image *Image, ristore ROImageStore, read return reflect.DeepEqual(layer.UIDMap, options.UIDMap) && reflect.DeepEqual(layer.GIDMap, options.GIDMap) } var layer, parentLayer *Layer - var layerHomeStore ROLayerStore // Locate the image's top layer and its parent, if it has one. for _, store := range append([]ROLayerStore{rlstore}, lstores...) { if store != rlstore { @@ -1027,7 +1030,6 @@ func (s *store) imageTopLayerForMapping(image *Image, ristore ROImageStore, read if layer == nil { layer = cLayer parentLayer = cParentLayer - layerHomeStore = store } } } @@ -1037,27 +1039,25 @@ func (s *store) imageTopLayerForMapping(image *Image, ristore ROImageStore, read } // The top layer's mappings don't match the ones we want, but it's in a read-only // image store, so we can't create and add a mapped copy of the layer to the image. - if !readWrite { + // We'll have to do the mapping for the container itself, elsewhere. + if !createMappedLayer { return layer, nil } // The top layer's mappings don't match the ones we want, and it's in an image store // that lets us edit image metadata... if istore, ok := ristore.(*imageStore); ok { - // ... so extract the layer's contents, create a new copy of it with the - // desired mappings, and register it as an alternate top layer in the image. - noCompression := archive.Uncompressed - diffOptions := DiffOptions{ - Compression: &noCompression, - } - rc, err := layerHomeStore.Diff("", layer.ID, &diffOptions) - if err != nil { - return nil, errors.Wrapf(err, "error reading layer %q to create an ID-mapped version of it", layer.ID) - } - defer rc.Close() - + // ... so create a duplicate of the layer with the desired mappings, and + // register it as an alternate top layer in the image. var layerOptions LayerOptions if s.graphDriver.SupportsShifting() { - layerOptions = LayerOptions{IDMappingOptions: IDMappingOptions{HostUIDMapping: true, HostGIDMapping: true, UIDMap: nil, GIDMap: nil}} + layerOptions = LayerOptions{ + IDMappingOptions: IDMappingOptions{ + HostUIDMapping: true, + HostGIDMapping: true, + UIDMap: nil, + GIDMap: nil, + }, + } } else { layerOptions = LayerOptions{ IDMappingOptions: IDMappingOptions{ @@ -1068,9 +1068,10 @@ func (s *store) imageTopLayerForMapping(image *Image, ristore ROImageStore, read }, } } - mappedLayer, _, err := rlstore.Put("", parentLayer, nil, layer.MountLabel, nil, &layerOptions, false, nil, rc) + layerOptions.TemplateLayer = layer.ID + mappedLayer, _, err := rlstore.Put("", parentLayer, nil, layer.MountLabel, nil, &layerOptions, false, nil, nil) if err != nil { - return nil, errors.Wrapf(err, "error creating ID-mapped copy of layer %q", layer.ID) + return nil, errors.Wrapf(err, "error creating an ID-mapped copy of layer %q", layer.ID) } if err = istore.addMappedTopLayer(image.ID, mappedLayer.ID); err != nil { if err2 := rlstore.Delete(mappedLayer.ID); err2 != nil { @@ -1144,7 +1145,9 @@ func (s *store) CreateContainer(id string, names []string, image, layer, metadat } imageID = cimage.ID - ilayer, err := s.imageTopLayerForMapping(cimage, imageHomeStore, imageHomeStore == istore, rlstore, lstores, idMappingsOptions) + createMappedLayer := imageHomeStore == istore + + ilayer, err := s.imageTopLayerForMapping(cimage, imageHomeStore, createMappedLayer, rlstore, lstores, idMappingsOptions) if err != nil { return nil, err } @@ -1170,7 +1173,14 @@ func (s *store) CreateContainer(id string, names []string, image, layer, metadat } var layerOptions *LayerOptions if s.graphDriver.SupportsShifting() { - layerOptions = &LayerOptions{IDMappingOptions: IDMappingOptions{HostUIDMapping: true, HostGIDMapping: true, UIDMap: nil, GIDMap: nil}} + layerOptions = &LayerOptions{ + IDMappingOptions: IDMappingOptions{ + HostUIDMapping: true, + HostGIDMapping: true, + UIDMap: nil, + GIDMap: nil, + }, + } } else { layerOptions = &LayerOptions{ IDMappingOptions: IDMappingOptions{ @@ -2091,10 +2101,10 @@ func (s *store) DeleteImage(id string, commit bool) (layers []string, err error) break } lastRemoved = layer - layersToRemove = append(layersToRemove, lastRemoved) if layer == image.TopLayer { layersToRemove = append(layersToRemove, image.MappedTopLayers...) } + layersToRemove = append(layersToRemove, lastRemoved) layer = parent } } else { @@ -3064,9 +3074,6 @@ type OptionsConfig struct { // Size Size string `toml:"size"` - // OverrideKernelCheck - OverrideKernelCheck string `toml:"override_kernel_check"` - // RemapUIDs is a list of default UID mappings to use for layers. RemapUIDs string `toml:"remap-uids"` // RemapGIDs is a list of default GID mappings to use for layers. @@ -3191,9 +3198,6 @@ func ReloadConfigurationFile(configFile string, storeOptions *StoreOptions) { if config.Storage.Options.MountOpt != "" { storeOptions.GraphDriverOptions = append(storeOptions.GraphDriverOptions, fmt.Sprintf("%s.mountopt=%s", config.Storage.Driver, config.Storage.Options.MountOpt)) } - if config.Storage.Options.OverrideKernelCheck != "" { - storeOptions.GraphDriverOptions = append(storeOptions.GraphDriverOptions, fmt.Sprintf("%s.override_kernel_check=%s", config.Storage.Driver, config.Storage.Options.OverrideKernelCheck)) - } if config.Storage.Options.RemapUser != "" && config.Storage.Options.RemapGroup == "" { config.Storage.Options.RemapGroup = config.Storage.Options.RemapUser } |