From 1307e96d5e209ce139527c01a56da57d75b66fd9 Mon Sep 17 00:00:00 2001 From: Peter Hunt Date: Mon, 1 Jul 2019 16:07:25 -0400 Subject: Configure container volumes for generate kube Signed-off-by: Peter Hunt --- libpod/kube.go | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) (limited to 'libpod') diff --git a/libpod/kube.go b/libpod/kube.go index 1622246d5..d22df1ee7 100644 --- a/libpod/kube.go +++ b/libpod/kube.go @@ -328,9 +328,14 @@ func libpodMaxAndMinToResourceList(c *Container) (v1.ResourceList, v1.ResourceLi func generateKubeVolumeMount(hostSourcePath string, mounts []specs.Mount) (v1.VolumeMount, error) { vm := v1.VolumeMount{} for _, m := range mounts { - if m.Source == hostSourcePath { + fmt.Println(m.Destination, hostSourcePath) + if m.Destination == hostSourcePath { // TODO Name is not provided and is required by Kube; therefore, this is disabled earlier - //vm.Name = + name, err := convertVolumePathToName(m.Source) + if err != nil { + return vm, err + } + vm.Name = name vm.MountPath = m.Source vm.SubPath = m.Destination if util.StringInSlice("ro", m.Options) { @@ -342,6 +347,22 @@ func generateKubeVolumeMount(hostSourcePath string, mounts []specs.Mount) (v1.Vo return vm, errors.New("unable to find mount source") } +func convertVolumePathToName(hostSourcePath string) (string, error) { + if len(hostSourcePath) == 0 { + return "", errors.Errorf("hostSourcePath must be specified to generate volume name") + } + if len(hostSourcePath) == 1 { + if hostSourcePath != "/" { + return "", errors.Errorf("hostSourcePath malformatted: %s", hostSourcePath) + } + // add special case name + return "root", nil + } + // First, trim trailing slashes, then replace slashes with dashes. + // Thus, /mnt/data/ will become mnt-data + return strings.Replace(strings.Trim(hostSourcePath, "/"), "/", "-", -1), nil +} + // libpodMountsToKubeVolumeMounts converts the containers mounts to a struct kube understands func libpodMountsToKubeVolumeMounts(c *Container) ([]v1.VolumeMount, error) { // At this point, I dont think we can distinguish between the default @@ -350,7 +371,7 @@ func libpodMountsToKubeVolumeMounts(c *Container) ([]v1.VolumeMount, error) { for _, hostSourcePath := range c.config.UserVolumes { vm, err := generateKubeVolumeMount(hostSourcePath, c.config.Spec.Mounts) if err != nil { - continue + return vms, err } vms = append(vms, vm) } -- cgit v1.2.3-54-g00ecf From aa9de67452af4cf0fdac57ec7657f0184cb0b6fd Mon Sep 17 00:00:00 2001 From: Peter Hunt Date: Mon, 1 Jul 2019 16:58:53 -0400 Subject: Pass along volumes to pod yaml Signed-off-by: Peter Hunt --- libpod/kube.go | 88 +++++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 59 insertions(+), 29 deletions(-) (limited to 'libpod') diff --git a/libpod/kube.go b/libpod/kube.go index d22df1ee7..01c70ced8 100644 --- a/libpod/kube.go +++ b/libpod/kube.go @@ -3,6 +3,7 @@ package libpod import ( "fmt" "math/rand" + "os" "strconv" "strings" "time" @@ -131,11 +132,12 @@ func containersToServicePorts(containers []v1.Container) []v1.ServicePort { func (p *Pod) podWithContainers(containers []*Container, ports []v1.ContainerPort) (*v1.Pod, error) { var ( podContainers []v1.Container + podVolumes []v1.Volume ) first := true for _, ctr := range containers { if !ctr.IsInfra() { - result, err := containerToV1Container(ctr) + result, volumes, err := containerToV1Container(ctr) if err != nil { return nil, err } @@ -152,12 +154,13 @@ func (p *Pod) podWithContainers(containers []*Container, ports []v1.ContainerPor first = false } podContainers = append(podContainers, result) + podVolumes = append(podVolumes, volumes...) } } - return addContainersToPodObject(podContainers, p.Name()), nil + return addContainersAndVolumesToPodObject(podContainers, podVolumes, p.Name()), nil } -func addContainersToPodObject(containers []v1.Container, podName string) *v1.Pod { +func addContainersAndVolumesToPodObject(containers []v1.Container, volumes []v1.Volume, podName string) *v1.Pod { tm := v12.TypeMeta{ Kind: "Pod", APIVersion: "v1", @@ -177,6 +180,7 @@ func addContainersToPodObject(containers []v1.Container, podName string) *v1.Pod } ps := v1.PodSpec{ Containers: containers, + Volumes: volumes, } p := v1.Pod{ TypeMeta: tm, @@ -190,56 +194,58 @@ func addContainersToPodObject(containers []v1.Container, podName string) *v1.Pod // for a single container. we "insert" that container description in a pod. func simplePodWithV1Container(ctr *Container) (*v1.Pod, error) { var containers []v1.Container - result, err := containerToV1Container(ctr) + kubeCtr, kubeVols, err := containerToV1Container(ctr) if err != nil { return nil, err } - containers = append(containers, result) - return addContainersToPodObject(containers, ctr.Name()), nil + containers = append(containers, kubeCtr) + return addContainersAndVolumesToPodObject(containers, kubeVols, ctr.Name()), nil } // containerToV1Container converts information we know about a libpod container // to a V1.Container specification. -func containerToV1Container(c *Container) (v1.Container, error) { +func containerToV1Container(c *Container) (v1.Container, []v1.Volume, error) { kubeContainer := v1.Container{} + kubeVolumes := []v1.Volume{} kubeSec, err := generateKubeSecurityContext(c) if err != nil { - return kubeContainer, err + return kubeContainer, kubeVolumes, err } if len(c.config.Spec.Linux.Devices) > 0 { // TODO Enable when we can support devices and their names devices, err := generateKubeVolumeDeviceFromLinuxDevice(c.Spec().Linux.Devices) if err != nil { - return kubeContainer, err + return kubeContainer, kubeVolumes, err } kubeContainer.VolumeDevices = devices - return kubeContainer, errors.Wrapf(define.ErrNotImplemented, "linux devices") + return kubeContainer, kubeVolumes, errors.Wrapf(define.ErrNotImplemented, "linux devices") } if len(c.config.UserVolumes) > 0 { // TODO When we until we can resolve what the volume name should be, this is disabled // Volume names need to be coordinated "globally" in the kube files. - volumes, err := libpodMountsToKubeVolumeMounts(c) + volumeMounts, volumes, err := libpodMountsToKubeVolumeMounts(c) if err != nil { - return kubeContainer, err + return kubeContainer, kubeVolumes, err } - kubeContainer.VolumeMounts = volumes + kubeContainer.VolumeMounts = volumeMounts + kubeVolumes = append(kubeVolumes, volumes...) } envVariables, err := libpodEnvVarsToKubeEnvVars(c.config.Spec.Process.Env) if err != nil { - return kubeContainer, err + return kubeContainer, kubeVolumes, err } portmappings, err := c.PortMappings() if err != nil { - return kubeContainer, err + return kubeContainer, kubeVolumes, err } ports, err := ocicniPortMappingToContainerPort(portmappings) if err != nil { - return kubeContainer, err + return kubeContainer, kubeVolumes, err } containerCommands := c.Command() @@ -263,7 +269,7 @@ func containerToV1Container(c *Container) (v1.Container, error) { kubeContainer.StdinOnce = false kubeContainer.TTY = c.config.Spec.Process.Terminal - return kubeContainer, nil + return kubeContainer, kubeVolumes, nil } // ocicniPortMappingToContainerPort takes an ocicni portmapping and converts @@ -325,26 +331,48 @@ func libpodMaxAndMinToResourceList(c *Container) (v1.ResourceList, v1.ResourceLi return maxResources, minResources } -func generateKubeVolumeMount(hostSourcePath string, mounts []specs.Mount) (v1.VolumeMount, error) { +func generateKubeVolumeMount(hostSourcePath string, mounts []specs.Mount) (v1.VolumeMount, v1.Volume, error) { vm := v1.VolumeMount{} + vo := v1.Volume{} for _, m := range mounts { - fmt.Println(m.Destination, hostSourcePath) if m.Destination == hostSourcePath { - // TODO Name is not provided and is required by Kube; therefore, this is disabled earlier name, err := convertVolumePathToName(m.Source) if err != nil { - return vm, err + return vm, vo, err } vm.Name = name - vm.MountPath = m.Source - vm.SubPath = m.Destination + vm.MountPath = m.Destination if util.StringInSlice("ro", m.Options) { vm.ReadOnly = true } - return vm, nil + + vo.Name = name + vo.HostPath = &v1.HostPathVolumeSource{} + vo.HostPath.Path = m.Source + isDir, err := isHostPathDirectory(m.Source) + // neither a directory or a file lives here, default to creating a directory + // TODO should this be an error instead? + var hostPathType v1.HostPathType + if err != nil { + hostPathType = v1.HostPathDirectoryOrCreate + } else if isDir { + hostPathType = v1.HostPathDirectory + } else { + hostPathType = v1.HostPathFile + } + vo.HostPath.Type = &hostPathType + return vm, vo, nil } } - return vm, errors.New("unable to find mount source") + return vm, vo, errors.New("unable to find mount source") +} + +func isHostPathDirectory(hostPathSource string) (bool, error) { + info, err := os.Stat(hostPathSource) + if err != nil { + return false, err + } + return info.Mode().IsDir(), nil } func convertVolumePathToName(hostSourcePath string) (string, error) { @@ -364,18 +392,20 @@ func convertVolumePathToName(hostSourcePath string) (string, error) { } // libpodMountsToKubeVolumeMounts converts the containers mounts to a struct kube understands -func libpodMountsToKubeVolumeMounts(c *Container) ([]v1.VolumeMount, error) { +func libpodMountsToKubeVolumeMounts(c *Container) ([]v1.VolumeMount, []v1.Volume, error) { // At this point, I dont think we can distinguish between the default // volume mounts and user added ones. For now, we pass them all. var vms []v1.VolumeMount + var vos []v1.Volume for _, hostSourcePath := range c.config.UserVolumes { - vm, err := generateKubeVolumeMount(hostSourcePath, c.config.Spec.Mounts) + vm, vo, err := generateKubeVolumeMount(hostSourcePath, c.config.Spec.Mounts) if err != nil { - return vms, err + return vms, vos, err } vms = append(vms, vm) + vos = append(vos, vo) } - return vms, nil + return vms, vos, nil } func determineCapAddDropFromCapabilities(defaultCaps, containerCaps []string) *v1.Capabilities { -- cgit v1.2.3-54-g00ecf From db2cc360334ec04e4c2c0a71335fd51540a1273f Mon Sep 17 00:00:00 2001 From: Peter Hunt Date: Tue, 2 Jul 2019 09:21:39 -0400 Subject: Deduplicate volumes for containers that share volumes, so the pod section doesn't list copies Signed-off-by: Peter Hunt --- libpod/kube.go | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) (limited to 'libpod') diff --git a/libpod/kube.go b/libpod/kube.go index 01c70ced8..6a52ff80b 100644 --- a/libpod/kube.go +++ b/libpod/kube.go @@ -132,31 +132,40 @@ func containersToServicePorts(containers []v1.Container) []v1.ServicePort { func (p *Pod) podWithContainers(containers []*Container, ports []v1.ContainerPort) (*v1.Pod, error) { var ( podContainers []v1.Container - podVolumes []v1.Volume ) + deDupPodVolumes := make(map[string]*v1.Volume) first := true for _, ctr := range containers { if !ctr.IsInfra() { - result, volumes, err := containerToV1Container(ctr) + ctr, volumes, err := containerToV1Container(ctr) if err != nil { return nil, err } // Since port bindings for the pod are handled by the // infra container, wipe them here. - result.Ports = nil + ctr.Ports = nil // We add the original port declarations from the libpod infra container // to the first kubernetes container description because otherwise we loose // the original container/port bindings. if first && len(ports) > 0 { - result.Ports = ports + ctr.Ports = ports first = false } - podContainers = append(podContainers, result) - podVolumes = append(podVolumes, volumes...) + podContainers = append(podContainers, ctr) + // Deduplicate volumes, so if containers in the pod share a volume, it's only + // listed in the volumes section once + for _, vol := range volumes { + deDupPodVolumes[vol.Name] = &vol + } } } + podVolumes := make([]v1.Volume, 0, len(deDupPodVolumes)) + for _, vol := range deDupPodVolumes { + podVolumes = append(podVolumes, *vol) + } + return addContainersAndVolumesToPodObject(podContainers, podVolumes, p.Name()), nil } -- cgit v1.2.3-54-g00ecf From aeabc45ccef10fc858677765675bd86944a93db6 Mon Sep 17 00:00:00 2001 From: Peter Hunt Date: Tue, 2 Jul 2019 11:40:14 -0400 Subject: Improve parsing of mounts Specifically, we were needlessly doing a double lookup to find which config mounts were user volumes. Improve this by refactoring a bit of code from inspect Signed-off-by: Peter Hunt --- libpod/container_inspect.go | 83 +++++++++++++++----------------------- libpod/container_internal.go | 31 +++++++++++++++ libpod/kube.go | 95 ++++++++++++++++++++++---------------------- 3 files changed, 112 insertions(+), 97 deletions(-) (limited to 'libpod') diff --git a/libpod/container_inspect.go b/libpod/container_inspect.go index 6085f1210..0a4d7016f 100644 --- a/libpod/container_inspect.go +++ b/libpod/container_inspect.go @@ -368,58 +368,41 @@ func (c *Container) getInspectMounts(ctrSpec *spec.Spec) ([]InspectMount, error) return inspectMounts, nil } - // We need to parse all named volumes and mounts into maps, so we don't - // end up with repeated lookups for each user volume. - // Map destination to struct, as destination is what is stored in - // UserVolumes. - namedVolumes := make(map[string]*ContainerNamedVolume) - mounts := make(map[string]spec.Mount) - for _, namedVol := range c.config.NamedVolumes { - namedVolumes[namedVol.Dest] = namedVol - } - for _, mount := range ctrSpec.Mounts { - mounts[mount.Destination] = mount - } + namedVolumes, mounts := c.sortUserVolumes(ctrSpec) + for _, volume := range namedVolumes { + mountStruct := InspectMount{} + mountStruct.Type = "volume" + mountStruct.Destination = volume.Dest + mountStruct.Name = volume.Name + + // For src and driver, we need to look up the named + // volume. + volFromDB, err := c.runtime.state.Volume(volume.Name) + if err != nil { + return nil, errors.Wrapf(err, "error looking up volume %s in container %s config", volume.Name, c.ID()) + } + mountStruct.Driver = volFromDB.Driver() + mountStruct.Source = volFromDB.MountPoint() + + parseMountOptionsForInspect(volume.Options, &mountStruct) - for _, vol := range c.config.UserVolumes { - // We need to look up the volumes. - // First: is it a named volume? - if volume, ok := namedVolumes[vol]; ok { - mountStruct := InspectMount{} - mountStruct.Type = "volume" - mountStruct.Destination = volume.Dest - mountStruct.Name = volume.Name - - // For src and driver, we need to look up the named - // volume. - volFromDB, err := c.runtime.state.Volume(volume.Name) - if err != nil { - return nil, errors.Wrapf(err, "error looking up volume %s in container %s config", volume.Name, c.ID()) - } - mountStruct.Driver = volFromDB.Driver() - mountStruct.Source = volFromDB.MountPoint() - - parseMountOptionsForInspect(volume.Options, &mountStruct) - - inspectMounts = append(inspectMounts, mountStruct) - } else if mount, ok := mounts[vol]; ok { - // It's a mount. - // Is it a tmpfs? If so, discard. - if mount.Type == "tmpfs" { - continue - } - - mountStruct := InspectMount{} - mountStruct.Type = "bind" - mountStruct.Source = mount.Source - mountStruct.Destination = mount.Destination - - parseMountOptionsForInspect(mount.Options, &mountStruct) - - inspectMounts = append(inspectMounts, mountStruct) + inspectMounts = append(inspectMounts, mountStruct) + } + for _, mount := range mounts { + // It's a mount. + // Is it a tmpfs? If so, discard. + if mount.Type == "tmpfs" { + continue } - // We couldn't find a mount. Log a warning. - logrus.Warnf("Could not find mount at destination %q when building inspect output for container %s", vol, c.ID()) + + mountStruct := InspectMount{} + mountStruct.Type = "bind" + mountStruct.Source = mount.Source + mountStruct.Destination = mount.Destination + + parseMountOptionsForInspect(mount.Options, &mountStruct) + + inspectMounts = append(inspectMounts, mountStruct) } return inspectMounts, nil diff --git a/libpod/container_internal.go b/libpod/container_internal.go index 43d2b6e61..884f5a0a9 100644 --- a/libpod/container_internal.go +++ b/libpod/container_internal.go @@ -1534,3 +1534,34 @@ func (c *Container) prepareCheckpointExport() (err error) { return nil } + +// sortUserVolumes sorts the volumes specified for a container +// between named and normal volumes +func (c *Container) sortUserVolumes(ctrSpec *spec.Spec) ([]*ContainerNamedVolume, []spec.Mount) { + namedUserVolumes := []*ContainerNamedVolume{} + userMounts := []spec.Mount{} + + // We need to parse all named volumes and mounts into maps, so we don't + // end up with repeated lookups for each user volume. + // Map destination to struct, as destination is what is stored in + // UserVolumes. + namedVolumes := make(map[string]*ContainerNamedVolume) + mounts := make(map[string]spec.Mount) + for _, namedVol := range c.config.NamedVolumes { + namedVolumes[namedVol.Dest] = namedVol + } + for _, mount := range ctrSpec.Mounts { + mounts[mount.Destination] = mount + } + + for _, vol := range c.config.UserVolumes { + if volume, ok := namedVolumes[vol]; ok { + namedUserVolumes = append(namedUserVolumes, volume) + } else if mount, ok := mounts[vol]; ok { + userMounts = append(userMounts, mount) + } else { + logrus.Warnf("Could not find mount at destination %q when parsing user volumes for container %s", vol, c.ID()) + } + } + return namedUserVolumes, userMounts +} diff --git a/libpod/kube.go b/libpod/kube.go index 6a52ff80b..08ab7a4a4 100644 --- a/libpod/kube.go +++ b/libpod/kube.go @@ -340,40 +340,58 @@ func libpodMaxAndMinToResourceList(c *Container) (v1.ResourceList, v1.ResourceLi return maxResources, minResources } -func generateKubeVolumeMount(hostSourcePath string, mounts []specs.Mount) (v1.VolumeMount, v1.Volume, error) { +// libpodMountsToKubeVolumeMounts converts the containers mounts to a struct kube understands +func libpodMountsToKubeVolumeMounts(c *Container) ([]v1.VolumeMount, []v1.Volume, error) { + var vms []v1.VolumeMount + var vos []v1.Volume + + // TjDO when named volumes are supported in play kube, also parse named volumes here + _, mounts := c.sortUserVolumes(c.config.Spec) + for _, m := range mounts { + vm, vo, err := generateKubeVolumeMount(m) + if err != nil { + return vms, vos, err + } + vms = append(vms, vm) + vos = append(vos, vo) + } + return vms, vos, nil +} + +// generateKubeVolumeMount takes a user specfied mount and returns +// a kubernetes VolumeMount (to be added to the container) and a kubernetes Volume +// (to be added to the pod) +func generateKubeVolumeMount(m specs.Mount) (v1.VolumeMount, v1.Volume, error) { vm := v1.VolumeMount{} vo := v1.Volume{} - for _, m := range mounts { - if m.Destination == hostSourcePath { - name, err := convertVolumePathToName(m.Source) - if err != nil { - return vm, vo, err - } - vm.Name = name - vm.MountPath = m.Destination - if util.StringInSlice("ro", m.Options) { - vm.ReadOnly = true - } - vo.Name = name - vo.HostPath = &v1.HostPathVolumeSource{} - vo.HostPath.Path = m.Source - isDir, err := isHostPathDirectory(m.Source) - // neither a directory or a file lives here, default to creating a directory - // TODO should this be an error instead? - var hostPathType v1.HostPathType - if err != nil { - hostPathType = v1.HostPathDirectoryOrCreate - } else if isDir { - hostPathType = v1.HostPathDirectory - } else { - hostPathType = v1.HostPathFile - } - vo.HostPath.Type = &hostPathType - return vm, vo, nil - } + name, err := convertVolumePathToName(m.Source) + if err != nil { + return vm, vo, err + } + vm.Name = name + vm.MountPath = m.Destination + if util.StringInSlice("ro", m.Options) { + vm.ReadOnly = true + } + + vo.Name = name + vo.HostPath = &v1.HostPathVolumeSource{} + vo.HostPath.Path = m.Source + isDir, err := isHostPathDirectory(m.Source) + // neither a directory or a file lives here, default to creating a directory + // TODO should this be an error instead? + var hostPathType v1.HostPathType + if err != nil { + hostPathType = v1.HostPathDirectoryOrCreate + } else if isDir { + hostPathType = v1.HostPathDirectory + } else { + hostPathType = v1.HostPathFile } - return vm, vo, errors.New("unable to find mount source") + vo.HostPath.Type = &hostPathType + + return vm, vo, nil } func isHostPathDirectory(hostPathSource string) (bool, error) { @@ -400,23 +418,6 @@ func convertVolumePathToName(hostSourcePath string) (string, error) { return strings.Replace(strings.Trim(hostSourcePath, "/"), "/", "-", -1), nil } -// libpodMountsToKubeVolumeMounts converts the containers mounts to a struct kube understands -func libpodMountsToKubeVolumeMounts(c *Container) ([]v1.VolumeMount, []v1.Volume, error) { - // At this point, I dont think we can distinguish between the default - // volume mounts and user added ones. For now, we pass them all. - var vms []v1.VolumeMount - var vos []v1.Volume - for _, hostSourcePath := range c.config.UserVolumes { - vm, vo, err := generateKubeVolumeMount(hostSourcePath, c.config.Spec.Mounts) - if err != nil { - return vms, vos, err - } - vms = append(vms, vm) - vos = append(vos, vo) - } - return vms, vos, nil -} - func determineCapAddDropFromCapabilities(defaultCaps, containerCaps []string) *v1.Capabilities { var ( drop []v1.Capability -- cgit v1.2.3-54-g00ecf