6 files changed, 290 insertions, 71 deletions

diff --git a/pkg/domain/infra/abi/containers.go b/pkg/domain/infra/abi/containers.go
index 637531ee9..24261e5ed 100644
--- a/pkg/domain/infra/abi/containers.go
+++ b/pkg/domain/infra/abi/containers.go
@@ -194,7 +194,7 @@ func (ic *ContainerEngine) ContainerStop(ctx context.Context, namesOrIds []strin
 func (ic *ContainerEngine) ContainerPrune(ctx context.Context, options entities.ContainerPruneOptions) ([]*reports.PruneReport, error) {
 	filterFuncs := make([]libpod.ContainerFilter, 0, len(options.Filters))
 	for k, v := range options.Filters {
-		generatedFunc, err := dfilters.GenerateContainerFilterFuncs(k, v, ic.Libpod)
+		generatedFunc, err := dfilters.GeneratePruneContainerFilterFuncs(k, v, ic.Libpod)
 		if err != nil {
 			return nil, err
 		}
diff --git a/pkg/domain/infra/abi/generate.go b/pkg/domain/infra/abi/generate.go
index 161becbfa..94f649e15 100644
--- a/pkg/domain/infra/abi/generate.go
+++ b/pkg/domain/infra/abi/generate.go
@@ -44,11 +44,10 @@ func (ic *ContainerEngine) GenerateSystemd(ctx context.Context, nameOrID string,
 func (ic *ContainerEngine) GenerateKube(ctx context.Context, nameOrIDs []string, options entities.GenerateKubeOptions) (*entities.GenerateKubeReport, error) {
 	var (
 		pods         []*libpod.Pod
-		podYAML      *k8sAPI.Pod
-		err          error
 		ctrs         []*libpod.Container
-		servicePorts []k8sAPI.ServicePort
-		serviceYAML  k8sAPI.Service
+		kubePods     []*k8sAPI.Pod
+		kubeServices []k8sAPI.Service
+		content      []byte
 	)
 	for _, nameOrID := range nameOrIDs {
 		// Get the container in question
@@ -59,9 +58,6 @@ func (ic *ContainerEngine) GenerateKube(ctx context.Context, nameOrIDs []string,
 				return nil, err
 			}
 			pods = append(pods, pod)
-			if len(pods) > 1 {
-				return nil, errors.New("can only generate single pod at a time")
-			}
 		} else {
 			if len(ctr.Dependencies()) > 0 {
 				return nil, errors.Wrapf(define.ErrNotImplemented, "containers with dependencies")
@@ -79,20 +75,29 @@ func (ic *ContainerEngine) GenerateKube(ctx context.Context, nameOrIDs []string,
 		return nil, errors.New("cannot generate pods and containers at the same time")
 	}
 
-	if len(pods) == 1 {
-		podYAML, servicePorts, err = pods[0].GenerateForKube()
+	if len(pods) >= 1 {
+		pos, svcs, err := getKubePods(pods, options.Service)
+		if err != nil {
+			return nil, err
+		}
+
+		kubePods = append(kubePods, pos...)
+		if options.Service {
+			kubeServices = append(kubeServices, svcs...)
+		}
 	} else {
-		podYAML, err = libpod.GenerateForKube(ctrs)
-	}
-	if err != nil {
-		return nil, err
-	}
+		po, err := libpod.GenerateForKube(ctrs)
+		if err != nil {
+			return nil, err
+		}
 
-	if options.Service {
-		serviceYAML = libpod.GenerateKubeServiceFromV1Pod(podYAML, servicePorts)
+		kubePods = append(kubePods, po)
+		if options.Service {
+			kubeServices = append(kubeServices, libpod.GenerateKubeServiceFromV1Pod(po, []k8sAPI.ServicePort{}))
+		}
 	}
 
-	content, err := generateKubeOutput(podYAML, &serviceYAML, options.Service)
+	content, err := generateKubeOutput(kubePods, kubeServices, options.Service)
 	if err != nil {
 		return nil, err
 	}
@@ -100,24 +105,56 @@ func (ic *ContainerEngine) GenerateKube(ctx context.Context, nameOrIDs []string,
 	return &entities.GenerateKubeReport{Reader: bytes.NewReader(content)}, nil
 }
 
-func generateKubeOutput(podYAML *k8sAPI.Pod, serviceYAML *k8sAPI.Service, hasService bool) ([]byte, error) {
-	var (
-		output            []byte
-		marshalledPod     []byte
-		marshalledService []byte
-		err               error
-	)
+func getKubePods(pods []*libpod.Pod, getService bool) ([]*k8sAPI.Pod, []k8sAPI.Service, error) {
+	kubePods := make([]*k8sAPI.Pod, 0)
+	kubeServices := make([]k8sAPI.Service, 0)
 
-	marshalledPod, err = yaml.Marshal(podYAML)
-	if err != nil {
-		return nil, err
+	for _, p := range pods {
+		po, svc, err := p.GenerateForKube()
+		if err != nil {
+			return nil, nil, err
+		}
+
+		kubePods = append(kubePods, po)
+		if getService {
+			kubeServices = append(kubeServices, libpod.GenerateKubeServiceFromV1Pod(po, svc))
+		}
 	}
 
-	if hasService {
-		marshalledService, err = yaml.Marshal(serviceYAML)
+	return kubePods, kubeServices, nil
+}
+
+func generateKubeOutput(kubePods []*k8sAPI.Pod, kubeServices []k8sAPI.Service, hasService bool) ([]byte, error) {
+	output := make([]byte, 0)
+	marshalledPods := make([]byte, 0)
+	marshalledServices := make([]byte, 0)
+
+	for i, p := range kubePods {
+		if i != 0 {
+			marshalledPods = append(marshalledPods, []byte("---\n")...)
+		}
+
+		b, err := yaml.Marshal(p)
 		if err != nil {
 			return nil, err
 		}
+
+		marshalledPods = append(marshalledPods, b...)
+	}
+
+	if hasService {
+		for i, s := range kubeServices {
+			if i != 0 {
+				marshalledServices = append(marshalledServices, []byte("---\n")...)
+			}
+
+			b, err := yaml.Marshal(s)
+			if err != nil {
+				return nil, err
+			}
+
+			marshalledServices = append(marshalledServices, b...)
+		}
 	}
 
 	header := `# Generation of Kubernetes YAML is still under development!
@@ -133,11 +170,12 @@ func generateKubeOutput(podYAML *k8sAPI.Pod, serviceYAML *k8sAPI.Service, hasSer
 	}
 
 	output = append(output, []byte(fmt.Sprintf(header, podmanVersion.Version))...)
-	output = append(output, marshalledPod...)
+	// kube generate order is based on helm install order (service, pod...)
 	if hasService {
+		output = append(output, marshalledServices...)
 		output = append(output, []byte("---\n")...)
-		output = append(output, marshalledService...)
 	}
+	output = append(output, marshalledPods...)
 
 	return output, nil
 }
diff --git a/pkg/domain/infra/abi/play.go b/pkg/domain/infra/abi/play.go
index efc7c86e3..3b5c141d7 100644
--- a/pkg/domain/infra/abi/play.go
+++ b/pkg/domain/infra/abi/play.go
@@ -1,6 +1,7 @@
 package abi
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"io"
@@ -8,6 +9,7 @@ import (
 	"os"
 	"strings"
 
+	"github.com/containers/common/pkg/secrets"
 	"github.com/containers/image/v5/types"
 	"github.com/containers/podman/v3/libpod"
 	"github.com/containers/podman/v3/libpod/define"
@@ -20,46 +22,79 @@ import (
 	"github.com/ghodss/yaml"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
+	yamlv3 "gopkg.in/yaml.v3"
 	v1apps "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
 )
 
 func (ic *ContainerEngine) PlayKube(ctx context.Context, path string, options entities.PlayKubeOptions) (*entities.PlayKubeReport, error) {
-	var (
-		kubeObject v1.ObjectReference
-	)
+	report := &entities.PlayKubeReport{}
+	validKinds := 0
 
+	// read yaml document
 	content, err := ioutil.ReadFile(path)
 	if err != nil {
 		return nil, err
 	}
 
-	if err := yaml.Unmarshal(content, &kubeObject); err != nil {
-		return nil, errors.Wrapf(err, "unable to read %q as YAML", path)
+	// split yaml document
+	documentList, err := splitMultiDocYAML(content)
+	if err != nil {
+		return nil, err
 	}
 
-	// NOTE: pkg/bindings/play is also parsing the file.
-	// A pkg/kube would be nice to refactor and abstract
-	// parts of the K8s-related code.
-	switch kubeObject.Kind {
-	case "Pod":
-		var podYAML v1.Pod
-		var podTemplateSpec v1.PodTemplateSpec
-		if err := yaml.Unmarshal(content, &podYAML); err != nil {
-			return nil, errors.Wrapf(err, "unable to read YAML %q as Kube Pod", path)
+	// create pod on each document if it is a pod or deployment
+	// any other kube kind will be skipped
+	for _, document := range documentList {
+		kind, err := getKubeKind(document)
+		if err != nil {
+			return nil, errors.Wrapf(err, "unable to read %q as kube YAML", path)
 		}
-		podTemplateSpec.ObjectMeta = podYAML.ObjectMeta
-		podTemplateSpec.Spec = podYAML.Spec
-		return ic.playKubePod(ctx, podTemplateSpec.ObjectMeta.Name, &podTemplateSpec, options)
-	case "Deployment":
-		var deploymentYAML v1apps.Deployment
-		if err := yaml.Unmarshal(content, &deploymentYAML); err != nil {
-			return nil, errors.Wrapf(err, "unable to read YAML %q as Kube Deployment", path)
+
+		switch kind {
+		case "Pod":
+			var podYAML v1.Pod
+			var podTemplateSpec v1.PodTemplateSpec
+
+			if err := yaml.Unmarshal(document, &podYAML); err != nil {
+				return nil, errors.Wrapf(err, "unable to read YAML %q as Kube Pod", path)
+			}
+
+			podTemplateSpec.ObjectMeta = podYAML.ObjectMeta
+			podTemplateSpec.Spec = podYAML.Spec
+
+			r, err := ic.playKubePod(ctx, podTemplateSpec.ObjectMeta.Name, &podTemplateSpec, options)
+			if err != nil {
+				return nil, err
+			}
+
+			report.Pods = append(report.Pods, r.Pods...)
+			validKinds++
+		case "Deployment":
+			var deploymentYAML v1apps.Deployment
+
+			if err := yaml.Unmarshal(document, &deploymentYAML); err != nil {
+				return nil, errors.Wrapf(err, "unable to read YAML %q as Kube Deployment", path)
+			}
+
+			r, err := ic.playKubeDeployment(ctx, &deploymentYAML, options)
+			if err != nil {
+				return nil, err
+			}
+
+			report.Pods = append(report.Pods, r.Pods...)
+			validKinds++
+		default:
+			logrus.Infof("kube kind %s not supported", kind)
+			continue
 		}
-		return ic.playKubeDeployment(ctx, &deploymentYAML, options)
-	default:
-		return nil, errors.Errorf("invalid YAML kind: %q. [Pod|Deployment] are the only supported Kubernetes Kinds", kubeObject.Kind)
 	}
+
+	if validKinds == 0 {
+		return nil, fmt.Errorf("YAML document does not contain any supported kube kind")
+	}
+
+	return report, nil
 }
 
 func (ic *ContainerEngine) playKubeDeployment(ctx context.Context, deploymentYAML *v1apps.Deployment, options entities.PlayKubeOptions) (*entities.PlayKubeReport, error) {
@@ -101,6 +136,12 @@ func (ic *ContainerEngine) playKubePod(ctx context.Context, podName string, podY
 		report        entities.PlayKubeReport
 	)
 
+	// Create the secret manager before hand
+	secretsManager, err := secrets.NewManager(ic.Libpod.GetSecretsStorageDir())
+	if err != nil {
+		return nil, err
+	}
+
 	// check for name collision between pod and container
 	if podName == "" {
 		return nil, errors.Errorf("pod does not have a name")
@@ -227,16 +268,17 @@ func (ic *ContainerEngine) playKubePod(ctx context.Context, podName string, podY
 		}
 
 		specgenOpts := kube.CtrSpecGenOptions{
-			Container:     container,
-			Image:         newImage,
-			Volumes:       volumes,
-			PodID:         pod.ID(),
-			PodName:       podName,
-			PodInfraID:    podInfraID,
-			ConfigMaps:    configMaps,
-			SeccompPaths:  seccompPaths,
-			RestartPolicy: ctrRestartPolicy,
-			NetNSIsHost:   p.NetNS.IsHost(),
+			Container:      container,
+			Image:          newImage,
+			Volumes:        volumes,
+			PodID:          pod.ID(),
+			PodName:        podName,
+			PodInfraID:     podInfraID,
+			ConfigMaps:     configMaps,
+			SeccompPaths:   seccompPaths,
+			RestartPolicy:  ctrRestartPolicy,
+			NetNSIsHost:    p.NetNS.IsHost(),
+			SecretsManager: secretsManager,
 		}
 		specGen, err := kube.ToSpecGen(ctx, &specgenOpts)
 		if err != nil {
@@ -290,3 +332,45 @@ func readConfigMapFromFile(r io.Reader) (v1.ConfigMap, error) {
 
 	return cm, nil
 }
+
+// splitMultiDocYAML reads mutiple documents in a YAML file and
+// returns them as a list.
+func splitMultiDocYAML(yamlContent []byte) ([][]byte, error) {
+	var documentList [][]byte
+
+	d := yamlv3.NewDecoder(bytes.NewReader(yamlContent))
+	for {
+		var o interface{}
+		// read individual document
+		err := d.Decode(&o)
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return nil, errors.Wrapf(err, "multi doc yaml could not be split")
+		}
+
+		if o != nil {
+			// back to bytes
+			document, err := yamlv3.Marshal(o)
+			if err != nil {
+				return nil, errors.Wrapf(err, "individual doc yaml could not be marshalled")
+			}
+
+			documentList = append(documentList, document)
+		}
+	}
+
+	return documentList, nil
+}
+
+// getKubeKind unmarshals a kube YAML document and returns its kind.
+func getKubeKind(obj []byte) (string, error) {
+	var kubeObject v1.ObjectReference
+
+	if err := yaml.Unmarshal(obj, &kubeObject); err != nil {
+		return "", err
+	}
+
+	return kubeObject.Kind, nil
+}
diff --git a/pkg/domain/infra/abi/play_test.go b/pkg/domain/infra/abi/play_test.go
index 4354a3835..bbc7c3493 100644
--- a/pkg/domain/infra/abi/play_test.go
+++ b/pkg/domain/infra/abi/play_test.go
@@ -89,3 +89,100 @@ data:
 		})
 	}
 }
+
+func TestGetKubeKind(t *testing.T) {
+	tests := []struct {
+		name             string
+		kubeYAML         string
+		expectError      bool
+		expectedErrorMsg string
+		expected         string
+	}{
+		{
+			"ValidKubeYAML",
+			`
+apiVersion: v1
+kind: Pod
+`,
+			false,
+			"",
+			"Pod",
+		},
+		{
+			"InvalidKubeYAML",
+			"InvalidKubeYAML",
+			true,
+			"cannot unmarshal",
+			"",
+		},
+	}
+
+	for _, test := range tests {
+		test := test
+		t.Run(test.name, func(t *testing.T) {
+			kind, err := getKubeKind([]byte(test.kubeYAML))
+			if test.expectError {
+				assert.Error(t, err)
+				assert.Contains(t, err.Error(), test.expectedErrorMsg)
+			} else {
+				assert.NoError(t, err)
+				assert.Equal(t, test.expected, kind)
+			}
+		})
+	}
+}
+
+func TestSplitMultiDocYAML(t *testing.T) {
+	tests := []struct {
+		name             string
+		kubeYAML         string
+		expectError      bool
+		expectedErrorMsg string
+		expected         int
+	}{
+		{
+			"ValidNumberOfDocs",
+			`
+apiVersion: v1
+kind: Pod
+---
+apiVersion: v1
+kind: Pod
+---
+apiVersion: v1
+kind: Pod
+`,
+			false,
+			"",
+			3,
+		},
+		{
+			"InvalidMultiDocYAML",
+			`
+apiVersion: v1
+kind: Pod
+---
+apiVersion: v1
+kind: Pod
+-
+`,
+			true,
+			"multi doc yaml could not be split",
+			0,
+		},
+	}
+
+	for _, test := range tests {
+		test := test
+		t.Run(test.name, func(t *testing.T) {
+			docs, err := splitMultiDocYAML([]byte(test.kubeYAML))
+			if test.expectError {
+				assert.Error(t, err)
+				assert.Contains(t, err.Error(), test.expectedErrorMsg)
+			} else {
+				assert.NoError(t, err)
+				assert.Equal(t, test.expected, len(docs))
+			}
+		})
+	}
+}
diff --git a/pkg/domain/infra/abi/system.go b/pkg/domain/infra/abi/system.go
index 9f7c8919b..a3e753384 100644
--- a/pkg/domain/infra/abi/system.go
+++ b/pkg/domain/infra/abi/system.go
@@ -67,7 +67,7 @@ func (ic *ContainerEngine) SetupRootless(_ context.Context, cmd *cobra.Command)
 	if os.Geteuid() == 0 {
 		ownsCgroup, err := cgroups.UserOwnsCurrentSystemdCgroup()
 		if err != nil {
-			logrus.Warnf("Failed to detect the owner for the current cgroup: %v", err)
+			logrus.Infof("Failed to detect the owner for the current cgroup: %v", err)
 		}
 		if !ownsCgroup {
 			conf, err := ic.Config(context.Background())
diff --git a/pkg/domain/infra/runtime_libpod.go b/pkg/domain/infra/runtime_libpod.go
index 8b6581c7b..b0d9dc797 100644
--- a/pkg/domain/infra/runtime_libpod.go
+++ b/pkg/domain/infra/runtime_libpod.go
@@ -15,8 +15,8 @@ import (
 	"github.com/containers/podman/v3/pkg/domain/entities"
 	"github.com/containers/podman/v3/pkg/namespaces"
 	"github.com/containers/podman/v3/pkg/rootless"
-	"github.com/containers/storage"
 	"github.com/containers/storage/pkg/idtools"
+	"github.com/containers/storage/types"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	flag "github.com/spf13/pflag"
@@ -100,7 +100,7 @@ func GetRuntimeNoStore(ctx context.Context, fs *flag.FlagSet, cfg *entities.Podm
 
 func getRuntime(ctx context.Context, fs *flag.FlagSet, opts *engineOpts) (*libpod.Runtime, error) {
 	options := []libpod.RuntimeOption{}
-	storageOpts := storage.StoreOptions{}
+	storageOpts := types.StoreOptions{}
 	cfg := opts.config
 
 	storageSet := false
@@ -237,8 +237,8 @@ func getRuntime(ctx context.Context, fs *flag.FlagSet, opts *engineOpts) (*libpo
 }
 
 // ParseIDMapping takes idmappings and subuid and subgid maps and returns a storage mapping
-func ParseIDMapping(mode namespaces.UsernsMode, uidMapSlice, gidMapSlice []string, subUIDMap, subGIDMap string) (*storage.IDMappingOptions, error) {
-	options := storage.IDMappingOptions{
+func ParseIDMapping(mode namespaces.UsernsMode, uidMapSlice, gidMapSlice []string, subUIDMap, subGIDMap string) (*types.IDMappingOptions, error) {
+	options := types.IDMappingOptions{
 		HostUIDMapping: true,
 		HostGIDMapping: true,
 	}