Merge pull request #11102 from cdoern/infraEnhance

InfraContainer Rework

9 files changed, 341 insertions, 165 deletions

diff --git a/pkg/domain/entities/engine_container.go b/pkg/domain/entities/engine_container.go
index 5acf7211c..bd011d309 100644
--- a/pkg/domain/entities/engine_container.go
+++ b/pkg/domain/entities/engine_container.go
@@ -68,7 +68,7 @@ type ContainerEngine interface {
 	NetworkRm(ctx context.Context, namesOrIds []string, options NetworkRmOptions) ([]*NetworkRmReport, error)
 	PlayKube(ctx context.Context, path string, opts PlayKubeOptions) (*PlayKubeReport, error)
 	PlayKubeDown(ctx context.Context, path string, opts PlayKubeDownOptions) (*PlayKubeReport, error)
-	PodCreate(ctx context.Context, opts PodCreateOptions) (*PodCreateReport, error)
+	PodCreate(ctx context.Context, specg PodSpec) (*PodCreateReport, error)
 	PodExists(ctx context.Context, nameOrID string) (*BoolReport, error)
 	PodInspect(ctx context.Context, options PodInspectOptions) (*PodInspectReport, error)
 	PodKill(ctx context.Context, namesOrIds []string, options PodKillOptions) ([]*PodKillReport, error)
diff --git a/pkg/domain/entities/pods.go b/pkg/domain/entities/pods.go
index c66bf96fc..10bd7e5ce 100644
--- a/pkg/domain/entities/pods.go
+++ b/pkg/domain/entities/pods.go
@@ -106,6 +106,14 @@ type PodRmReport struct {
 	Id  string //nolint
 }
 
+// PddSpec is an abstracted version of PodSpecGen designed to eventually accept options
+// not meant to be in a specgen
+type PodSpec struct {
+	PodSpecGen specgen.PodSpecGenerator
+}
+
+// PodCreateOptions provides all possible options for creating a pod and its infra container
+// swagger:model PodCreateOptions
 type PodCreateOptions struct {
 	CGroupParent       string
 	CreateCommand      []string
@@ -125,6 +133,123 @@ type PodCreateOptions struct {
 	Userns             specgen.Namespace
 }
 
+type ContainerCreateOptions struct {
+	Annotation        []string
+	Attach            []string
+	Authfile          string
+	BlkIOWeight       string
+	BlkIOWeightDevice []string
+	CapAdd            []string
+	CapDrop           []string
+	CgroupNS          string
+	CGroupsMode       string
+	CGroupParent      string
+	CIDFile           string
+	ConmonPIDFile     string
+	CPUPeriod         uint64
+	CPUQuota          int64
+	CPURTPeriod       uint64
+	CPURTRuntime      int64
+	CPUShares         uint64
+	CPUS              float64
+	CPUSetCPUs        string
+	CPUSetMems        string
+	Devices           []string
+	DeviceCGroupRule  []string
+	DeviceReadBPs     []string
+	DeviceReadIOPs    []string
+	DeviceWriteBPs    []string
+	DeviceWriteIOPs   []string
+	Entrypoint        *string
+	Env               []string
+	EnvHost           bool
+	EnvFile           []string
+	Expose            []string
+	GIDMap            []string
+	GroupAdd          []string
+	HealthCmd         string
+	HealthInterval    string
+	HealthRetries     uint
+	HealthStartPeriod string
+	HealthTimeout     string
+	Hostname          string
+	HTTPProxy         bool
+	ImageVolume       string
+	Init              bool
+	InitContainerType string
+	InitPath          string
+	Interactive       bool
+	IPC               string
+	KernelMemory      string
+	Label             []string
+	LabelFile         []string
+	LogDriver         string
+	LogOptions        []string
+	Memory            string
+	MemoryReservation string
+	MemorySwap        string
+	MemorySwappiness  int64
+	Name              string
+	NoHealthCheck     bool
+	OOMKillDisable    bool
+	OOMScoreAdj       int
+	Arch              string
+	OS                string
+	Variant           string
+	PID               string
+	PIDsLimit         *int64
+	Platform          string
+	Pod               string
+	PodIDFile         string
+	Personality       string
+	PreserveFDs       uint
+	Privileged        bool
+	PublishAll        bool
+	Pull              string
+	Quiet             bool
+	ReadOnly          bool
+	ReadOnlyTmpFS     bool
+	Restart           string
+	Replace           bool
+	Requires          []string
+	Rm                bool
+	RootFS            bool
+	Secrets           []string
+	SecurityOpt       []string
+	SdNotifyMode      string
+	ShmSize           string
+	SignaturePolicy   string
+	StopSignal        string
+	StopTimeout       uint
+	StorageOpt        []string
+	SubUIDName        string
+	SubGIDName        string
+	Sysctl            []string
+	Systemd           string
+	Timeout           uint
+	TLSVerify         bool
+	TmpFS             []string
+	TTY               bool
+	Timezone          string
+	Umask             string
+	UIDMap            []string
+	Ulimit            []string
+	User              string
+	UserNS            string
+	UTS               string
+	Mount             []string
+	Volume            []string
+	VolumesFrom       []string
+	Workdir           string
+	SeccompPolicy     string
+	PidFile           string
+	IsInfra           bool
+
+	Net *NetOptions
+
+	CgroupConf []string
+}
+
 type PodCreateReport struct {
 	Id string //nolint
 }
@@ -149,21 +274,15 @@ func (p *PodCreateOptions) CPULimits() *specs.LinuxCPU {
 	return cpu
 }
 
-func setNamespaces(p *PodCreateOptions) ([4]specgen.Namespace, error) {
-	allNS := [4]specgen.Namespace{}
-	if p.Pid != "" {
-		pid, err := specgen.ParseNamespace(p.Pid)
-		if err != nil {
-			return [4]specgen.Namespace{}, err
-		}
-		allNS[0] = pid
-	}
-	return allNS, nil
-}
-
-func (p *PodCreateOptions) ToPodSpecGen(s *specgen.PodSpecGenerator) error {
+func ToPodSpecGen(s specgen.PodSpecGenerator, p *PodCreateOptions) (*specgen.PodSpecGenerator, error) {
 	// Basic Config
 	s.Name = p.Name
+	s.InfraName = p.InfraName
+	out, err := specgen.ParseNamespace(p.Pid)
+	if err != nil {
+		return nil, err
+	}
+	s.Pid = out
 	s.Hostname = p.Hostname
 	s.Labels = p.Labels
 	s.NoInfra = !p.Infra
@@ -174,32 +293,26 @@ func (p *PodCreateOptions) ToPodSpecGen(s *specgen.PodSpecGenerator) error {
 		s.InfraConmonPidFile = p.InfraConmonPidFile
 	}
 	s.InfraImage = p.InfraImage
-	s.InfraName = p.InfraName
 	s.SharedNamespaces = p.Share
 	s.PodCreateCommand = p.CreateCommand
 
 	// Networking config
-	s.NetNS = p.Net.Network
-	s.StaticIP = p.Net.StaticIP
-	s.StaticMAC = p.Net.StaticMAC
-	s.PortMappings = p.Net.PublishPorts
-	s.CNINetworks = p.Net.CNINetworks
-	s.NetworkOptions = p.Net.NetworkOptions
-	if p.Net.UseImageResolvConf {
-		s.NoManageResolvConf = true
-	}
-	s.DNSServer = p.Net.DNSServers
-	s.DNSSearch = p.Net.DNSSearch
-	s.DNSOption = p.Net.DNSOptions
-	s.NoManageHosts = p.Net.NoHosts
-	s.HostAdd = p.Net.AddHosts
 
-	namespaces, err := setNamespaces(p)
-	if err != nil {
-		return err
-	}
-	if !namespaces[0].IsDefault() {
-		s.Pid = namespaces[0]
+	if p.Net != nil {
+		s.NetNS = p.Net.Network
+		s.StaticIP = p.Net.StaticIP
+		s.StaticMAC = p.Net.StaticMAC
+		s.PortMappings = p.Net.PublishPorts
+		s.CNINetworks = p.Net.CNINetworks
+		s.NetworkOptions = p.Net.NetworkOptions
+		if p.Net.UseImageResolvConf {
+			s.NoManageResolvConf = true
+		}
+		s.DNSServer = p.Net.DNSServers
+		s.DNSSearch = p.Net.DNSSearch
+		s.DNSOption = p.Net.DNSOptions
+		s.NoManageHosts = p.Net.NoHosts
+		s.HostAdd = p.Net.AddHosts
 	}
 
 	// Cgroup
@@ -219,7 +332,7 @@ func (p *PodCreateOptions) ToPodSpecGen(s *specgen.PodSpecGenerator) error {
 		}
 	}
 	s.Userns = p.Userns
-	return nil
+	return &s, nil
 }
 
 type PodPruneOptions struct {
diff --git a/pkg/domain/entities/types.go b/pkg/domain/entities/types.go
index db4c6bb8a..ec4d4a902 100644
--- a/pkg/domain/entities/types.go
+++ b/pkg/domain/entities/types.go
@@ -31,21 +31,33 @@ type VolumeDeleteReport struct{ Report }
 
 // NetOptions reflect the shared network options between
 // pods and containers
+type NetFlags struct {
+	AddHosts     []string `json:"add-host,omitempty"`
+	DNS          []string `json:"dns,omitempty"`
+	DNSOpt       []string `json:"dns-opt,omitempty"`
+	DNDSearch    []string `json:"dns-search,omitempty"`
+	MacAddr      string   `json:"mac-address,omitempty"`
+	Publish      []string `json:"publish,omitempty"`
+	IP           string   `json:"ip,omitempty"`
+	NoHosts      bool     `json:"no-hosts,omitempty"`
+	Network      string   `json:"network,omitempty"`
+	NetworkAlias []string `json:"network-alias,omitempty"`
+}
 type NetOptions struct {
-	AddHosts           []string
-	Aliases            []string
-	CNINetworks        []string
-	UseImageResolvConf bool
-	DNSOptions         []string
-	DNSSearch          []string
-	DNSServers         []net.IP
-	Network            specgen.Namespace
-	NoHosts            bool
-	PublishPorts       []types.PortMapping
-	StaticIP           *net.IP
-	StaticMAC          *net.HardwareAddr
+	AddHosts           []string            `json:"hostadd,omitempty"`
+	Aliases            []string            `json:"network_alias,omitempty"`
+	CNINetworks        []string            `json:"cni_networks,omitempty"`
+	UseImageResolvConf bool                `json:"no_manage_resolv_conf,omitempty"`
+	DNSOptions         []string            `json:"dns_option,omitempty"`
+	DNSSearch          []string            `json:"dns_search,omitempty"`
+	DNSServers         []net.IP            `json:"dns_server,omitempty"`
+	Network            specgen.Namespace   `json:"netns,omitempty"`
+	NoHosts            bool                `json:"no_manage_hosts,omitempty"`
+	PublishPorts       []types.PortMapping `json:"portmappings,omitempty"`
+	StaticIP           *net.IP             `json:"static_ip,omitempty"`
+	StaticMAC          *net.HardwareAddr   `json:"static_mac,omitempty"`
 	// NetworkOptions are additional options for each network
-	NetworkOptions map[string][]string
+	NetworkOptions map[string][]string `json:"network_options,omitempty"`
 }
 
 // All CLI inspect commands and inspect sub-commands use the same options
diff --git a/pkg/domain/infra/abi/containers.go b/pkg/domain/infra/abi/containers.go
index a74b65ab9..8b2a5bfae 100644
--- a/pkg/domain/infra/abi/containers.go
+++ b/pkg/domain/infra/abi/containers.go
@@ -583,7 +583,11 @@ func (ic *ContainerEngine) ContainerCreate(ctx context.Context, s *specgen.SpecG
 	for _, w := range warn {
 		fmt.Fprintf(os.Stderr, "%s\n", w)
 	}
-	ctr, err := generate.MakeContainer(ctx, ic.Libpod, s)
+	rtSpec, spec, opts, err := generate.MakeContainer(context.Background(), ic.Libpod, s)
+	if err != nil {
+		return nil, err
+	}
+	ctr, err := generate.ExecuteCreate(ctx, ic.Libpod, rtSpec, spec, false, opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -915,7 +919,11 @@ func (ic *ContainerEngine) ContainerRun(ctx context.Context, opts entities.Conta
 	for _, w := range warn {
 		fmt.Fprintf(os.Stderr, "%s\n", w)
 	}
-	ctr, err := generate.MakeContainer(ctx, ic.Libpod, opts.Spec)
+	rtSpec, spec, optsN, err := generate.MakeContainer(ctx, ic.Libpod, opts.Spec)
+	if err != nil {
+		return nil, err
+	}
+	ctr, err := generate.ExecuteCreate(ctx, ic.Libpod, rtSpec, spec, false, optsN...)
 	if err != nil {
 		return nil, err
 	}
diff --git a/pkg/domain/infra/abi/generate.go b/pkg/domain/infra/abi/generate.go
index b0853b554..2d7bc15f5 100644
--- a/pkg/domain/infra/abi/generate.go
+++ b/pkg/domain/infra/abi/generate.go
@@ -60,9 +60,7 @@ func (ic *ContainerEngine) GenerateKube(ctx context.Context, nameOrIDs []string,
 				return nil, err
 			}
 		} else {
-			if len(ctr.Dependencies()) > 0 {
-				return nil, errors.Wrapf(define.ErrNotImplemented, "containers with dependencies")
-			}
+			//  now that infra holds NS data, we need to support dependencies.
 			// we cannot deal with ctrs already in a pod.
 			if len(ctr.PodID()) > 0 {
 				return nil, errors.Errorf("container %s is associated with pod %s: use generate on the pod itself", ctr.ID(), ctr.PodID())
diff --git a/pkg/domain/infra/abi/play.go b/pkg/domain/infra/abi/play.go
index f22b2dbbb..2799df794 100644
--- a/pkg/domain/infra/abi/play.go
+++ b/pkg/domain/infra/abi/play.go
@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"net"
 	"os"
 	"path/filepath"
 	"strconv"
@@ -22,6 +23,7 @@ import (
 	"github.com/containers/podman/v3/pkg/specgen"
 	"github.com/containers/podman/v3/pkg/specgen/generate"
 	"github.com/containers/podman/v3/pkg/specgen/generate/kube"
+	"github.com/containers/podman/v3/pkg/specgenutil"
 	"github.com/containers/podman/v3/pkg/util"
 	"github.com/ghodss/yaml"
 	"github.com/pkg/errors"
@@ -179,10 +181,12 @@ func (ic *ContainerEngine) playKubePod(ctx context.Context, podName string, podY
 		}
 	}
 
-	p, err := kube.ToPodGen(ctx, podName, podYAML)
+	podOpt := entities.PodCreateOptions{Infra: true, Net: &entities.NetOptions{StaticIP: &net.IP{}, StaticMAC: &net.HardwareAddr{}}}
+	podOpt, err = kube.ToPodOpt(ctx, podName, podOpt, podYAML)
 	if err != nil {
 		return nil, err
 	}
+
 	if options.Network != "" {
 		ns, cniNets, netOpts, err := specgen.ParseNetworkString(options.Network)
 		if err != nil {
@@ -193,42 +197,37 @@ func (ic *ContainerEngine) playKubePod(ctx context.Context, podName string, podY
 			return nil, errors.Errorf("invalid value passed to --network: bridge or host networking must be configured in YAML")
 		}
 		logrus.Debugf("Pod %q joining CNI networks: %v", podName, cniNets)
-		p.NetNS.NSMode = specgen.Bridge
-		p.CNINetworks = append(p.CNINetworks, cniNets...)
+		podOpt.Net.Network.NSMode = specgen.Bridge
+		podOpt.Net.CNINetworks = append(podOpt.Net.CNINetworks, cniNets...)
 		if len(netOpts) > 0 {
-			p.NetworkOptions = netOpts
+			podOpt.Net.NetworkOptions = netOpts
 		}
 	}
 
 	if len(options.StaticIPs) > *ipIndex {
-		p.StaticIP = &options.StaticIPs[*ipIndex]
+		podOpt.Net.StaticIP = &options.StaticIPs[*ipIndex]
 	} else if len(options.StaticIPs) > 0 {
 		// only warn if the user has set at least one ip
 		logrus.Warn("No more static ips left using a random one")
 	}
 	if len(options.StaticMACs) > *ipIndex {
-		p.StaticMAC = &options.StaticMACs[*ipIndex]
+		podOpt.Net.StaticMAC = &options.StaticMACs[*ipIndex]
 	} else if len(options.StaticIPs) > 0 {
 		// only warn if the user has set at least one mac
 		logrus.Warn("No more static macs left using a random one")
 	}
 	*ipIndex++
 
-	// Create the Pod
-	pod, err := generate.MakePod(p, ic.Libpod)
+	p := specgen.NewPodSpecGenerator()
 	if err != nil {
 		return nil, err
 	}
 
-	podInfraID, err := pod.InfraContainerID()
+	p, err = entities.ToPodSpecGen(*p, &podOpt)
 	if err != nil {
 		return nil, err
 	}
-
-	if !options.Quiet {
-		writer = os.Stderr
-	}
-
+	podSpec := entities.PodSpec{PodSpecGen: *p}
 	volumes, err := kube.InitializeVolumes(podYAML.Spec.Volumes)
 	if err != nil {
 		return nil, err
@@ -267,112 +266,146 @@ func (ic *ContainerEngine) playKubePod(ctx context.Context, podName string, podY
 		configMaps = append(configMaps, cm)
 	}
 
-	containers := make([]*libpod.Container, 0, len(podYAML.Spec.Containers))
-	cwd, err := os.Getwd()
-	if err != nil {
-		return nil, err
-	}
-	for _, container := range podYAML.Spec.Containers {
-		// Contains all labels obtained from kube
-		labels := make(map[string]string)
-		var pulledImage *libimage.Image
-		buildFile, err := getBuildFile(container.Image, cwd)
+	if podOpt.Infra {
+		imagePull := config.DefaultInfraImage
+		if podOpt.InfraImage != config.DefaultInfraImage && podOpt.InfraImage != "" {
+			imagePull = podOpt.InfraImage
+		}
+
+		pulledImages, err := pullImage(ic, writer, imagePull, options, config.PullPolicyNewer)
 		if err != nil {
 			return nil, err
 		}
-		existsLocally, err := ic.Libpod.LibimageRuntime().Exists(container.Image)
+		infraOptions := entities.ContainerCreateOptions{ImageVolume: "bind"}
+
+		podSpec.PodSpecGen.InfraImage = pulledImages[0].Names()[0]
+		podSpec.PodSpecGen.NoInfra = false
+		podSpec.PodSpecGen.InfraContainerSpec = specgen.NewSpecGenerator(pulledImages[0].Names()[0], false)
+		podSpec.PodSpecGen.InfraContainerSpec.NetworkOptions = p.NetworkOptions
+
+		err = specgenutil.FillOutSpecGen(podSpec.PodSpecGen.InfraContainerSpec, &infraOptions, []string{})
 		if err != nil {
 			return nil, err
 		}
-		if (len(buildFile) > 0 && !existsLocally) || (len(buildFile) > 0 && options.Build) {
-			buildOpts := new(buildahDefine.BuildOptions)
-			commonOpts := new(buildahDefine.CommonBuildOptions)
-			buildOpts.ConfigureNetwork = buildahDefine.NetworkDefault
-			buildOpts.Isolation = buildahDefine.IsolationChroot
-			buildOpts.CommonBuildOpts = commonOpts
-			buildOpts.Output = container.Image
-			if _, _, err := ic.Libpod.Build(ctx, *buildOpts, []string{buildFile}...); err != nil {
+	}
+
+	// Create the Pod
+	pod, err := generate.MakePod(&podSpec, ic.Libpod)
+	if err != nil {
+		return nil, err
+	}
+
+	podInfraID, err := pod.InfraContainerID()
+	if err != nil {
+		return nil, err
+	}
+
+	if !options.Quiet {
+		writer = os.Stderr
+	}
+
+	containers := make([]*libpod.Container, 0, len(podYAML.Spec.Containers))
+	cwd, err := os.Getwd()
+	if err != nil {
+		return nil, err
+	}
+	for _, container := range podYAML.Spec.Containers {
+		if !strings.Contains("infra", container.Name) {
+			// Contains all labels obtained from kube
+			labels := make(map[string]string)
+			var pulledImage *libimage.Image
+			buildFile, err := getBuildFile(container.Image, cwd)
+			if err != nil {
 				return nil, err
 			}
-			i, _, err := ic.Libpod.LibimageRuntime().LookupImage(container.Image, new(libimage.LookupImageOptions))
+			existsLocally, err := ic.Libpod.LibimageRuntime().Exists(container.Image)
 			if err != nil {
 				return nil, err
 			}
-			pulledImage = i
-		} else {
-			// NOTE: set the pull policy to "newer".  This will cover cases
-			// where the "latest" tag requires a pull and will also
-			// transparently handle "localhost/" prefixed files which *may*
-			// refer to a locally built image OR an image running a
-			// registry on localhost.
-			pullPolicy := config.PullPolicyNewer
-			if len(container.ImagePullPolicy) > 0 {
-				// Make sure to lower the strings since K8s pull policy
-				// may be capitalized (see bugzilla.redhat.com/show_bug.cgi?id=1985905).
-				rawPolicy := string(container.ImagePullPolicy)
-				pullPolicy, err = config.ParsePullPolicy(strings.ToLower(rawPolicy))
+			if (len(buildFile) > 0 && !existsLocally) || (len(buildFile) > 0 && options.Build) {
+				buildOpts := new(buildahDefine.BuildOptions)
+				commonOpts := new(buildahDefine.CommonBuildOptions)
+				buildOpts.ConfigureNetwork = buildahDefine.NetworkDefault
+				buildOpts.Isolation = buildahDefine.IsolationChroot
+				buildOpts.CommonBuildOpts = commonOpts
+				buildOpts.Output = container.Image
+				if _, _, err := ic.Libpod.Build(ctx, *buildOpts, []string{buildFile}...); err != nil {
+					return nil, err
+				}
+				i, _, err := ic.Libpod.LibimageRuntime().LookupImage(container.Image, new(libimage.LookupImageOptions))
+				if err != nil {
+					return nil, err
+				}
+				pulledImage = i
+			} else {
+				// NOTE: set the pull policy to "newer".  This will cover cases
+				// where the "latest" tag requires a pull and will also
+				// transparently handle "localhost/" prefixed files which *may*
+				// refer to a locally built image OR an image running a
+				// registry on localhost.
+				pullPolicy := config.PullPolicyNewer
+				if len(container.ImagePullPolicy) > 0 {
+					// Make sure to lower the strings since K8s pull policy
+					// may be capitalized (see bugzilla.redhat.com/show_bug.cgi?id=1985905).
+					rawPolicy := string(container.ImagePullPolicy)
+					pullPolicy, err = config.ParsePullPolicy(strings.ToLower(rawPolicy))
+					if err != nil {
+						return nil, err
+					}
+				}
+				pulledImages, err := pullImage(ic, writer, container.Image, options, pullPolicy)
 				if err != nil {
 					return nil, err
 				}
+				pulledImage = pulledImages[0]
+			}
+
+			// Handle kube annotations
+			for k, v := range annotations {
+				switch k {
+				// Auto update annotation without container name will apply to
+				// all containers within the pod
+				case autoupdate.Label, autoupdate.AuthfileLabel:
+					labels[k] = v
+				// Auto update annotation with container name will apply only
+				// to the specified container
+				case fmt.Sprintf("%s/%s", autoupdate.Label, container.Name),
+					fmt.Sprintf("%s/%s", autoupdate.AuthfileLabel, container.Name):
+					prefixAndCtr := strings.Split(k, "/")
+					labels[prefixAndCtr[0]] = v
+				}
 			}
-			// This ensures the image is the image store
-			pullOptions := &libimage.PullOptions{}
-			pullOptions.AuthFilePath = options.Authfile
-			pullOptions.CertDirPath = options.CertDir
-			pullOptions.SignaturePolicyPath = options.SignaturePolicy
-			pullOptions.Writer = writer
-			pullOptions.Username = options.Username
-			pullOptions.Password = options.Password
-			pullOptions.InsecureSkipTLSVerify = options.SkipTLSVerify
-
-			pulledImages, err := ic.Libpod.LibimageRuntime().Pull(ctx, container.Image, pullPolicy, pullOptions)
+
+			specgenOpts := kube.CtrSpecGenOptions{
+				Container:      container,
+				Image:          pulledImage,
+				Volumes:        volumes,
+				PodID:          pod.ID(),
+				PodName:        podName,
+				PodInfraID:     podInfraID,
+				ConfigMaps:     configMaps,
+				SeccompPaths:   seccompPaths,
+				RestartPolicy:  ctrRestartPolicy,
+				NetNSIsHost:    p.NetNS.IsHost(),
+				SecretsManager: secretsManager,
+				LogDriver:      options.LogDriver,
+				Labels:         labels,
+			}
+			specGen, err := kube.ToSpecGen(ctx, &specgenOpts)
 			if err != nil {
 				return nil, err
 			}
-			pulledImage = pulledImages[0]
-		}
 
-		// Handle kube annotations
-		for k, v := range annotations {
-			switch k {
-			// Auto update annotation without container name will apply to
-			// all containers within the pod
-			case autoupdate.Label, autoupdate.AuthfileLabel:
-				labels[k] = v
-			// Auto update annotation with container name will apply only
-			// to the specified container
-			case fmt.Sprintf("%s/%s", autoupdate.Label, container.Name),
-				fmt.Sprintf("%s/%s", autoupdate.AuthfileLabel, container.Name):
-				prefixAndCtr := strings.Split(k, "/")
-				labels[prefixAndCtr[0]] = v
+			rtSpec, spec, opts, err := generate.MakeContainer(ctx, ic.Libpod, specGen)
+			if err != nil {
+				return nil, err
 			}
+			ctr, err := generate.ExecuteCreate(ctx, ic.Libpod, rtSpec, spec, false, opts...)
+			if err != nil {
+				return nil, err
+			}
+			containers = append(containers, ctr)
 		}
-
-		specgenOpts := kube.CtrSpecGenOptions{
-			Container:      container,
-			Image:          pulledImage,
-			Volumes:        volumes,
-			PodID:          pod.ID(),
-			PodName:        podName,
-			PodInfraID:     podInfraID,
-			ConfigMaps:     configMaps,
-			SeccompPaths:   seccompPaths,
-			RestartPolicy:  ctrRestartPolicy,
-			NetNSIsHost:    p.NetNS.IsHost(),
-			SecretsManager: secretsManager,
-			LogDriver:      options.LogDriver,
-			Labels:         labels,
-		}
-		specGen, err := kube.ToSpecGen(ctx, &specgenOpts)
-		if err != nil {
-			return nil, err
-		}
-
-		ctr, err := generate.MakeContainer(ctx, ic.Libpod, specGen)
-		if err != nil {
-			return nil, err
-		}
-		containers = append(containers, ctr)
 	}
 
 	if options.Start != types.OptionalBoolFalse {
@@ -383,6 +416,7 @@ func (ic *ContainerEngine) playKubePod(ctx context.Context, podName string, podY
 		}
 		for id, err := range podStartErrors {
 			playKubePod.ContainerErrors = append(playKubePod.ContainerErrors, errors.Wrapf(err, "error starting container %s", id).Error())
+			fmt.Println(playKubePod.ContainerErrors)
 		}
 	}
 
@@ -656,3 +690,21 @@ func (ic *ContainerEngine) PlayKubeDown(ctx context.Context, path string, _ enti
 	}
 	return reports, nil
 }
+
+// pullImage is a helper function to set up the proper pull options and pull the image for certain containers
+func pullImage(ic *ContainerEngine, writer io.Writer, imagePull string, options entities.PlayKubeOptions, pullPolicy config.PullPolicy) ([]*libimage.Image, error) {
+	// This ensures the image is the image store
+	pullOptions := &libimage.PullOptions{}
+	pullOptions.AuthFilePath = options.Authfile
+	pullOptions.CertDirPath = options.CertDir
+	pullOptions.SignaturePolicyPath = options.SignaturePolicy
+	pullOptions.Writer = writer
+	pullOptions.Username = options.Username
+	pullOptions.Password = options.Password
+	pullOptions.InsecureSkipTLSVerify = options.SkipTLSVerify
+	pulledImages, err := ic.Libpod.LibimageRuntime().Pull(context.Background(), imagePull, pullPolicy, pullOptions)
+	if err != nil {
+		return nil, err
+	}
+	return pulledImages, nil
+}
diff --git a/pkg/domain/infra/abi/pods.go b/pkg/domain/infra/abi/pods.go
index 055c495d5..98233f60d 100644
--- a/pkg/domain/infra/abi/pods.go
+++ b/pkg/domain/infra/abi/pods.go
@@ -8,7 +8,6 @@ import (
 	"github.com/containers/podman/v3/pkg/domain/entities"
 	dfilters "github.com/containers/podman/v3/pkg/domain/filters"
 	"github.com/containers/podman/v3/pkg/signal"
-	"github.com/containers/podman/v3/pkg/specgen"
 	"github.com/containers/podman/v3/pkg/specgen/generate"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
@@ -248,12 +247,8 @@ func (ic *ContainerEngine) prunePodHelper(ctx context.Context) ([]*entities.PodP
 	return reports, nil
 }
 
-func (ic *ContainerEngine) PodCreate(ctx context.Context, opts entities.PodCreateOptions) (*entities.PodCreateReport, error) {
-	podSpec := specgen.NewPodSpecGenerator()
-	if err := opts.ToPodSpecGen(podSpec); err != nil {
-		return nil, err
-	}
-	pod, err := generate.MakePod(podSpec, ic.Libpod)
+func (ic *ContainerEngine) PodCreate(ctx context.Context, specg entities.PodSpec) (*entities.PodCreateReport, error) {
+	pod, err := generate.MakePod(&specg, ic.Libpod)
 	if err != nil {
 		return nil, err
 	}
diff --git a/pkg/domain/infra/tunnel/events.go b/pkg/domain/infra/tunnel/events.go
index 6e2c3f8ba..203550c5d 100644
--- a/pkg/domain/infra/tunnel/events.go
+++ b/pkg/domain/infra/tunnel/events.go
@@ -7,6 +7,7 @@ import (
 	"github.com/containers/podman/v3/libpod/events"
 	"github.com/containers/podman/v3/pkg/bindings/system"
 	"github.com/containers/podman/v3/pkg/domain/entities"
+
 	"github.com/pkg/errors"
 )
 
diff --git a/pkg/domain/infra/tunnel/pods.go b/pkg/domain/infra/tunnel/pods.go
index 82f062b2c..480adb88a 100644
--- a/pkg/domain/infra/tunnel/pods.go
+++ b/pkg/domain/infra/tunnel/pods.go
@@ -6,7 +6,6 @@ import (
 	"github.com/containers/podman/v3/libpod/define"
 	"github.com/containers/podman/v3/pkg/bindings/pods"
 	"github.com/containers/podman/v3/pkg/domain/entities"
-	"github.com/containers/podman/v3/pkg/specgen"
 	"github.com/containers/podman/v3/pkg/util"
 	"github.com/pkg/errors"
 )
@@ -179,10 +178,8 @@ func (ic *ContainerEngine) PodPrune(ctx context.Context, opts entities.PodPruneO
 	return pods.Prune(ic.ClientCtx, nil)
 }
 
-func (ic *ContainerEngine) PodCreate(ctx context.Context, opts entities.PodCreateOptions) (*entities.PodCreateReport, error) {
-	podSpec := specgen.NewPodSpecGenerator()
-	opts.ToPodSpecGen(podSpec)
-	return pods.CreatePodFromSpec(ic.ClientCtx, podSpec, nil)
+func (ic *ContainerEngine) PodCreate(ctx context.Context, specg entities.PodSpec) (*entities.PodCreateReport, error) {
+	return pods.CreatePodFromSpec(ic.ClientCtx, &specg)
 }
 
 func (ic *ContainerEngine) PodTop(ctx context.Context, opts entities.PodTopOptions) (*entities.StringSliceReport, error) {