InfraContainer Rework

InfraContainer should go through the same creation process as regular containers. This change was from the cmd level
down, involving new container CLI opts and specgen creating functions. What now happens is that both container and pod
cli options are populated in cmd and used to create a podSpecgen and a containerSpecgen. The process then goes as follows

FillOutSpecGen (infra) -> MapSpec (podOpts -> infraOpts) -> PodCreate -> MakePod -> createPodOptions -> NewPod -> CompleteSpec (infra) -> MakeContainer -> NewContainer -> newContainer -> AddInfra (to pod state)

Signed-off-by: cdoern <cdoern@redhat.com>

9 files changed, 341 insertions, 165 deletions

diff --git a/pkg/domain/entities/engine_container.go b/pkg/domain/entities/engine_container.go
index 5acf7211c..bd011d309 100644
--- a/pkg/domain/entities/engine_container.go
+++ b/pkg/domain/entities/engine_container.go
@@ -68,7 +68,7 @@ type ContainerEngine interface {
 	NetworkRm(ctx context.Context, namesOrIds []string, options NetworkRmOptions) ([]*NetworkRmReport, error)
 	PlayKube(ctx context.Context, path string, opts PlayKubeOptions) (*PlayKubeReport, error)
 	PlayKubeDown(ctx context.Context, path string, opts PlayKubeDownOptions) (*PlayKubeReport, error)
-	PodCreate(ctx context.Context, opts PodCreateOptions) (*PodCreateReport, error)
+	PodCreate(ctx context.Context, specg PodSpec) (*PodCreateReport, error)
 	PodExists(ctx context.Context, nameOrID string) (*BoolReport, error)
 	PodInspect(ctx context.Context, options PodInspectOptions) (*PodInspectReport, error)
 	PodKill(ctx context.Context, namesOrIds []string, options PodKillOptions) ([]*PodKillReport, error)
diff --git a/pkg/domain/entities/pods.go b/pkg/domain/entities/pods.go
index c66bf96fc..10bd7e5ce 100644
--- a/pkg/domain/entities/pods.go
+++ b/pkg/domain/entities/pods.go
@@ -106,6 +106,14 @@ type PodRmReport struct {
 	Id  string //nolint
 }
 
+// PddSpec is an abstracted version of PodSpecGen designed to eventually accept options
+// not meant to be in a specgen
+type PodSpec struct {
+	PodSpecGen specgen.PodSpecGenerator
+}
+
+// PodCreateOptions provides all possible options for creating a pod and its infra container
+// swagger:model PodCreateOptions
 type PodCreateOptions struct {
 	CGroupParent       string
 	CreateCommand      []string
@@ -125,6 +133,123 @@ type PodCreateOptions struct {
 	Userns             specgen.Namespace
 }
 
+type ContainerCreateOptions struct {
+	Annotation        []string
+	Attach            []string
+	Authfile          string
+	BlkIOWeight       string
+	BlkIOWeightDevice []string
+	CapAdd            []string
+	CapDrop           []string
+	CgroupNS          string
+	CGroupsMode       string
+	CGroupParent      string
+	CIDFile           string
+	ConmonPIDFile     string
+	CPUPeriod         uint64
+	CPUQuota          int64
+	CPURTPeriod       uint64
+	CPURTRuntime      int64
+	CPUShares         uint64
+	CPUS              float64
+	CPUSetCPUs        string
+	CPUSetMems        string
+	Devices           []string
+	DeviceCGroupRule  []string
+	DeviceReadBPs     []string
+	DeviceReadIOPs    []string
+	DeviceWriteBPs    []string
+	DeviceWriteIOPs   []string
+	Entrypoint        *string
+	Env               []string
+	EnvHost           bool
+	EnvFile           []string
+	Expose            []string
+	GIDMap            []string
+	GroupAdd          []string
+	HealthCmd         string
+	HealthInterval    string
+	HealthRetries     uint
+	HealthStartPeriod string
+	HealthTimeout     string
+	Hostname          string
+	HTTPProxy         bool
+	ImageVolume       string
+	Init              bool
+	InitContainerType string
+	InitPath          string
+	Interactive       bool
+	IPC               string
+	KernelMemory      string
+	Label             []string
+	LabelFile         []string
+	LogDriver         string
+	LogOptions        []string
+	Memory            string
+	MemoryReservation string
+	MemorySwap        string
+	MemorySwappiness  int64
+	Name              string
+	NoHealthCheck     bool
+	OOMKillDisable    bool
+	OOMScoreAdj       int
+	Arch              string
+	OS                string
+	Variant           string
+	PID               string
+	PIDsLimit         *int64
+	Platform          string
+	Pod               string
+	PodIDFile         string
+	Personality       string
+	PreserveFDs       uint
+	Privileged        bool
+	PublishAll        bool
+	Pull              string
+	Quiet             bool
+	ReadOnly          bool
+	ReadOnlyTmpFS     bool
+	Restart           string
+	Replace           bool
+	Requires          []string
+	Rm                bool
+	RootFS            bool
+	Secrets           []string
+	SecurityOpt       []string
+	SdNotifyMode      string
+	ShmSize           string
+	SignaturePolicy   string
+	StopSignal        string
+	StopTimeout       uint
+	StorageOpt        []string
+	SubUIDName        string
+	SubGIDName        string
+	Sysctl            []string
+	Systemd           string
+	Timeout           uint
+	TLSVerify         bool
+	TmpFS             []string
+	TTY               bool
+	Timezone          string
+	Umask             string
+	UIDMap            []string
+	Ulimit            []string
+	User              string
+	UserNS            string
+	UTS               string
+	Mount             []string
+	Volume            []string
+	VolumesFrom       []string
+	Workdir           string
+	SeccompPolicy     string
+	PidFile           string
+	IsInfra           bool
+
+	Net *NetOptions
+
+	CgroupConf []string
+}
+
 type PodCreateReport struct {
 	Id string //nolint
 }
@@ -149,21 +274,15 @@ func (p *PodCreateOptions) CPULimits() *specs.LinuxCPU {
 	return cpu
 }
 
-func setNamespaces(p *PodCreateOptions) ([4]specgen.Namespace, error) {
-	allNS := [4]specgen.Namespace{}
-	if p.Pid != "" {
-		pid, err := specgen.ParseNamespace(p.Pid)
-		if err != nil {
-			return [4]specgen.Namespace{}, err
-		}
-		allNS[0] = pid
-	}
-	return allNS, nil
-}
-
-func (p *PodCreateOptions) ToPodSpecGen(s *specgen.PodSpecGenerator) error {
+func ToPodSpecGen(s specgen.PodSpecGenerator, p *PodCreateOptions) (*specgen.PodSpecGenerator, error) {
 	// Basic Config
 	s.Name = p.Name
+	s.InfraName = p.InfraName
+	out, err := specgen.ParseNamespace(p.Pid)
+	if err != nil {
+		return nil, err
+	}
+	s.Pid = out
 	s.Hostname = p.Hostname
 	s.Labels = p.Labels
 	s.NoInfra = !p.Infra
@@ -174,32 +293,26 @@ func (p *PodCreateOptions) ToPodSpecGen(s *specgen.PodSpecGenerator) error {
 		s.InfraConmonPidFile = p.InfraConmonPidFile
 	}
 	s.InfraImage = p.InfraImage
-	s.InfraName = p.InfraName
 	s.SharedNamespaces = p.Share
 	s.PodCreateCommand = p.CreateCommand
 
 	// Networking config
-	s.NetNS = p.Net.Network
-	s.StaticIP = p.Net.StaticIP
-	s.StaticMAC = p.Net.StaticMAC
-	s.PortMappings = p.Net.PublishPorts
-	s.CNINetworks = p.Net.CNINetworks
-	s.NetworkOptions = p.Net.NetworkOptions
-	if p.Net.UseImageResolvConf {
-		s.NoManageResolvConf = true
-	}
-	s.DNSServer = p.Net.DNSServers
-	s.DNSSearch = p.Net.DNSSearch
-	s.DNSOption = p.Net.DNSOptions
-	s.NoManageHosts = p.Net.NoHosts
-	s.HostAdd = p.Net.AddHosts
 
-	namespaces, err := setNamespaces(p)
-	if err != nil {
-		return err
-	}
-	if !namespaces[0].IsDefault() {
-		s.Pid = namespaces[0]
+	if p.Net != nil {
+		s.NetNS = p.Net.Network
+		s.StaticIP = p.Net.StaticIP
+		s.StaticMAC = p.Net.StaticMAC
+		s.PortMappings = p.Net.PublishPorts
+		s.CNINetworks = p.Net.CNINetworks
+		s.NetworkOptions = p.Net.NetworkOptions
+		if p.Net.UseImageResolvConf {
+			s.NoManageResolvConf = true
+		}
+		s.DNSServer = p.Net.DNSServers
+		s.DNSSearch = p.Net.DNSSearch
+		s.DNSOption = p.Net.DNSOptions
+		s.NoManageHosts = p.Net.NoHosts
+		s.HostAdd = p.Net.AddHosts
 	}
 
 	// Cgroup
@@ -219,7 +332,7 @@ func (p *PodCreateOptions) ToPodSpecGen(s *specgen.PodSpecGenerator) error {
 		}
 	}
 	s.Userns = p.Userns
-	return nil
+	return &s, nil
 }
 
 type PodPruneOptions struct {
diff --git a/pkg/domain/entities/types.go b/pkg/domain/entities/types.go
index db4c6bb8a..ec4d4a902 100644
--- a/pkg/domain/entities/types.go
+++ b/pkg/domain/entities/types.go
@@ -31,21 +31,33 @@ type VolumeDeleteReport struct{ Report }
 
 // NetOptions reflect the shared network options between
 // pods and containers
+type NetFlags struct {
+	AddHosts     []string `json:"add-host,omitempty"`
+	DNS          []string `json:"dns,omitempty"`
+	DNSOpt       []string `json:"dns-opt,omitempty"`
+	DNDSearch    []string `json:"dns-search,omitempty"`
+	MacAddr      string   `json:"mac-address,omitempty"`
+	Publish      []string `json:"publish,omitempty"`
+	IP           string   `json:"ip,omitempty"`
+	NoHosts      bool     `json:"no-hosts,omitempty"`
+	Network      string   `json:"network,omitempty"`
+	NetworkAlias []string `json:"network-alias,omitempty"`
+}
 type NetOptions struct {
-	AddHosts           []string
-	Aliases            []string
-	CNINetworks        []string
-	UseImageResolvConf bool
-	DNSOptions         []string
-	DNSSearch          []string
-	DNSServers         []net.IP
-	Network            specgen.Namespace
-	NoHosts            bool
-	PublishPorts       []types.PortMapping
-	StaticIP           *net.IP
-	StaticMAC          *net.HardwareAddr
+	AddHosts           []string            `json:"hostadd,omitempty"`
+	Aliases            []string            `json:"network_alias,omitempty"`
+	CNINetworks        []string            `json:"cni_networks,omitempty"`
+	UseImageResolvConf bool                `json:"no_manage_resolv_conf,omitempty"`
+	DNSOptions         []string            `json:"dns_option,omitempty"`
+	DNSSearch          []string            `json:"dns_search,omitempty"`
+	DNSServers         []net.IP            `json:"dns_server,omitempty"`
+	Network            specgen.Namespace   `json:"netns,omitempty"`
+	NoHosts            bool                `json:"no_manage_hosts,omitempty"`
+	PublishPorts       []types.PortMapping `json:"portmappings,omitempty"`
+	StaticIP           *net.IP             `json:"static_ip,omitempty"`
+	StaticMAC          *net.HardwareAddr   `json:"static_mac,omitempty"`
 	// NetworkOptions are additional options for each network
-	NetworkOptions map[string][]string
+	NetworkOptions map[string][]string `json:"network_options,omitempty"`
 }
 
 // All CLI inspect commands and inspect sub-commands use the same options
diff --git a/pkg/domain/infra/abi/containers.go b/pkg/domain/infra/abi/containers.go
index a74b65ab9..8b2a5bfae 100644
--- a/pkg/domain/infra/abi/containers.go
+++ b/pkg/domain/infra/abi/containers.go
@@ -583,7 +583,11 @@ func (ic *ContainerEngine) ContainerCreate(ctx context.Context, s *specgen.SpecG
 	for _, w := range warn {
 		fmt.Fprintf(os.Stderr, "%s\n", w)
 	}
-	ctr, err := generate.MakeContainer(ctx, ic.Libpod, s)
+	rtSpec, spec, opts, err := generate.MakeContainer(context.Background(), ic.Libpod, s)
+	if err != nil {
+		return nil, err
+	}
+	ctr, err := generate.ExecuteCreate(ctx, ic.Libpod, rtSpec, spec, false, opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -915,7 +919,11 @@ func (ic *ContainerEngine) ContainerRun(ctx context.Context, opts entities.Conta
 	for _, w := range warn {
 		fmt.Fprintf(os.Stderr, "%s\n", w)
 	}
-	ctr, err := generate.MakeContainer(ctx, ic.Libpod, opts.Spec)
+	rtSpec, spec, optsN, err := generate.MakeContainer(ctx, ic.Libpod, opts.Spec)
+	if err != nil {
+		return nil, err
+	}
+	ctr, err := generate.ExecuteCreate(ctx, ic.Libpod, rtSpec, spec, false, optsN...)
 	if err != nil {
 		return nil, err
 	}
diff --git a/pkg/domain/infra/abi/generate.go b/pkg/domain/infra/abi/generate.go
index b0853b554..2d7bc15f5 100644
--- a/pkg/domain/infra/abi/generate.go
+++ b/pkg/domain/infra/abi/generate.go
@@ -60,9 +60,7 @@ func (ic *ContainerEngine) GenerateKube(ctx context.Context, nameOrIDs []string,
 				return nil, err
 			}
 		} else {
-			if len(ctr.Dependencies()) > 0 {
-				return nil, errors.Wrapf(define.ErrNotImplemented, "containers with dependencies")
-			}
+			//  now that infra holds NS data, we need to support dependencies.
 			// we cannot deal with ctrs already in a pod.
 			if len(ctr.PodID()) > 0 {
 				return nil, errors.Errorf("container %s is associated with pod %s: use generate on the pod itself", ctr.ID(), ctr.PodID())
diff --git a/pkg/domain/infra/abi/play.go b/pkg/domain/infra/abi/play.go
index f22b2dbbb..2799df794 100644
--- a/pkg/domain/infra/abi/play.go
+++ b/pkg/domain/infra/abi/play.go
@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"net"
 	"os"
 	"path/filepath"
 	"strconv"
@@ -22,6 +23,7 @@ import (
 	"github.com/containers/podman/v3/pkg/specgen"
 	"github.com/containers/podman/v3/pkg/specgen/generate"
 	"github.com/containers/podman/v3/pkg/specgen/generate/kube"
+	"github.com/containers/podman/v3/pkg/specgenutil"
 	"github.com/containers/podman/v3/pkg/util"
 	"github.com/ghodss/yaml"
 	"github.com/pkg/errors"
@@ -179,10 +181,12 @@ func (ic *ContainerEngine) playKubePod(ctx context.Context, podName string, podY
 		}
 	}
 
-	p, err := kube.ToPodGen(ctx, podName, podYAML)
+	podOpt := entities.PodCreateOptions{Infra: true, Net: &entities.NetOptions{StaticIP: &net.IP{}, StaticMAC: &net.HardwareAddr{}}}
+	podOpt, err = kube.ToPodOpt(ctx, podName, podOpt, podYAML)
 	if err != nil {
 		return nil, err
 	}
+
 	if options.Network != "" {
 		ns, cniNets, netOpts, err := specgen.ParseNetworkString(options.Network)
 		if err != nil {
@@ -193,42 +197,37 @@ func (ic *ContainerEngine) playKubePod(ctx context.Context, podName string, podY
 			return nil, errors.Errorf("invalid value passed to --network: bridge or host networking must be configured in YAML")
 		}
 		logrus.Debugf("Pod %q joining CNI networks: %v", podName, cniNets)
-		p.NetNS.NSMode = specgen.Bridge
-		p.CNINetworks = append(p.CNINetworks, cniNets...)
+		podOpt.Net.Network.NSMode = specgen.Bridge
+		podOpt.Net.CNINetworks = append(podOpt.Net.CNINetworks, cniNets...)
 		if len(netOpts) > 0 {
-			p.NetworkOptions = netOpts
+			podOpt.Net.NetworkOptions = netOpts
 		}
 	}
 
 	if len(options.StaticIPs) > *ipIndex {
-		p.StaticIP = &options.StaticIPs[*ipIndex]
+		podOpt.Net.StaticIP = &options.StaticIPs[*ipIndex]
 	} else if len(options.StaticIPs) > 0 {
 		// only warn if the user has set at least one ip
 		logrus.Warn("No more static ips left using a random one")
 	}
 	if len(options.StaticMACs) > *ipIndex {
-		p.StaticMAC = &options.StaticMACs[*ipIndex]
+		podOpt.Net.StaticMAC = &options.StaticMACs[*ipIndex]
 	} else if len(options.StaticIPs) > 0 {
 		// only warn if the user has set at least one mac
 		logrus.Warn("No more static macs left using a random one")
 	}
 	*ipIndex++
 
-	// Create the Pod
-	pod, err := generate.MakePod(p, ic.Libpod)
+	p := specgen.NewPodSpecGenerator()
 	if err != nil {
 		return nil, err
 	}
 
-	podInfraID, err := pod.InfraContainerID()
+	p, err = entities.ToPodSpecGen(*p, &podOpt)
 	if err != nil {
 		return nil, err
 	}
-
-	if !options.Quiet {
-		writer = os.Stderr
-	}
-
+	podSpec := entities.PodSpec{PodSpecGen: *p}
 	volumes, err := kube.InitializeVolumes(podYAML.Spec.Volumes)
 	if err != nil {
 		return nil, err
@@ -267,112 +266,146 @@ func (ic *ContainerEngine) playKubePod(ctx context.Context, podName string, podY
 		configMaps = append(configMaps, cm)
 	}
 
-	containers := make([]*libpod.Container, 0, len(podYAML.Spec.Containers))
-	cwd, err := os.Getwd()
-	if err != nil {
-		return nil, err
-	}
-	for _, container := range podYAML.Spec.Containers {
-		// Contains all labels obtained from kube
-		labels := make(map[string]string)
-		var pulledImage *libimage.Image
-		buildFile, err := getBuildFile(container.Image, cwd)
+	if podOpt.Infra {
+		imagePull := config.DefaultInfraImage
+		if podOpt.InfraImage != config.DefaultInfraImage && podOpt.InfraImage != "" {
+			imagePull = podOpt.InfraImage
+		}
+
+		pulledImages, err := pullImage(ic, writer, imagePull, options, config.PullPolicyNewer)
 		if err != nil {
 			return nil, err
 		}
-		existsLocally, err := ic.Libpod.LibimageRuntime().Exists(container.Image)
+		infraOptions := entities.ContainerCreateOptions{ImageVolume: "bind"}
+
+		podSpec.PodSpecGen.InfraImage = pulledImages[0].Names()[0]
+		podSpec.PodSpecGen.NoInfra = false
+		podSpec.PodSpecGen.InfraContainerSpec = specgen.NewSpecGenerator(pulledImages[0].Names()[0], false)
+		podSpec.PodSpecGen.InfraContainerSpec.NetworkOptions = p.NetworkOptions
+
+		err = specgenutil.FillOutSpecGen(podSpec.PodSpecGen.InfraContainerSpec, &infraOptions, []string{})
 		if err != nil {
 			return nil, err
 		}
-		if (len(buildFile) > 0 && !existsLocally) || (len(buildFile) > 0 && options.Build) {
-			buildOpts := new(buildahDefine.BuildOptions)
-			commonOpts := new(buildahDefine.CommonBuildOptions)
-			buildOpts.ConfigureNetwork = buildahDefine.NetworkDefault
-			buildOpts.Isolation = buildahDefine.IsolationChroot
-			buildOpts.CommonBuildOpts = commonOpts
-			buildOpts.Output = container.Image
-			if _, _, err := ic.Libpod.Build(ctx, *buildOpts, []string{buildFile}...); err != nil {
+	}
+
+	// Create the Pod
+	pod, err := generate.MakePod(&podSpec, ic.Libpod)
+	if err != nil {
+		return nil, err
+	}
+
+	podInfraID, err := pod.InfraContainerID()
+	if err != nil {
+		return nil, err
+	}
+
+	if !options.Quiet {
+		writer = os.Stderr
+	}
+
+	containers := make([]*libpod.Container, 0, len(podYAML.Spec.Containers))
+	cwd, err := os.Getwd()
+	if err != nil {
+		return nil, err
+	}
+	for _, container := range podYAML.Spec.Containers {
+		if !strings.Contains("infra", container.Name) {
+			// Contains all labels obtained from kube
+			labels := make(map[string]string)
+			var pulledImage *libimage.Image
+			buildFile, err := getBuildFile(container.Image, cwd)
+			if err != nil {
 				return nil, err
 			}
-			i, _, err := ic.Libpod.LibimageRuntime().LookupImage(container.Image, new(libimage.LookupImageOptions))
+			existsLocally, err := ic.Libpod.LibimageRuntime().Exists(container.Image)
 			if err != nil {
 				return nil, err
 			}
-			pulledImage = i
-		} else {
-			// NOTE: set the pull policy to "newer".  This will cover cases
-			// where the "latest" tag requires a pull and will also
-			// transparently handle "localhost/" prefixed files which *may*
-			// refer to a locally built image OR an image running a
-			// registry on localhost.
-			pullPolicy := config.PullPolicyNewer
-			if len(container.ImagePullPolicy) > 0 {
-				// Make sure to lower the strings since K8s pull policy
-				// may be capitalized (see bugzilla.redhat.com/show_bug.cgi?id=1985905).
-				rawPolicy := string(container.ImagePullPolicy)
-				pullPolicy, err = config.ParsePullPolicy(strings.ToLower(rawPolicy))
+			if (len(buildFile) > 0 && !existsLocally) || (len(buildFile) > 0 && options.Build) {
+				buildOpts := new(buildahDefine.BuildOptions)
+				commonOpts := new(buildahDefine.CommonBuildOptions)
+				buildOpts.ConfigureNetwork = buildahDefine.NetworkDefault
+				buildOpts.Isolation = buildahDefine.IsolationChroot
+				buildOpts.CommonBuildOpts = commonOpts
+				buildOpts.Output = container.Image
+				if _, _, err := ic.Libpod.Build(ctx, *buildOpts, []string{buildFile}...); err != nil {
+					return nil, err
+				}
+				i, _, err := ic.Libpod.LibimageRuntime().LookupImage(container.Image, new(libimage.LookupImageOptions))
+				if err != nil {
+					return nil, err
+				}
+				pulledImage = i
+			} else {
+				// NOTE: set the pull policy to "newer".  This will cover cases
+				// where the "latest" tag requires a pull and will also
+				// transparently handle "localhost/" prefixed files which *may*
+				// refer to a locally built image OR an image running a
+				// registry on localhost.
+				pullPolicy := config.PullPolicyNewer
+				if len(container.ImagePullPolicy) > 0 {
+					// Make sure to lower the strings since K8s pull policy
+					// may be capitalized (see bugzilla.redhat.com/show_bug.cgi?id=1985905).
+					rawPolicy := string(container.ImagePullPolicy)
+					pullPolicy, err = config.ParsePullPolicy(strings.ToLower(rawPolicy))
+					if err != nil {
+						return nil, err
+					}
+				}
+				pulledImages, err := pullImage(ic, writer, container.Image, options, pullPolicy)
 				if err != nil {
 					return nil, err
 				}
+				pulledImage = pulledImages[0]
+			}
+
+			// Handle kube annotations
+			for k, v := range annotations {
+				switch k {
+				// Auto update annotation without container name will apply to
+				// all containers within the pod
+				case autoupdate.Label, autoupdate.AuthfileLabel:
+					labels[k] = v
+				// Auto update annotation with container name will apply only
+				// to the specified container
+				case fmt.Sprintf("%s/%s", autoupdate.Label, container.Name),
+					fmt.Sprintf("%s/%s", autoupdate.AuthfileLabel, container.Name):
+					prefixAndCtr := strings.Split(k, "/")
+					labels[prefixAndCtr[0]] = v
+				}
 			}
-			// This ensures the image is the image store
-			pullOptions := &libimage.PullOptions{}
-			pullOptions.AuthFilePath = options.Authfile
-			pullOptions.CertDirPath = options.CertDir
-			pullOptions.SignaturePolicyPath = options.SignaturePolicy
-			pullOptions.Writer = writer
-			pullOptions.Username = options.Username
-			pullOptions.Password = options.Password
-			pullOptions.InsecureSkipTLSVerify = options.SkipTLSVerify
-
-			pulledImages, err := ic.Libpod.LibimageRuntime().Pull(ctx, container.Image, pullPolicy, pullOptions)
+
+			specgenOpts := kube.CtrSpecGenOptions{
+				Container:      container,
+				Image:          pulledImage,
+				Volumes:        volumes,
+				PodID:          pod.ID(),
+				PodName:        podName,
+				PodInfraID:     podInfraID,
+				ConfigMaps:     configMaps,
+				SeccompPaths:   seccompPaths,
+				RestartPolicy:  ctrRestartPolicy,
+				NetNSIsHost:    p.NetNS.IsHost(),
+				SecretsManager: secretsManager,
+				LogDriver:      options.LogDriver,
+				Labels:         labels,
+			}
+			specGen, err := kube.ToSpecGen(ctx, &specgenOpts)
 			if err != nil {
 				return nil, err
 			}
-			pulledImage = pulledImages[0]
-		}
 
-		// Handle kube annotations
-		for k, v := range annotations {
-			switch k {
-			// Auto update annotation without container name will apply to
-			// all containers within the pod
-			case autoupdate.Label, autoupdate.AuthfileLabel:
-				labels[k] = v
-			// Auto update annotation with container name will apply only
-			// to the specified container
-			case fmt.Sprintf("%s/%s", autoupdate.Label, container.Name),
-				fmt.Sprintf("%s/%s", autoupdate.AuthfileLabel, container.Name):
-				prefixAndCtr := strings.Split(k, "/")
-				labels[prefixAndCtr[0]] = v
+			rtSpec, spec, opts, err := generate.MakeContainer(ctx, ic.Libpod, specGen)
+			if err != nil {
+				return nil, err
 			}
+			ctr, err := generate.ExecuteCreate(ctx, ic.Libpod, rtSpec, spec, false, opts...)
+			if err != nil {
+				return nil, err
+			}
+			containers = append(containers, ctr)
 		}
-
-		specgenOpts := kube.CtrSpecGenOptions{
-			Container:      container,
-			Image:          pulledImage,
-			Volumes:        volumes,
-			PodID:          pod.ID(),
-			PodName:        podName,
-			PodInfraID:     podInfraID,
-			ConfigMaps:     configMaps,
-			SeccompPaths:   seccompPaths,
-			RestartPolicy:  ctrRestartPolicy,
-			NetNSIsHost:    p.NetNS.IsHost(),
-			SecretsManager: secretsManager,
-			LogDriver:      options.LogDriver,
-			Labels:         labels,
-		}
-		specGen, err := kube.ToSpecGen(ctx, &specgenOpts)
-		if err != nil {
-			return nil, err
-		}
-
-		ctr, err := generate.MakeContainer(ctx, ic.Libpod, specGen)
-		if err != nil {
-			return nil, err
-		}
-		containers = append(containers, ctr)
 	}
 
 	if options.Start != types.OptionalBoolFalse {
@@ -383,6 +416,7 @@ func (ic *ContainerEngine) playKubePod(ctx context.Context, podName string, podY
 		}
 		for id, err := range podStartErrors {
 			playKubePod.ContainerErrors = append(playKubePod.ContainerErrors, errors.Wrapf(err, "error starting container %s", id).Error())
+			fmt.Println(playKubePod.ContainerErrors)
 		}
 	}
 
@@ -656,3 +690,21 @@ func (ic *ContainerEngine) PlayKubeDown(ctx context.Context, path string, _ enti
 	}
 	return reports, nil
 }
+
+// pullImage is a helper function to set up the proper pull options and pull the image for certain containers
+func pullImage(ic *ContainerEngine, writer io.Writer, imagePull string, options entities.PlayKubeOptions, pullPolicy config.PullPolicy) ([]*libimage.Image, error) {
+	// This ensures the image is the image store
+	pullOptions := &libimage.PullOptions{}
+	pullOptions.AuthFilePath = options.Authfile
+	pullOptions.CertDirPath = options.CertDir
+	pullOptions.SignaturePolicyPath = options.SignaturePolicy
+	pullOptions.Writer = writer
+	pullOptions.Username = options.Username
+	pullOptions.Password = options.Password
+	pullOptions.InsecureSkipTLSVerify = options.SkipTLSVerify
+	pulledImages, err := ic.Libpod.LibimageRuntime().Pull(context.Background(), imagePull, pullPolicy, pullOptions)
+	if err != nil {
+		return nil, err
+	}
+	return pulledImages, nil
+}
diff --git a/pkg/domain/infra/abi/pods.go b/pkg/domain/infra/abi/pods.go
index 055c495d5..98233f60d 100644
--- a/pkg/domain/infra/abi/pods.go
+++ b/pkg/domain/infra/abi/pods.go
@@ -8,7 +8,6 @@ import (
 	"github.com/containers/podman/v3/pkg/domain/entities"
 	dfilters "github.com/containers/podman/v3/pkg/domain/filters"
 	"github.com/containers/podman/v3/pkg/signal"
-	"github.com/containers/podman/v3/pkg/specgen"
 	"github.com/containers/podman/v3/pkg/specgen/generate"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
@@ -248,12 +247,8 @@ func (ic *ContainerEngine) prunePodHelper(ctx context.Context) ([]*entities.PodP
 	return reports, nil
 }
 
-func (ic *ContainerEngine) PodCreate(ctx context.Context, opts entities.PodCreateOptions) (*entities.PodCreateReport, error) {
-	podSpec := specgen.NewPodSpecGenerator()
-	if err := opts.ToPodSpecGen(podSpec); err != nil {
-		return nil, err
-	}
-	pod, err := generate.MakePod(podSpec, ic.Libpod)
+func (ic *ContainerEngine) PodCreate(ctx context.Context, specg entities.PodSpec) (*entities.PodCreateReport, error) {
+	pod, err := generate.MakePod(&specg, ic.Libpod)
 	if err != nil {
 		return nil, err
 	}
diff --git a/pkg/domain/infra/tunnel/events.go b/pkg/domain/infra/tunnel/events.go
index 6e2c3f8ba..203550c5d 100644
--- a/pkg/domain/infra/tunnel/events.go
+++ b/pkg/domain/infra/tunnel/events.go
@@ -7,6 +7,7 @@ import (
 	"github.com/containers/podman/v3/libpod/events"
 	"github.com/containers/podman/v3/pkg/bindings/system"
 	"github.com/containers/podman/v3/pkg/domain/entities"
+
 	"github.com/pkg/errors"
 )
 
diff --git a/pkg/domain/infra/tunnel/pods.go b/pkg/domain/infra/tunnel/pods.go
index 82f062b2c..480adb88a 100644
--- a/pkg/domain/infra/tunnel/pods.go
+++ b/pkg/domain/infra/tunnel/pods.go
@@ -6,7 +6,6 @@ import (
 	"github.com/containers/podman/v3/libpod/define"
 	"github.com/containers/podman/v3/pkg/bindings/pods"
 	"github.com/containers/podman/v3/pkg/domain/entities"
-	"github.com/containers/podman/v3/pkg/specgen"
 	"github.com/containers/podman/v3/pkg/util"
 	"github.com/pkg/errors"
 )
@@ -179,10 +178,8 @@ func (ic *ContainerEngine) PodPrune(ctx context.Context, opts entities.PodPruneO
 	return pods.Prune(ic.ClientCtx, nil)
 }
 
-func (ic *ContainerEngine) PodCreate(ctx context.Context, opts entities.PodCreateOptions) (*entities.PodCreateReport, error) {
-	podSpec := specgen.NewPodSpecGenerator()
-	opts.ToPodSpecGen(podSpec)
-	return pods.CreatePodFromSpec(ic.ClientCtx, podSpec, nil)
+func (ic *ContainerEngine) PodCreate(ctx context.Context, specg entities.PodSpec) (*entities.PodCreateReport, error) {
+	return pods.CreatePodFromSpec(ic.ClientCtx, &specg)
 }
 
 func (ic *ContainerEngine) PodTop(ctx context.Context, opts entities.PodTopOptions) (*entities.StringSliceReport, error) {