Merge pull request #11011 from baude/initcontainers

implement init containers in podman

16 files changed, 366 insertions, 20 deletions

diff --git a/cmd/podman/common/create_opts.go b/cmd/podman/common/create_opts.go
index 42e0efe5d..61f08b73b 100644
--- a/cmd/podman/common/create_opts.go
+++ b/cmd/podman/common/create_opts.go
@@ -61,6 +61,7 @@ type ContainerCLIOpts struct {
 	HTTPProxy         bool
 	ImageVolume       string
 	Init              bool
+	InitContainerType string
 	InitPath          string
 	Interactive       bool
 	IPC               string
diff --git a/cmd/podman/common/specgen.go b/cmd/podman/common/specgen.go
index 42f515ace..118091855 100644
--- a/cmd/podman/common/specgen.go
+++ b/cmd/podman/common/specgen.go
@@ -659,6 +659,8 @@ func FillOutSpecGen(s *specgen.SpecGenerator, c *ContainerCLIOpts, args []string
 	s.PidFile = c.PidFile
 	s.Volatile = c.Rm
 
+	// Initcontainers
+	s.InitContainerType = c.InitContainerType
 	return nil
 }
 
diff --git a/cmd/podman/containers/create.go b/cmd/podman/containers/create.go
index c63c074f7..895736144 100644
--- a/cmd/podman/containers/create.go
+++ b/cmd/podman/containers/create.go
@@ -7,6 +7,7 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/containers/common/pkg/completion"
 	"github.com/containers/common/pkg/config"
 	"github.com/containers/image/v5/transports/alltransports"
 	"github.com/containers/podman/v3/cmd/podman/common"
@@ -49,12 +50,20 @@ var (
 )
 
 var (
-	cliVals common.ContainerCLIOpts
+	cliVals           common.ContainerCLIOpts
+	InitContainerType string
 )
 
 func createFlags(cmd *cobra.Command) {
 	flags := cmd.Flags()
 
+	initContainerFlagName := "init-ctr"
+	flags.StringVar(
+		&InitContainerType,
+		initContainerFlagName, "",
+		"Make this a pod init container.",
+	)
+
 	flags.SetInterspersed(false)
 	common.DefineCreateFlags(cmd, &cliVals)
 	common.DefineNetFlags(cmd)
@@ -65,6 +74,8 @@ func createFlags(cmd *cobra.Command) {
 		_ = flags.MarkHidden("conmon-pidfile")
 		_ = flags.MarkHidden("pidfile")
 	}
+
+	_ = cmd.RegisterFlagCompletionFunc(initContainerFlagName, completion.AutocompleteDefault)
 }
 
 func init() {
@@ -89,6 +100,17 @@ func create(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
+	// Check if initctr is used with --pod and the value is correct
+	if initctr := InitContainerType; cmd.Flags().Changed("init-ctr") {
+		if !cmd.Flags().Changed("pod") {
+			return errors.New("must specify pod value with init-ctr")
+		}
+		if !util.StringInSlice(initctr, []string{"always", "oneshot"}) {
+			return errors.New("init-ctr value must be 'always' or 'oneshot'")
+		}
+		cliVals.InitContainerType = initctr
+	}
+
 	if err := createInit(cmd); err != nil {
 		return err
 	}
diff --git a/docs/source/markdown/podman-create.1.md b/docs/source/markdown/podman-create.1.md
index 7f9cf0e75..b2f7260ae 100644
--- a/docs/source/markdown/podman-create.1.md
+++ b/docs/source/markdown/podman-create.1.md
@@ -447,6 +447,21 @@ content that disappears when the container is stopped.
 
 Run an init inside the container that forwards signals and reaps processes.
 
+#### **--init-ctr**=*type* (pods only)
+
+When using pods, create an init style container, which is run after the infra container is started
+but before regular pod containers are started.  Init containers are useful for running
+setup operations for the pod's applications.
+
+Valid values for `init-ctr` type are *always* or *oneshot*.  The *always* value
+means the container will run with each and every `pod start`, whereas the *oneshot*
+value means is will ony run once when the pod is started and then the container is
+removed.
+
+Init containers are only run on pod `start`.  Restarting a pod will not execute any init
+containers should they be present.  Furthermore, init containers can only be created in a
+pod when that pod is not running.
+
 #### **--init-path**=*path*
 
 Path to the container-init binary.
diff --git a/libpod/container_config.go b/libpod/container_config.go
index 0de79fde3..72a969fe6 100644
--- a/libpod/container_config.go
+++ b/libpod/container_config.go
@@ -375,4 +375,7 @@ type ContainerMiscConfig struct {
 	CDIDevices []string `json:"cdiDevices,omitempty"`
 	// EnvSecrets are secrets that are set as environment variables
 	EnvSecrets map[string]*secrets.Secret `json:"secret_env,omitempty"`
+	// InitContainerType specifies if the container is an initcontainer
+	// and if so, what type: always or oneshot are possible non-nil entries
+	InitContainerType string `json:"init_container_type,omitempty"`
 }
diff --git a/libpod/container_graph.go b/libpod/container_graph.go
index 3ae7cfbc7..32fb264f1 100644
--- a/libpod/container_graph.go
+++ b/libpod/container_graph.go
@@ -259,7 +259,7 @@ func startNode(ctx context.Context, node *containerNode, setError bool, ctrError
 	}
 
 	// Start the container (only if it is not running)
-	if !ctrErrored {
+	if !ctrErrored && len(node.container.config.InitContainerType) < 1 {
 		if !restart && node.container.state.State != define.ContainerStateRunning {
 			if err := node.container.initAndStart(ctx); err != nil {
 				ctrErrored = true
diff --git a/libpod/container_validate.go b/libpod/container_validate.go
index 6ff46f1b1..91ebe93fb 100644
--- a/libpod/container_validate.go
+++ b/libpod/container_validate.go
@@ -131,5 +131,11 @@ func (c *Container) validate() error {
 	if c.config.User == "" && (c.config.Spec.Process.User.UID != 0 || c.config.Spec.Process.User.GID != 0) {
 		return errors.Wrapf(define.ErrInvalidArg, "please set User explicitly via WithUser() instead of in OCI spec directly")
 	}
+
+	// Init-ctrs must be used inside a Pod.  Check if a init container type is
+	// passed and if no pod is passed
+	if len(c.config.InitContainerType) > 0 && len(c.config.Pod) < 1 {
+		return errors.Wrap(define.ErrInvalidArg, "init containers must be created in a pod")
+	}
 	return nil
 }
diff --git a/libpod/define/container.go b/libpod/define/container.go
index f3125afa9..f0aca92aa 100644
--- a/libpod/define/container.go
+++ b/libpod/define/container.go
@@ -26,3 +26,13 @@ var RestartPolicyMap = map[string]string{
 	RestartPolicyOnFailure:     RestartPolicyOnFailure,
 	RestartPolicyUnlessStopped: RestartPolicyUnlessStopped,
 }
+
+// InitContainerTypes
+const (
+	// AlwaysInitContainer is an init container than runs on each
+	// pod start (including restart)
+	AlwaysInitContainer = "always"
+	// OneShotInitContainer is a container that only runs as init once
+	// and is then deleted.
+	OneShotInitContainer = "oneshot"
+)
diff --git a/libpod/options.go b/libpod/options.go
index b021b9f50..071b085e7 100644
--- a/libpod/options.go
+++ b/libpod/options.go
@@ -1794,6 +1794,21 @@ func WithPidFile(pidFile string) CtrCreateOption {
 	}
 }
 
+// WithInitCtrType indicates the container is a initcontainer
+func WithInitCtrType(containerType string) CtrCreateOption {
+	return func(ctr *Container) error {
+		if ctr.valid {
+			return define.ErrCtrFinalized
+		}
+		// Make sure the type is valid
+		if containerType == define.OneShotInitContainer || containerType == define.AlwaysInitContainer {
+			ctr.config.InitContainerType = containerType
+			return nil
+		}
+		return errors.Errorf("%s is invalid init container type", containerType)
+	}
+}
+
 // Pod Creation Options
 
 // WithInfraImage sets the infra image for libpod.
diff --git a/libpod/pod.go b/libpod/pod.go
index 62f5c9e5b..0fef7f6f3 100644
--- a/libpod/pod.go
+++ b/libpod/pod.go
@@ -3,6 +3,7 @@ package libpod
 import (
 	"context"
 	"net"
+	"sort"
 	"time"
 
 	"github.com/containers/podman/v3/libpod/define"
@@ -332,17 +333,20 @@ func (p *Pod) SharesNamespaces() bool {
 	return p.SharesPID() || p.SharesIPC() || p.SharesNet() || p.SharesMount() || p.SharesUser() || p.SharesUTS()
 }
 
+// infraContainerID returns the infra ID without a lock
+func (p *Pod) infraContainerID() (string, error) {
+	if err := p.updatePod(); err != nil {
+		return "", err
+	}
+	return p.state.InfraContainerID, nil
+}
+
 // InfraContainerID returns the infra container ID for a pod.
 // If the container returned is "", the pod has no infra container.
 func (p *Pod) InfraContainerID() (string, error) {
 	p.lock.Lock()
 	defer p.lock.Unlock()
-
-	if err := p.updatePod(); err != nil {
-		return "", err
-	}
-
-	return p.state.InfraContainerID, nil
+	return p.infraContainerID()
 }
 
 // InfraContainer returns the infra container.
@@ -350,7 +354,6 @@ func (p *Pod) InfraContainer() (*Container, error) {
 	if !p.HasInfraContainer() {
 		return nil, errors.Wrap(define.ErrNoSuchCtr, "pod has no infra container")
 	}
-
 	id, err := p.InfraContainerID()
 	if err != nil {
 		return nil, err
@@ -420,3 +423,23 @@ func (p *Pod) ProcessLabel() (string, error) {
 	}
 	return ctr.ProcessLabel(), nil
 }
+
+// initContainers returns the list of initcontainers
+// in a pod sorted by create time
+func (p *Pod) initContainers() ([]*Container, error) {
+	initCons := make([]*Container, 0)
+	// the pod is already locked when this is called
+	cons, err := p.allContainers()
+	if err != nil {
+		return nil, err
+	}
+	// Sort the pod containers by created time
+	sort.Slice(cons, func(i, j int) bool { return cons[i].CreatedTime().Before(cons[j].CreatedTime()) })
+	// Iterate sorted containers and add ids for any init containers
+	for _, c := range cons {
+		if len(c.config.InitContainerType) > 0 {
+			initCons = append(initCons, c)
+		}
+	}
+	return initCons, nil
+}
diff --git a/libpod/pod_api.go b/libpod/pod_api.go
index 1ab012a8b..90d67dbb0 100644
--- a/libpod/pod_api.go
+++ b/libpod/pod_api.go
@@ -12,6 +12,45 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
+// startInitContainers starts a pod's init containers.
+func (p *Pod) startInitContainers(ctx context.Context) error {
+	initCtrs, err := p.initContainers()
+	if err != nil {
+		return err
+	}
+	// Now iterate init containers
+	for _, initCon := range initCtrs {
+		if err := initCon.Start(ctx, true); err != nil {
+			return err
+		}
+		// Check that the init container waited correctly and the exit
+		// code is good
+		rc, err := initCon.Wait(ctx)
+		if err != nil {
+			return err
+		}
+		if rc != 0 {
+			return errors.Errorf("init container %s exited with code %d", initCon.ID(), rc)
+		}
+		// If the container is an oneshot init container, we need to remove it
+		// after it runs
+		if initCon.Config().InitContainerType == define.OneShotInitContainer {
+			icLock := initCon.lock
+			icLock.Lock()
+			if err := p.runtime.removeContainer(ctx, initCon, false, false, true); err != nil {
+				icLock.Unlock()
+				return errors.Wrapf(err, "failed to remove oneshot init container %s", initCon.ID())
+			}
+			// Removing a container this way requires an explicit call to clean up the db
+			if err := p.runtime.state.RemoveContainerFromPod(p, initCon); err != nil {
+				logrus.Errorf("Error removing container %s from database: %v", initCon.ID(), err)
+			}
+			icLock.Unlock()
+		}
+	}
+	return nil
+}
+
 // Start starts all containers within a pod.
 // It combines the effects of Init() and Start() on a container.
 // If a container has already been initialized it will be started,
@@ -34,26 +73,29 @@ func (p *Pod) Start(ctx context.Context) (map[string]error, error) {
 		return nil, define.ErrPodRemoved
 	}
 
+	// Before "regular" containers start in the pod, all init containers
+	// must have run and exited successfully.
+	if err := p.startInitContainers(ctx); err != nil {
+		return nil, err
+	}
 	allCtrs, err := p.runtime.state.PodContainers(p)
 	if err != nil {
 		return nil, err
 	}
-
 	// Build a dependency graph of containers in the pod
 	graph, err := BuildContainerGraph(allCtrs)
 	if err != nil {
 		return nil, errors.Wrapf(err, "error generating dependency graph for pod %s", p.ID())
 	}
-
-	ctrErrors := make(map[string]error)
-	ctrsVisited := make(map[string]bool)
-
 	// If there are no containers without dependencies, we can't start
 	// Error out
 	if len(graph.noDepNodes) == 0 {
 		return nil, errors.Wrapf(define.ErrNoSuchCtr, "no containers in pod %s have no dependencies, cannot start pod", p.ID())
 	}
 
+	ctrErrors := make(map[string]error)
+	ctrsVisited := make(map[string]bool)
+
 	// Traverse the graph beginning at nodes with no dependencies
 	for _, node := range graph.noDepNodes {
 		startNode(ctx, node, false, ctrErrors, ctrsVisited, false)
@@ -449,12 +491,18 @@ func (p *Pod) Status() (map[string]define.ContainerStatus, error) {
 	if !p.valid {
 		return nil, define.ErrPodRemoved
 	}
-
 	allCtrs, err := p.runtime.state.PodContainers(p)
 	if err != nil {
 		return nil, err
 	}
-	return containerStatusFromContainers(allCtrs)
+	noInitCtrs := make([]*Container, 0)
+	// Do not add init containers into status
+	for _, ctr := range allCtrs {
+		if ctrType := ctr.config.InitContainerType; len(ctrType) < 1 {
+			noInitCtrs = append(noInitCtrs, ctr)
+		}
+	}
+	return containerStatusFromContainers(noInitCtrs)
 }
 
 func containerStatusFromContainers(allCtrs []*Container) (map[string]define.ContainerStatus, error) {
@@ -504,7 +552,10 @@ func (p *Pod) Inspect() (*define.InspectPodData, error) {
 			Name:  c.Name(),
 			State: containerStatus,
 		})
-		ctrStatuses[c.ID()] = c.state.State
+		// Do not add init containers fdr status
+		if len(c.config.InitContainerType) < 1 {
+			ctrStatuses[c.ID()] = c.state.State
+		}
 	}
 	podState, err := createPodStatusResults(ctrStatuses)
 	if err != nil {
diff --git a/pkg/domain/infra/abi/pods.go b/pkg/domain/infra/abi/pods.go
index 9f033a4c0..055c495d5 100644
--- a/pkg/domain/infra/abi/pods.go
+++ b/pkg/domain/infra/abi/pods.go
@@ -250,7 +250,9 @@ func (ic *ContainerEngine) prunePodHelper(ctx context.Context) ([]*entities.PodP
 
 func (ic *ContainerEngine) PodCreate(ctx context.Context, opts entities.PodCreateOptions) (*entities.PodCreateReport, error) {
 	podSpec := specgen.NewPodSpecGenerator()
-	opts.ToPodSpecGen(podSpec)
+	if err := opts.ToPodSpecGen(podSpec); err != nil {
+		return nil, err
+	}
 	pod, err := generate.MakePod(podSpec, ic.Libpod)
 	if err != nil {
 		return nil, err
diff --git a/pkg/specgen/generate/container.go b/pkg/specgen/generate/container.go
index 1f6d00eb7..ae26807a9 100644
--- a/pkg/specgen/generate/container.go
+++ b/pkg/specgen/generate/container.go
@@ -140,10 +140,29 @@ func CompleteSpec(ctx context.Context, r *libpod.Runtime, s *specgen.SpecGenerat
 	//   VM, which is the default behavior
 	// - "container" denotes the container should join the VM of the SandboxID
 	//   (the infra container)
-
 	if len(s.Pod) > 0 {
 		annotations[ann.SandboxID] = s.Pod
 		annotations[ann.ContainerType] = ann.ContainerTypeContainer
+		// Check if this is an init-ctr and if so, check if
+		// the pod is running.  we do not want to add init-ctrs to
+		// a running pod because it creates confusion for us.
+		if len(s.InitContainerType) > 0 {
+			p, err := r.LookupPod(s.Pod)
+			if err != nil {
+				return nil, err
+			}
+			containerStatuses, err := p.Status()
+			if err != nil {
+				return nil, err
+			}
+			// If any one of the containers is running, the pod is considered to be
+			// running
+			for _, con := range containerStatuses {
+				if con == define.ContainerStateRunning {
+					return nil, errors.New("cannot add init-ctr to a running pod")
+				}
+			}
+		}
 	}
 
 	for _, v := range rtc.Containers.Annotations {
diff --git a/pkg/specgen/generate/container_create.go b/pkg/specgen/generate/container_create.go
index 4e3a86ae4..5101a6ccb 100644
--- a/pkg/specgen/generate/container_create.go
+++ b/pkg/specgen/generate/container_create.go
@@ -144,11 +144,14 @@ func MakeContainer(ctx context.Context, rt *libpod.Runtime, s *specgen.SpecGener
 		options = append(options, libpod.WithNetworkAliases(s.Aliases))
 	}
 
+	if containerType := s.InitContainerType; len(containerType) > 0 {
+		options = append(options, libpod.WithInitCtrType(containerType))
+	}
+
 	if len(s.Devices) > 0 {
 		opts = extractCDIDevices(s)
 		options = append(options, opts...)
 	}
-
 	runtimeSpec, err := SpecGenToOCI(ctx, s, rt, rtc, newImage, finalMounts, pod, command)
 	if err != nil {
 		return nil, err
diff --git a/pkg/specgen/specgen.go b/pkg/specgen/specgen.go
index 7eec48a55..b4ac337b5 100644
--- a/pkg/specgen/specgen.go
+++ b/pkg/specgen/specgen.go
@@ -183,6 +183,9 @@ type ContainerBasicConfig struct {
 	// EnvSecrets are secrets that will be set as environment variables
 	// Optional.
 	EnvSecrets map[string]string `json:"secret_env,omitempty"`
+	// InitContainerType describes if this container is an init container
+	// and if so, what type: always or oneshot
+	InitContainerType string `json:"init_container_type"`
 }
 
 // ContainerStorageConfig contains information on the storage configuration of a
diff --git a/test/e2e/pod_initcontainers_test.go b/test/e2e/pod_initcontainers_test.go
new file mode 100644
index 000000000..606294f51
--- /dev/null
+++ b/test/e2e/pod_initcontainers_test.go
@@ -0,0 +1,171 @@
+package integration
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/containers/podman/v3/libpod/define"
+	. "github.com/containers/podman/v3/test/utils"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	. "github.com/onsi/gomega/gexec"
+)
+
+var _ = Describe("Podman init containers", func() {
+	var (
+		tempdir    string
+		err        error
+		podmanTest *PodmanTestIntegration
+	)
+
+	BeforeEach(func() {
+		tempdir, err = CreateTempDirInTempDir()
+		if err != nil {
+			os.Exit(1)
+		}
+		podmanTest = PodmanTestCreate(tempdir)
+		podmanTest.Setup()
+		podmanTest.SeedImages()
+	})
+
+	AfterEach(func() {
+		podmanTest.Cleanup()
+		f := CurrentGinkgoTestDescription()
+		processTestResult(f)
+
+	})
+
+	It("podman create init container without --pod should fail", func() {
+		session := podmanTest.Podman([]string{"create", "--init-ctr", "always", ALPINE, "top"})
+		session.WaitWithDefaultTimeout()
+		Expect(session).Should(Exit(125))
+	})
+
+	It("podman create init container with bad init type should fail", func() {
+		session := podmanTest.Podman([]string{"create", "--init-ctr", "unknown", "--pod", "new:foobar", ALPINE, "top"})
+		session.WaitWithDefaultTimeout()
+		Expect(session).Should(Exit(125))
+	})
+
+	It("podman init containers should not degrade pod status", func() {
+		// create a pod
+		topPod := podmanTest.Podman([]string{"create", "-t", "--pod", "new:foobar", ALPINE, "top"})
+		topPod.WaitWithDefaultTimeout()
+		Expect(topPod).Should(Exit(0))
+		// add an init container
+		session := podmanTest.Podman([]string{"create", "--init-ctr", "always", "--pod", "foobar", ALPINE, "date"})
+		session.WaitWithDefaultTimeout()
+		Expect(session).Should(Exit(0))
+		// start a pod
+		start := podmanTest.Podman([]string{"pod", "start", "foobar"})
+		start.WaitWithDefaultTimeout()
+		Expect(start).Should(Exit(0))
+
+		inspect := podmanTest.Podman([]string{"pod", "inspect", "foobar"})
+		inspect.WaitWithDefaultTimeout()
+		Expect(inspect).Should(Exit(0))
+		data := inspect.InspectPodToJSON()
+		Expect(data.State).To(Equal(define.PodStateRunning))
+	})
+
+	It("podman create init container should fail in running pod", func() {
+		// create a running pod
+		topPod := podmanTest.Podman([]string{"run", "-dt", "--pod", "new:foobar", ALPINE, "top"})
+		topPod.WaitWithDefaultTimeout()
+		Expect(topPod).Should(Exit(0))
+		// adding init-ctr to running pod should fail
+		session := podmanTest.Podman([]string{"create", "--init-ctr", "always", "--pod", "foobar", ALPINE, "date"})
+		session.WaitWithDefaultTimeout()
+		Expect(session).Should(Exit(125))
+	})
+
+	It("podman make sure init container runs before pod containers", func() {
+		filename := filepath.Join("/dev/shm", RandomString(12))
+		content := RandomString(16)
+		session := podmanTest.Podman([]string{"create", "--init-ctr", "always", "--pod", "new:foobar", ALPINE, "bin/sh", "-c", fmt.Sprintf("echo %s > %s", content, filename)})
+		session.WaitWithDefaultTimeout()
+		Expect(session).Should(Exit(0))
+		verify := podmanTest.Podman([]string{"create", "--pod", "foobar", "-t", ALPINE, "top"})
+		verify.WaitWithDefaultTimeout()
+		Expect(verify).Should(Exit(0))
+		start := podmanTest.Podman([]string{"pod", "start", "foobar"})
+		start.WaitWithDefaultTimeout()
+		Expect(start).Should(Exit(0))
+		checkLog := podmanTest.Podman([]string{"exec", "-it", verify.OutputToString(), "cat", filename})
+		checkLog.WaitWithDefaultTimeout()
+		Expect(checkLog).Should(Exit(0))
+		Expect(checkLog.OutputToString()).To(Equal(content))
+	})
+
+	It("podman make sure oneshot container is removed", func() {
+		filename := filepath.Join("/dev/shm", RandomString(12))
+		content := RandomString(16)
+		session := podmanTest.Podman([]string{"create", "--init-ctr", "oneshot", "--pod", "new:foobar", ALPINE, "bin/sh", "-c", fmt.Sprintf("echo %s > %s", content, filename)})
+		session.WaitWithDefaultTimeout()
+		initContainerID := session.OutputToString()
+		Expect(session).Should(Exit(0))
+		verify := podmanTest.Podman([]string{"create", "--pod", "foobar", "-t", ALPINE, "top"})
+		verify.WaitWithDefaultTimeout()
+		Expect(verify).Should(Exit(0))
+		start := podmanTest.Podman([]string{"pod", "start", "foobar"})
+		start.WaitWithDefaultTimeout()
+		Expect(start).Should(Exit(0))
+		check := podmanTest.Podman([]string{"container", "exists", initContainerID})
+		check.WaitWithDefaultTimeout()
+		// Container was rm'd
+		//Expect(check).Should(Exit(1))
+		Expect(check.ExitCode()).To(Equal(1), "I dont understand why the other way does not work")
+		// Lets double check with a stop and start
+		stopPod := podmanTest.Podman([]string{"pod", "stop", "foobar"})
+		stopPod.WaitWithDefaultTimeout()
+		Expect(stopPod).Should(Exit(0))
+		startPod := podmanTest.Podman([]string{"pod", "start", "foobar"})
+		startPod.WaitWithDefaultTimeout()
+		Expect(startPod).Should(Exit(0))
+
+		// Because no init was run, the file should not even exist
+		doubleCheck := podmanTest.Podman([]string{"exec", "-it", verify.OutputToString(), "cat", filename})
+		doubleCheck.WaitWithDefaultTimeout()
+		Expect(doubleCheck).Should(Exit(1))
+
+	})
+
+	It("podman ensure always init containers always run", func() {
+		filename := filepath.Join("/dev/shm", RandomString(12))
+
+		// Write the date to a file
+		session := podmanTest.Podman([]string{"create", "--init-ctr", "always", "--pod", "new:foobar", ALPINE, "bin/sh", "-c", fmt.Sprintf("date > %s", filename)})
+		session.WaitWithDefaultTimeout()
+		Expect(session).Should(Exit(0))
+		verify := podmanTest.Podman([]string{"create", "--pod", "foobar", "-t", ALPINE, "top"})
+		verify.WaitWithDefaultTimeout()
+		Expect(verify).Should(Exit(0))
+		start := podmanTest.Podman([]string{"pod", "start", "foobar"})
+		start.WaitWithDefaultTimeout()
+		Expect(start).Should(Exit(0))
+
+		// capture the date written
+		checkLog := podmanTest.Podman([]string{"exec", "-it", verify.OutputToString(), "cat", filename})
+		checkLog.WaitWithDefaultTimeout()
+		firstResult := checkLog.OutputToString()
+		Expect(checkLog).Should(Exit(0))
+
+		// Stop and start the pod
+		stopPod := podmanTest.Podman([]string{"pod", "stop", "foobar"})
+		stopPod.WaitWithDefaultTimeout()
+		Expect(stopPod).Should(Exit(0))
+		startPod := podmanTest.Podman([]string{"pod", "start", "foobar"})
+		startPod.WaitWithDefaultTimeout()
+		Expect(startPod).Should(Exit(0))
+
+		// Check the file again with exec
+		secondCheckLog := podmanTest.Podman([]string{"exec", "-it", verify.OutputToString(), "cat", filename})
+		secondCheckLog.WaitWithDefaultTimeout()
+		Expect(secondCheckLog).Should(Exit(0))
+
+		// Dates should not match
+		Expect(firstResult).ToNot(Equal(secondCheckLog.OutputToString()))
+	})
+
+})