Implement Podman Container Clone

podman container clone takes the id of an existing continer and creates a specgen from the given container's config
recreating all proper namespaces and overriding spec options like resource limits and the container name if given in the cli options

this command utilizes the common function DefineCreateFlags meaning that we can funnel as many create options as we want
into clone over time allowing the user to clone with as much or as little of the original config as they want.

container clone takes a second argument which is a new name and a third argument which is an image name to use instead of the original container's

the current supported flags are:

--destroy (remove the original container)
--name (new ctr name)
--cpus (sets cpu period and quota)
--cpuset-cpus
--cpu-period
--cpu-rt-period
--cpu-rt-runtime
--cpu-shares
--cpuset-mems
--memory
--run

resolves #10875

Signed-off-by: cdoern <cdoern@redhat.com>
Signed-off-by: cdoern <cbdoer23@g.holycross.edu>
Signed-off-by: cdoern <cdoern@redhat.com>

13 files changed, 514 insertions, 96 deletions

diff --git a/pkg/api/handlers/libpod/containers_create.go b/pkg/api/handlers/libpod/containers_create.go
index 8e5fc1c1c..61f437faf 100644
--- a/pkg/api/handlers/libpod/containers_create.go
+++ b/pkg/api/handlers/libpod/containers_create.go
@@ -33,7 +33,7 @@ func CreateContainer(w http.ResponseWriter, r *http.Request) {
 		utils.InternalServerError(w, err)
 		return
 	}
-	rtSpec, spec, opts, err := generate.MakeContainer(context.Background(), runtime, &sg)
+	rtSpec, spec, opts, err := generate.MakeContainer(context.Background(), runtime, &sg, false, nil)
 	if err != nil {
 		utils.InternalServerError(w, err)
 		return
diff --git a/pkg/domain/entities/containers.go b/pkg/domain/entities/containers.go
index d39a70732..e9bce0eb7 100644
--- a/pkg/domain/entities/containers.go
+++ b/pkg/domain/entities/containers.go
@@ -463,3 +463,13 @@ type ContainerRenameOptions struct {
 	// NewName is the new name that will be given to the container.
 	NewName string
 }
+
+// ContainerCloneOptions contains options for cloning an existing continer
+type ContainerCloneOptions struct {
+	ID           string
+	Destroy      bool
+	CreateOpts   ContainerCreateOptions
+	Image        string
+	RawImageName string
+	Run          bool
+}
diff --git a/pkg/domain/entities/engine_container.go b/pkg/domain/entities/engine_container.go
index 5dedceacc..21272d33f 100644
--- a/pkg/domain/entities/engine_container.go
+++ b/pkg/domain/entities/engine_container.go
@@ -19,6 +19,7 @@ type ContainerEngine interface {
 	ContainerAttach(ctx context.Context, nameOrID string, options AttachOptions) error
 	ContainerCheckpoint(ctx context.Context, namesOrIds []string, options CheckpointOptions) ([]*CheckpointReport, error)
 	ContainerCleanup(ctx context.Context, namesOrIds []string, options ContainerCleanupOptions) ([]*ContainerCleanupReport, error)
+	ContainerClone(ctx context.Context, ctrClone ContainerCloneOptions) (*ContainerCreateReport, error)
 	ContainerCommit(ctx context.Context, nameOrID string, options CommitOptions) (*CommitReport, error)
 	ContainerCopyFromArchive(ctx context.Context, nameOrID, path string, reader io.Reader, options CopyOptions) (ContainerCopyFunc, error)
 	ContainerCopyToArchive(ctx context.Context, nameOrID string, path string, writer io.Writer) (ContainerCopyFunc, error)
diff --git a/pkg/domain/entities/pods.go b/pkg/domain/entities/pods.go
index 7922db4e6..6fb3db1b5 100644
--- a/pkg/domain/entities/pods.go
+++ b/pkg/domain/entities/pods.go
@@ -264,6 +264,7 @@ type ContainerCreateOptions struct {
 	SeccompPolicy     string
 	PidFile           string
 	IsInfra           bool
+	IsClone           bool
 
 	Net *NetOptions `json:"net,omitempty"`
 
diff --git a/pkg/domain/infra/abi/containers.go b/pkg/domain/infra/abi/containers.go
index a5c4647d7..92f5b1a80 100644
--- a/pkg/domain/infra/abi/containers.go
+++ b/pkg/domain/infra/abi/containers.go
@@ -6,6 +6,7 @@ import (
 	"io/ioutil"
 	"os"
 	"strconv"
+	"strings"
 	"sync"
 	"time"
 
@@ -648,7 +649,7 @@ func (ic *ContainerEngine) ContainerCreate(ctx context.Context, s *specgen.SpecG
 	for _, w := range warn {
 		fmt.Fprintf(os.Stderr, "%s\n", w)
 	}
-	rtSpec, spec, opts, err := generate.MakeContainer(context.Background(), ic.Libpod, s)
+	rtSpec, spec, opts, err := generate.MakeContainer(context.Background(), ic.Libpod, s, false, nil)
 	if err != nil {
 		return nil, err
 	}
@@ -971,7 +972,7 @@ func (ic *ContainerEngine) ContainerRun(ctx context.Context, opts entities.Conta
 		fmt.Fprintf(os.Stderr, "%s\n", w)
 	}
 
-	rtSpec, spec, optsN, err := generate.MakeContainer(ctx, ic.Libpod, opts.Spec)
+	rtSpec, spec, optsN, err := generate.MakeContainer(ctx, ic.Libpod, opts.Spec, false, nil)
 	if err != nil {
 		return nil, err
 	}
@@ -1490,3 +1491,89 @@ func (ic *ContainerEngine) ContainerRename(ctx context.Context, nameOrID string,
 
 	return nil
 }
+
+func (ic *ContainerEngine) ContainerClone(ctx context.Context, ctrCloneOpts entities.ContainerCloneOptions) (*entities.ContainerCreateReport, error) {
+	spec := specgen.NewSpecGenerator(ctrCloneOpts.Image, ctrCloneOpts.CreateOpts.RootFS)
+	var c *libpod.Container
+	c, err := generate.ConfigToSpec(ic.Libpod, spec, ctrCloneOpts.ID)
+	if err != nil {
+		return nil, err
+	}
+
+	err = specgenutil.FillOutSpecGen(spec, &ctrCloneOpts.CreateOpts, []string{})
+	if err != nil {
+		return nil, err
+	}
+	out, err := generate.CompleteSpec(ctx, ic.Libpod, spec)
+	if err != nil {
+		return nil, err
+	}
+
+	// Print warnings
+	if len(out) > 0 {
+		for _, w := range out {
+			fmt.Println("Could not properly complete the spec as expected:")
+			fmt.Fprintf(os.Stderr, "%s\n", w)
+		}
+	}
+
+	if len(ctrCloneOpts.CreateOpts.Name) > 0 {
+		spec.Name = ctrCloneOpts.CreateOpts.Name
+	} else {
+		n := c.Name()
+		_, err := ic.Libpod.LookupContainer(c.Name() + "-clone")
+		if err == nil {
+			n += "-clone"
+		}
+		switch {
+		case strings.Contains(n, "-clone"):
+			ind := strings.Index(n, "-clone") + 6
+			num, _ := strconv.Atoi(n[ind:])
+			if num == 0 { // clone1 is hard to get with this logic, just check for it here.
+				_, err = ic.Libpod.LookupContainer(n + "1")
+				if err != nil {
+					spec.Name = n + "1"
+					break
+				}
+			} else {
+				n = n[0:ind]
+			}
+			err = nil
+			count := num
+			for err == nil {
+				count++
+				tempN := n + strconv.Itoa(count)
+				_, err = ic.Libpod.LookupContainer(tempN)
+			}
+			n += strconv.Itoa(count)
+			spec.Name = n
+		default:
+			spec.Name = c.Name() + "-clone"
+		}
+	}
+
+	rtSpec, spec, opts, err := generate.MakeContainer(context.Background(), ic.Libpod, spec, true, c)
+	if err != nil {
+		return nil, err
+	}
+	ctr, err := generate.ExecuteCreate(ctx, ic.Libpod, rtSpec, spec, false, opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	if ctrCloneOpts.Destroy {
+		var time *uint
+		err := ic.Libpod.RemoveContainer(context.Background(), c, false, false, time)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if ctrCloneOpts.Run {
+		if err := ctr.Start(ctx, true); err != nil {
+			return nil, err
+		}
+	}
+
+	return &entities.ContainerCreateReport{Id: ctr.ID()}, nil
+}
diff --git a/pkg/domain/infra/abi/play.go b/pkg/domain/infra/abi/play.go
index cad8c4609..25e8f8556 100644
--- a/pkg/domain/infra/abi/play.go
+++ b/pkg/domain/infra/abi/play.go
@@ -392,7 +392,7 @@ func (ic *ContainerEngine) playKubePod(ctx context.Context, podName string, podY
 		if err != nil {
 			return nil, err
 		}
-		rtSpec, spec, opts, err := generate.MakeContainer(ctx, ic.Libpod, specGen)
+		rtSpec, spec, opts, err := generate.MakeContainer(ctx, ic.Libpod, specGen, false, nil)
 		if err != nil {
 			return nil, err
 		}
@@ -435,7 +435,7 @@ func (ic *ContainerEngine) playKubePod(ctx context.Context, podName string, podY
 			if err != nil {
 				return nil, err
 			}
-			rtSpec, spec, opts, err := generate.MakeContainer(ctx, ic.Libpod, specGen)
+			rtSpec, spec, opts, err := generate.MakeContainer(ctx, ic.Libpod, specGen, false, nil)
 			if err != nil {
 				return nil, err
 			}
diff --git a/pkg/domain/infra/tunnel/containers.go b/pkg/domain/infra/tunnel/containers.go
index 67e709486..aa4baf846 100644
--- a/pkg/domain/infra/tunnel/containers.go
+++ b/pkg/domain/infra/tunnel/containers.go
@@ -958,3 +958,7 @@ func (ic *ContainerEngine) ShouldRestart(_ context.Context, id string) (bool, er
 func (ic *ContainerEngine) ContainerRename(ctx context.Context, nameOrID string, opts entities.ContainerRenameOptions) error {
 	return containers.Rename(ic.ClientCtx, nameOrID, new(containers.RenameOptions).WithName(opts.NewName))
 }
+
+func (ic *ContainerEngine) ContainerClone(ctx context.Context, ctrCloneOpts entities.ContainerCloneOptions) (*entities.ContainerCreateReport, error) {
+	return nil, errors.New("cloning a container is not supported on the remote client")
+}
diff --git a/pkg/specgen/generate/container.go b/pkg/specgen/generate/container.go
index a4d862a60..64669f34d 100644
--- a/pkg/specgen/generate/container.go
+++ b/pkg/specgen/generate/container.go
@@ -2,6 +2,7 @@ package generate
 
 import (
 	"context"
+	"encoding/json"
 	"os"
 	"strings"
 	"time"
@@ -335,3 +336,152 @@ func FinishThrottleDevices(s *specgen.SpecGenerator) error {
 	}
 	return nil
 }
+
+// ConfigToSpec takes a completed container config and converts it back into a specgenerator for purposes of cloning an exisiting container
+func ConfigToSpec(rt *libpod.Runtime, specg *specgen.SpecGenerator, contaierID string) (*libpod.Container, error) {
+	c, err := rt.LookupContainer(contaierID)
+	if err != nil {
+		return nil, err
+	}
+	conf := c.Config()
+
+	tmpSystemd := conf.Systemd
+	tmpMounts := conf.Mounts
+
+	conf.Systemd = nil
+	conf.Mounts = []string{}
+
+	specg.Pod = conf.Pod
+
+	matching, err := json.Marshal(conf)
+	if err != nil {
+		return nil, err
+	}
+
+	err = json.Unmarshal(matching, specg)
+	if err != nil {
+		return nil, err
+	}
+	conf.Systemd = tmpSystemd
+	conf.Mounts = tmpMounts
+
+	if conf.Spec != nil && conf.Spec.Linux != nil && conf.Spec.Linux.Resources != nil {
+		if specg.ResourceLimits == nil {
+			specg.ResourceLimits = conf.Spec.Linux.Resources
+		}
+	}
+
+	nameSpaces := []string{"pid", "net", "cgroup", "ipc", "uts", "user"}
+	containers := []string{conf.PIDNsCtr, conf.NetNsCtr, conf.CgroupNsCtr, conf.IPCNsCtr, conf.UTSNsCtr, conf.UserNsCtr}
+	place := []*specgen.Namespace{&specg.PidNS, &specg.NetNS, &specg.CgroupNS, &specg.IpcNS, &specg.UtsNS, &specg.UserNS}
+	for i, ns := range containers {
+		if len(ns) > 0 {
+			ns := specgen.Namespace{NSMode: specgen.FromContainer, Value: ns}
+			place[i] = &ns
+		} else {
+			switch nameSpaces[i] {
+			case "pid":
+				specg.PidNS = specgen.Namespace{NSMode: specgen.Default} //default
+			case "net":
+				switch {
+				case conf.NetMode.IsBridge():
+					toExpose := make(map[uint16]string, len(conf.ExposedPorts))
+					for _, expose := range []map[uint16][]string{conf.ExposedPorts} {
+						for port, proto := range expose {
+							toExpose[port] = strings.Join(proto, ",")
+						}
+					}
+					specg.Expose = toExpose
+					specg.PortMappings = conf.PortMappings
+					specg.NetNS = specgen.Namespace{NSMode: specgen.Bridge}
+				case conf.NetMode.IsSlirp4netns():
+					toExpose := make(map[uint16]string, len(conf.ExposedPorts))
+					for _, expose := range []map[uint16][]string{conf.ExposedPorts} {
+						for port, proto := range expose {
+							toExpose[port] = strings.Join(proto, ",")
+						}
+					}
+					specg.Expose = toExpose
+					specg.PortMappings = conf.PortMappings
+					netMode := strings.Split(string(conf.NetMode), ":")
+					var val string
+					if len(netMode) > 1 {
+						val = netMode[1]
+					}
+					specg.NetNS = specgen.Namespace{NSMode: specgen.Slirp, Value: val}
+				case conf.NetMode.IsPrivate():
+					specg.NetNS = specgen.Namespace{NSMode: specgen.Private}
+				case conf.NetMode.IsDefault():
+					specg.NetNS = specgen.Namespace{NSMode: specgen.Default}
+				case conf.NetMode.IsUserDefined():
+					specg.NetNS = specgen.Namespace{NSMode: specgen.Path, Value: strings.Split(string(conf.NetMode), ":")[1]}
+				case conf.NetMode.IsContainer():
+					specg.NetNS = specgen.Namespace{NSMode: specgen.FromContainer, Value: strings.Split(string(conf.NetMode), ":")[1]}
+				case conf.NetMode.IsPod():
+					specg.NetNS = specgen.Namespace{NSMode: specgen.FromPod, Value: strings.Split(string(conf.NetMode), ":")[1]}
+				}
+			case "cgroup":
+				specg.CgroupNS = specgen.Namespace{NSMode: specgen.Default} //default
+			case "ipc":
+				if conf.ShmDir == "/dev/shm" {
+					specg.IpcNS = specgen.Namespace{NSMode: specgen.Host}
+				} else {
+					specg.IpcNS = specgen.Namespace{NSMode: specgen.Default} //default
+				}
+			case "uts":
+				specg.UtsNS = specgen.Namespace{NSMode: specgen.Default} //default
+			case "user":
+				if conf.AddCurrentUserPasswdEntry {
+					specg.UserNS = specgen.Namespace{NSMode: specgen.KeepID}
+				} else {
+					specg.UserNS = specgen.Namespace{NSMode: specgen.Default} //default
+				}
+			}
+		}
+	}
+
+	specg.IDMappings = &conf.IDMappings
+	specg.ContainerCreateCommand = conf.CreateCommand
+	if len(specg.Rootfs) == 0 {
+		specg.Rootfs = conf.Rootfs
+	}
+	if len(specg.Image) == 0 {
+		specg.Image = conf.RootfsImageID
+	}
+	var named []*specgen.NamedVolume
+	if len(conf.NamedVolumes) != 0 {
+		for _, v := range conf.NamedVolumes {
+			named = append(named, &specgen.NamedVolume{
+				Name:    v.Name,
+				Dest:    v.Dest,
+				Options: v.Options,
+			})
+		}
+	}
+	specg.Volumes = named
+	var image []*specgen.ImageVolume
+	if len(conf.ImageVolumes) != 0 {
+		for _, v := range conf.ImageVolumes {
+			image = append(image, &specgen.ImageVolume{
+				Source:      v.Source,
+				Destination: v.Dest,
+				ReadWrite:   v.ReadWrite,
+			})
+		}
+	}
+	specg.ImageVolumes = image
+	var overlay []*specgen.OverlayVolume
+	if len(conf.OverlayVolumes) != 0 {
+		for _, v := range conf.OverlayVolumes {
+			overlay = append(overlay, &specgen.OverlayVolume{
+				Source:      v.Source,
+				Destination: v.Dest,
+				Options:     v.Options,
+			})
+		}
+	}
+	specg.OverlayVolumes = overlay
+	specg.Mounts = conf.Spec.Mounts
+	specg.HostDeviceList = conf.DeviceHostSrc
+	return c, nil
+}
diff --git a/pkg/specgen/generate/container_create.go b/pkg/specgen/generate/container_create.go
index a7a7353d0..c0b23953f 100644
--- a/pkg/specgen/generate/container_create.go
+++ b/pkg/specgen/generate/container_create.go
@@ -8,6 +8,7 @@ import (
 
 	cdi "github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
 	"github.com/containers/common/libimage"
+	"github.com/containers/common/pkg/cgroups"
 	"github.com/containers/podman/v4/libpod"
 	"github.com/containers/podman/v4/libpod/define"
 	"github.com/containers/podman/v4/pkg/namespaces"
@@ -22,7 +23,7 @@ import (
 // MakeContainer creates a container based on the SpecGenerator.
 // Returns the created, container and any warnings resulting from creating the
 // container, or an error.
-func MakeContainer(ctx context.Context, rt *libpod.Runtime, s *specgen.SpecGenerator) (*spec.Spec, *specgen.SpecGenerator, []libpod.CtrCreateOption, error) {
+func MakeContainer(ctx context.Context, rt *libpod.Runtime, s *specgen.SpecGenerator, clone bool, c *libpod.Container) (*spec.Spec, *specgen.SpecGenerator, []libpod.CtrCreateOption, error) {
 	rtc, err := rt.GetConfigNoCopy()
 	if err != nil {
 		return nil, nil, nil, err
@@ -170,6 +171,42 @@ func MakeContainer(ctx context.Context, rt *libpod.Runtime, s *specgen.SpecGener
 		options = append(options, opts...)
 	}
 	runtimeSpec, err := SpecGenToOCI(ctx, s, rt, rtc, newImage, finalMounts, pod, command, compatibleOptions)
+	if clone { // the container fails to start if cloned due to missing Linux spec entries
+		if c == nil {
+			return nil, nil, nil, errors.New("the given container could not be retrieved")
+		}
+		conf := c.Config()
+		out, err := json.Marshal(conf.Spec.Linux)
+		if err != nil {
+			return nil, nil, nil, err
+		}
+		err = json.Unmarshal(out, runtimeSpec.Linux)
+		if err != nil {
+			return nil, nil, nil, err
+		}
+
+		switch {
+		case s.ResourceLimits.CPU != nil:
+			runtimeSpec.Linux.Resources.CPU = s.ResourceLimits.CPU
+		case s.ResourceLimits.Memory != nil:
+			runtimeSpec.Linux.Resources.Memory = s.ResourceLimits.Memory
+		case s.ResourceLimits.BlockIO != nil:
+			runtimeSpec.Linux.Resources.BlockIO = s.ResourceLimits.BlockIO
+		case s.ResourceLimits.Devices != nil:
+			runtimeSpec.Linux.Resources.Devices = s.ResourceLimits.Devices
+		}
+
+		cgroup2, err := cgroups.IsCgroup2UnifiedMode()
+		if err != nil {
+			return nil, nil, nil, err
+		}
+		if cgroup2 && s.ResourceLimits.Memory != nil && s.ResourceLimits.Memory.Swappiness != nil { // conf.Spec.Linux contains memory swappiness established after the spec process we need to remove that
+			s.ResourceLimits.Memory.Swappiness = nil
+			if runtimeSpec.Linux.Resources.Memory != nil {
+				runtimeSpec.Linux.Resources.Memory.Swappiness = nil
+			}
+		}
+	}
 	if err != nil {
 		return nil, nil, nil, err
 	}
diff --git a/pkg/specgen/generate/oci.go b/pkg/specgen/generate/oci.go
index 945c994ea..8b3550e36 100644
--- a/pkg/specgen/generate/oci.go
+++ b/pkg/specgen/generate/oci.go
@@ -301,7 +301,14 @@ func SpecGenToOCI(ctx context.Context, s *specgen.SpecGenerator, rt *libpod.Runt
 	}
 
 	if compatibleOptions.InfraResources == nil && s.ResourceLimits != nil {
-		g.Config.Linux.Resources = s.ResourceLimits
+		out, err := json.Marshal(s.ResourceLimits)
+		if err != nil {
+			return nil, err
+		}
+		err = json.Unmarshal(out, g.Config.Linux.Resources)
+		if err != nil {
+			return nil, err
+		}
 	} else if s.ResourceLimits != nil { // if we have predefined resource limits we need to make sure we keep the infra and container limits
 		originalResources, err := json.Marshal(s.ResourceLimits)
 		if err != nil {
diff --git a/pkg/specgen/generate/pod_create.go b/pkg/specgen/generate/pod_create.go
index 68fda3ad7..8450fe7ce 100644
--- a/pkg/specgen/generate/pod_create.go
+++ b/pkg/specgen/generate/pod_create.go
@@ -135,7 +135,7 @@ func MakePod(p *entities.PodSpec, rt *libpod.Runtime) (*libpod.Pod, error) {
 			return nil, err
 		}
 		p.PodSpecGen.InfraContainerSpec.User = "" // infraSpec user will get incorrectly assigned via the container creation process, overwrite here
-		rtSpec, spec, opts, err := MakeContainer(context.Background(), rt, p.PodSpecGen.InfraContainerSpec)
+		rtSpec, spec, opts, err := MakeContainer(context.Background(), rt, p.PodSpecGen.InfraContainerSpec, false, nil)
 		if err != nil {
 			return nil, err
 		}
diff --git a/pkg/specgenutil/createparse.go b/pkg/specgenutil/createparse.go
index 9666e4be0..a51396227 100644
--- a/pkg/specgenutil/createparse.go
+++ b/pkg/specgenutil/createparse.go
@@ -26,6 +26,8 @@ func validate(c *entities.ContainerCreateOptions) error {
 	if _, ok := imageVolType[c.ImageVolume]; !ok {
 		if c.IsInfra {
 			c.ImageVolume = "bind"
+		} else if c.IsClone { // the image volume type will be deduced later from the container we are cloning
+			return nil
 		} else {
 			return errors.Errorf("invalid image-volume type %q. Pick one of bind, tmpfs, or ignore", c.ImageVolume)
 		}
diff --git a/pkg/specgenutil/specgen.go b/pkg/specgenutil/specgen.go
index 17699a038..b037e14cc 100644
--- a/pkg/specgenutil/specgen.go
+++ b/pkg/specgenutil/specgen.go
@@ -256,38 +256,43 @@ func FillOutSpecGen(s *specgen.SpecGenerator, c *entities.ContainerCreateOptions
 	if err := setNamespaces(s, c); err != nil {
 		return err
 	}
-	userNS := namespaces.UsernsMode(s.UserNS.NSMode)
-	tempIDMap, err := util.ParseIDMapping(namespaces.UsernsMode(c.UserNS), []string{}, []string{}, "", "")
-	if err != nil {
-		return err
-	}
-	s.IDMappings, err = util.ParseIDMapping(userNS, c.UIDMap, c.GIDMap, c.SubUIDName, c.SubGIDName)
-	if err != nil {
-		return err
-	}
-	if len(s.IDMappings.GIDMap) == 0 {
-		s.IDMappings.AutoUserNsOpts.AdditionalGIDMappings = tempIDMap.AutoUserNsOpts.AdditionalGIDMappings
-		if s.UserNS.NSMode == specgen.NamespaceMode("auto") {
-			s.IDMappings.AutoUserNs = true
+
+	if s.IDMappings == nil {
+		userNS := namespaces.UsernsMode(s.UserNS.NSMode)
+		tempIDMap, err := util.ParseIDMapping(namespaces.UsernsMode(c.UserNS), []string{}, []string{}, "", "")
+		if err != nil {
+			return err
 		}
-	}
-	if len(s.IDMappings.UIDMap) == 0 {
-		s.IDMappings.AutoUserNsOpts.AdditionalUIDMappings = tempIDMap.AutoUserNsOpts.AdditionalUIDMappings
-		if s.UserNS.NSMode == specgen.NamespaceMode("auto") {
-			s.IDMappings.AutoUserNs = true
+		s.IDMappings, err = util.ParseIDMapping(userNS, c.UIDMap, c.GIDMap, c.SubUIDName, c.SubGIDName)
+		if err != nil {
+			return err
+		}
+		if len(s.IDMappings.GIDMap) == 0 {
+			s.IDMappings.AutoUserNsOpts.AdditionalGIDMappings = tempIDMap.AutoUserNsOpts.AdditionalGIDMappings
+			if s.UserNS.NSMode == specgen.NamespaceMode("auto") {
+				s.IDMappings.AutoUserNs = true
+			}
+		}
+		if len(s.IDMappings.UIDMap) == 0 {
+			s.IDMappings.AutoUserNsOpts.AdditionalUIDMappings = tempIDMap.AutoUserNsOpts.AdditionalUIDMappings
+			if s.UserNS.NSMode == specgen.NamespaceMode("auto") {
+				s.IDMappings.AutoUserNs = true
+			}
+		}
+		if tempIDMap.AutoUserNsOpts.Size != 0 {
+			s.IDMappings.AutoUserNsOpts.Size = tempIDMap.AutoUserNsOpts.Size
+		}
+		// If some mappings are specified, assume a private user namespace
+		if userNS.IsDefaultValue() && (!s.IDMappings.HostUIDMapping || !s.IDMappings.HostGIDMapping) {
+			s.UserNS.NSMode = specgen.Private
+		} else {
+			s.UserNS.NSMode = specgen.NamespaceMode(userNS)
 		}
-	}
-	if tempIDMap.AutoUserNsOpts.Size != 0 {
-		s.IDMappings.AutoUserNsOpts.Size = tempIDMap.AutoUserNsOpts.Size
-	}
-	// If some mappings are specified, assume a private user namespace
-	if userNS.IsDefaultValue() && (!s.IDMappings.HostUIDMapping || !s.IDMappings.HostGIDMapping) {
-		s.UserNS.NSMode = specgen.Private
-	} else {
-		s.UserNS.NSMode = specgen.NamespaceMode(userNS)
 	}
 
-	s.Terminal = c.TTY
+	if !s.Terminal {
+		s.Terminal = c.TTY
+	}
 
 	if err := verifyExpose(c.Expose); err != nil {
 		return err
@@ -297,8 +302,13 @@ func FillOutSpecGen(s *specgen.SpecGenerator, c *entities.ContainerCreateOptions
 	if c.Net != nil {
 		s.PortMappings = c.Net.PublishPorts
 	}
-	s.PublishExposedPorts = c.PublishAll
-	s.Pod = c.Pod
+	if !s.PublishExposedPorts {
+		s.PublishExposedPorts = c.PublishAll
+	}
+
+	if len(s.Pod) == 0 {
+		s.Pod = c.Pod
+	}
 
 	if len(c.PodIDFile) > 0 {
 		if len(s.Pod) > 0 {
@@ -315,7 +325,10 @@ func FillOutSpecGen(s *specgen.SpecGenerator, c *entities.ContainerCreateOptions
 	if err != nil {
 		return err
 	}
-	s.Expose = expose
+
+	if len(s.Expose) == 0 {
+		s.Expose = expose
+	}
 
 	if sig := c.StopSignal; len(sig) > 0 {
 		stopSignal, err := util.ParseSignal(sig)
@@ -341,8 +354,13 @@ func FillOutSpecGen(s *specgen.SpecGenerator, c *entities.ContainerCreateOptions
 		return errors.Wrap(err, "error parsing host environment variables")
 	}
 
-	s.EnvHost = c.EnvHost
-	s.HTTPProxy = c.HTTPProxy
+	if !s.EnvHost {
+		s.EnvHost = c.EnvHost
+	}
+
+	if !s.HTTPProxy {
+		s.HTTPProxy = c.HTTPProxy
+	}
 
 	// env-file overrides any previous variables
 	for _, f := range c.EnvFile {
@@ -359,7 +377,9 @@ func FillOutSpecGen(s *specgen.SpecGenerator, c *entities.ContainerCreateOptions
 		return err
 	}
 
-	s.Env = envLib.Join(env, parsedEnv)
+	if len(s.Env) == 0 {
+		s.Env = envLib.Join(env, parsedEnv)
+	}
 
 	// LABEL VARIABLES
 	labels, err := parse.GetAllLabels(c.LabelFile, c.Label)
@@ -371,7 +391,9 @@ func FillOutSpecGen(s *specgen.SpecGenerator, c *entities.ContainerCreateOptions
 		labels[systemdDefine.EnvVariable] = systemdUnit
 	}
 
-	s.Labels = labels
+	if len(s.Labels) == 0 {
+		s.Labels = labels
+	}
 
 	// ANNOTATIONS
 	annotations := make(map[string]string)
@@ -390,7 +412,9 @@ func FillOutSpecGen(s *specgen.SpecGenerator, c *entities.ContainerCreateOptions
 		}
 		annotations[splitAnnotation[0]] = splitAnnotation[1]
 	}
-	s.Annotations = annotations
+	if len(s.Annotations) == 0 {
+		s.Annotations = annotations
+	}
 
 	if len(c.StorageOpts) > 0 {
 		opts := make(map[string]string, len(c.StorageOpts))
@@ -403,7 +427,9 @@ func FillOutSpecGen(s *specgen.SpecGenerator, c *entities.ContainerCreateOptions
 		}
 		s.StorageOpts = opts
 	}
-	s.WorkDir = c.Workdir
+	if len(s.WorkDir) == 0 {
+		s.WorkDir = c.Workdir
+	}
 	if c.Entrypoint != nil {
 		entrypoint := []string{}
 		// Check if entrypoint specified is json
@@ -415,7 +441,9 @@ func FillOutSpecGen(s *specgen.SpecGenerator, c *entities.ContainerCreateOptions
 
 	// Include the command used to create the container.
 
-	s.ContainerCreateCommand = os.Args
+	if len(s.ContainerCreateCommand) == 0 {
+		s.ContainerCreateCommand = os.Args
+	}
 
 	if len(inputCommand) > 0 {
 		s.Command = inputCommand
@@ -444,24 +472,37 @@ func FillOutSpecGen(s *specgen.SpecGenerator, c *entities.ContainerCreateOptions
 		s.NetworkOptions = c.Net.NetworkOptions
 		s.UseImageHosts = c.Net.NoHosts
 	}
-	s.HostUsers = c.HostUsers
-	s.ImageVolumeMode = c.ImageVolume
+	if len(s.HostUsers) == 0 || len(c.HostUsers) != 0 {
+		s.HostUsers = c.HostUsers
+	}
+	if len(s.ImageVolumeMode) == 0 || len(c.ImageVolume) != 0 {
+		s.ImageVolumeMode = c.ImageVolume
+	}
 	if s.ImageVolumeMode == "bind" {
 		s.ImageVolumeMode = "anonymous"
 	}
 
-	s.Systemd = strings.ToLower(c.Systemd)
-	s.SdNotifyMode = c.SdNotifyMode
+	if len(s.Systemd) == 0 || len(c.Systemd) != 0 {
+		s.Systemd = strings.ToLower(c.Systemd)
+	}
+	if len(s.SdNotifyMode) == 0 || len(c.SdNotifyMode) != 0 {
+		s.SdNotifyMode = c.SdNotifyMode
+	}
 	if s.ResourceLimits == nil {
 		s.ResourceLimits = &specs.LinuxResources{}
 	}
-	s.ResourceLimits.Memory, err = getMemoryLimits(s, c)
-	if err != nil {
-		return err
+
+	if s.ResourceLimits.Memory == nil || (len(c.Memory) != 0 || len(c.MemoryReservation) != 0 || len(c.MemorySwap) != 0 || c.MemorySwappiness != 0) {
+		s.ResourceLimits.Memory, err = getMemoryLimits(s, c)
+		if err != nil {
+			return err
+		}
 	}
-	s.ResourceLimits.BlockIO, err = getIOLimits(s, c)
-	if err != nil {
-		return err
+	if s.ResourceLimits.BlockIO == nil || (len(c.BlkIOWeight) != 0 || len(c.BlkIOWeightDevice) != 0) {
+		s.ResourceLimits.BlockIO, err = getIOLimits(s, c)
+		if err != nil {
+			return err
+		}
 	}
 	if c.PIDsLimit != nil {
 		pids := specs.LinuxPids{
@@ -470,7 +511,10 @@ func FillOutSpecGen(s *specgen.SpecGenerator, c *entities.ContainerCreateOptions
 
 		s.ResourceLimits.Pids = &pids
 	}
-	s.ResourceLimits.CPU = getCPULimits(c)
+
+	if s.ResourceLimits.CPU == nil || (c.CPUPeriod != 0 || c.CPUQuota != 0 || c.CPURTPeriod != 0 || c.CPURTRuntime != 0 || c.CPUS != 0 || len(c.CPUSetCPUs) != 0 || len(c.CPUSetMems) != 0 || c.CPUShares != 0) {
+		s.ResourceLimits.CPU = getCPULimits(c)
+	}
 
 	unifieds := make(map[string]string)
 	for _, unified := range c.CgroupConf {
@@ -495,8 +539,12 @@ func FillOutSpecGen(s *specgen.SpecGenerator, c *entities.ContainerCreateOptions
 	if ld := c.LogDriver; len(ld) > 0 {
 		s.LogConfiguration.Driver = ld
 	}
-	s.CgroupParent = c.CgroupParent
-	s.CgroupsMode = c.CgroupsMode
+	if len(s.CgroupParent) == 0 || len(c.CgroupParent) != 0 {
+		s.CgroupParent = c.CgroupParent
+	}
+	if len(s.CgroupsMode) == 0 {
+		s.CgroupsMode = c.CgroupsMode
+	}
 	if s.CgroupsMode == "" {
 		rtc, err := config.Default()
 		if err != nil {
@@ -506,9 +554,13 @@ func FillOutSpecGen(s *specgen.SpecGenerator, c *entities.ContainerCreateOptions
 		s.CgroupsMode = rtc.Cgroups()
 	}
 
-	s.Groups = c.GroupAdd
+	if len(s.Groups) == 0 || len(c.GroupAdd) != 0 {
+		s.Groups = c.GroupAdd
+	}
 
-	s.Hostname = c.Hostname
+	if len(s.Hostname) == 0 || len(c.Hostname) != 0 {
+		s.Hostname = c.Hostname
+	}
 	sysctl := map[string]string{}
 	if ctl := c.Sysctl; len(ctl) > 0 {
 		sysctl, err = util.ValidateSysctls(ctl)
@@ -516,15 +568,29 @@ func FillOutSpecGen(s *specgen.SpecGenerator, c *entities.ContainerCreateOptions
 			return err
 		}
 	}
-	s.Sysctl = sysctl
+	if len(s.Sysctl) == 0 || len(c.Sysctl) != 0 {
+		s.Sysctl = sysctl
+	}
 
-	s.CapAdd = c.CapAdd
-	s.CapDrop = c.CapDrop
-	s.Privileged = c.Privileged
-	s.ReadOnlyFilesystem = c.ReadOnly
-	s.ConmonPidFile = c.ConmonPIDFile
+	if len(s.CapAdd) == 0 || len(c.CapAdd) != 0 {
+		s.CapAdd = c.CapAdd
+	}
+	if len(s.CapDrop) == 0 || len(c.CapDrop) != 0 {
+		s.CapDrop = c.CapDrop
+	}
+	if !s.Privileged {
+		s.Privileged = c.Privileged
+	}
+	if !s.ReadOnlyFilesystem {
+		s.ReadOnlyFilesystem = c.ReadOnly
+	}
+	if len(s.ConmonPidFile) == 0 || len(c.ConmonPIDFile) != 0 {
+		s.ConmonPidFile = c.ConmonPIDFile
+	}
 
-	s.DependencyContainers = c.Requires
+	if len(s.DependencyContainers) == 0 || len(c.Requires) != 0 {
+		s.DependencyContainers = c.Requires
+	}
 
 	// TODO
 	// outside of specgen and oci though
@@ -540,7 +606,9 @@ func FillOutSpecGen(s *specgen.SpecGenerator, c *entities.ContainerCreateOptions
 		}
 		sysmap[splitCtl[0]] = splitCtl[1]
 	}
-	s.Sysctl = sysmap
+	if len(s.Sysctl) == 0 || len(c.Sysctl) != 0 {
+		s.Sysctl = sysmap
+	}
 
 	if c.CIDFile != "" {
 		s.Annotations[define.InspectAnnotationCIDFile] = c.CIDFile
@@ -584,9 +652,13 @@ func FillOutSpecGen(s *specgen.SpecGenerator, c *entities.ContainerCreateOptions
 		}
 	}
 
-	s.SeccompPolicy = c.SeccompPolicy
+	if len(s.SeccompPolicy) == 0 || len(c.SeccompPolicy) != 0 {
+		s.SeccompPolicy = c.SeccompPolicy
+	}
 
-	s.VolumesFrom = c.VolumesFrom
+	if len(s.VolumesFrom) == 0 || len(c.VolumesFrom) != 0 {
+		s.VolumesFrom = c.VolumesFrom
+	}
 
 	// Only add read-only tmpfs mounts in case that we are read-only and the
 	// read-only tmpfs flag has been set.
@@ -594,10 +666,19 @@ func FillOutSpecGen(s *specgen.SpecGenerator, c *entities.ContainerCreateOptions
 	if err != nil {
 		return err
 	}
-	s.Mounts = mounts
-	s.Volumes = volumes
-	s.OverlayVolumes = overlayVolumes
-	s.ImageVolumes = imageVolumes
+	if len(s.Mounts) == 0 || len(c.Mount) != 0 {
+		s.Mounts = mounts
+	}
+	if len(s.Volumes) == 0 || len(c.Volume) != 0 {
+		s.Volumes = volumes
+	}
+	// TODO make sure these work in clone
+	if len(s.OverlayVolumes) == 0 {
+		s.OverlayVolumes = overlayVolumes
+	}
+	if len(s.ImageVolumes) == 0 {
+		s.ImageVolumes = imageVolumes
+	}
 
 	for _, dev := range c.Devices {
 		s.Devices = append(s.Devices, specs.LinuxDevice{Path: dev})
@@ -611,9 +692,15 @@ func FillOutSpecGen(s *specgen.SpecGenerator, c *entities.ContainerCreateOptions
 		s.DeviceCgroupRule = append(s.DeviceCgroupRule, dev)
 	}
 
-	s.Init = c.Init
-	s.InitPath = c.InitPath
-	s.Stdin = c.Interactive
+	if !s.Init {
+		s.Init = c.Init
+	}
+	if len(s.InitPath) == 0 || len(c.InitPath) != 0 {
+		s.InitPath = c.InitPath
+	}
+	if !s.Stdin {
+		s.Stdin = c.Interactive
+	}
 	// quiet
 	// DeviceCgroupRules: c.StringSlice("device-cgroup-rule"),
 
@@ -656,11 +743,19 @@ func FillOutSpecGen(s *specgen.SpecGenerator, c *entities.ContainerCreateOptions
 			logOpts[split[0]] = split[1]
 		}
 	}
-	s.LogConfiguration.Options = logOpts
-	s.Name = c.Name
-	s.PreserveFDs = c.PreserveFDs
+	if len(s.LogConfiguration.Options) == 0 || len(c.LogOptions) != 0 {
+		s.LogConfiguration.Options = logOpts
+	}
+	if len(s.Name) == 0 || len(c.Name) != 0 {
+		s.Name = c.Name
+	}
+	if s.PreserveFDs == 0 || c.PreserveFDs != 0 {
+		s.PreserveFDs = c.PreserveFDs
+	}
 
-	s.OOMScoreAdj = &c.OOMScoreAdj
+	if s.OOMScoreAdj == nil || c.OOMScoreAdj != 0 {
+		s.OOMScoreAdj = &c.OOMScoreAdj
+	}
 	if c.Restart != "" {
 		splitRestart := strings.Split(c.Restart, ":")
 		switch len(splitRestart) {
@@ -685,9 +780,11 @@ func FillOutSpecGen(s *specgen.SpecGenerator, c *entities.ContainerCreateOptions
 		s.RestartPolicy = splitRestart[0]
 	}
 
-	s.Secrets, s.EnvSecrets, err = parseSecrets(c.Secrets)
-	if err != nil {
-		return err
+	if len(s.Secrets) == 0 || len(c.Secrets) != 0 {
+		s.Secrets, s.EnvSecrets, err = parseSecrets(c.Secrets)
+		if err != nil {
+			return err
+		}
 	}
 
 	if c.Personality != "" {
@@ -695,21 +792,43 @@ func FillOutSpecGen(s *specgen.SpecGenerator, c *entities.ContainerCreateOptions
 		s.Personality.Domain = specs.LinuxPersonalityDomain(c.Personality)
 	}
 
-	s.Remove = c.Rm
-	s.StopTimeout = &c.StopTimeout
-	s.Timeout = c.Timeout
-	s.Timezone = c.Timezone
-	s.Umask = c.Umask
-	s.PidFile = c.PidFile
-	s.Volatile = c.Rm
-	s.UnsetEnv = c.UnsetEnv
-	s.UnsetEnvAll = c.UnsetEnvAll
+	if !s.Remove {
+		s.Remove = c.Rm
+	}
+	if s.StopTimeout == nil || c.StopTimeout != 0 {
+		s.StopTimeout = &c.StopTimeout
+	}
+	if s.Timeout == 0 || c.Timeout != 0 {
+		s.Timeout = c.Timeout
+	}
+	if len(s.Timezone) == 0 || len(c.Timezone) != 0 {
+		s.Timezone = c.Timezone
+	}
+	if len(s.Umask) == 0 || len(c.Umask) != 0 {
+		s.Umask = c.Umask
+	}
+	if len(s.PidFile) == 0 || len(c.PidFile) != 0 {
+		s.PidFile = c.PidFile
+	}
+	if !s.Volatile {
+		s.Volatile = c.Rm
+	}
+	if len(s.UnsetEnv) == 0 || len(c.UnsetEnv) != 0 {
+		s.UnsetEnv = c.UnsetEnv
+	}
+	if !s.UnsetEnvAll {
+		s.UnsetEnvAll = c.UnsetEnvAll
+	}
 
 	// Initcontainers
-	s.InitContainerType = c.InitContainerType
+	if len(s.InitContainerType) == 0 || len(c.InitContainerType) != 0 {
+		s.InitContainerType = c.InitContainerType
+	}
 
 	t := true
-	s.Passwd = &t
+	if s.Passwd == nil {
+		s.Passwd = &t
+	}
 	return nil
 }