Merge pull request #9237 from mheon/backports_300_RC3

Backports for v3.0,0-RC3

17 files changed, 321 insertions, 61 deletions

diff --git a/pkg/api/handlers/compat/containers.go b/pkg/api/handlers/compat/containers.go
index 5c5586323..a8f850823 100644
--- a/pkg/api/handlers/compat/containers.go
+++ b/pkg/api/handlers/compat/containers.go
@@ -20,6 +20,7 @@ import (
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/go-connections/nat"
+	"github.com/docker/go-units"
 	"github.com/gorilla/mux"
 	"github.com/gorilla/schema"
 	"github.com/pkg/errors"
@@ -263,6 +264,7 @@ func LibpodToContainer(l *libpod.Container, sz bool) (*handlers.Container, error
 		sizeRootFs int64
 		sizeRW     int64
 		state      define.ContainerStatus
+		status     string
 	)
 
 	if state, err = l.State(); err != nil {
@@ -273,6 +275,35 @@ func LibpodToContainer(l *libpod.Container, sz bool) (*handlers.Container, error
 		stateStr = "created"
 	}
 
+	if state == define.ContainerStateConfigured || state == define.ContainerStateCreated {
+		status = "Created"
+	} else if state == define.ContainerStateStopped || state == define.ContainerStateExited {
+		exitCode, _, err := l.ExitCode()
+		if err != nil {
+			return nil, err
+		}
+		finishedTime, err := l.FinishedTime()
+		if err != nil {
+			return nil, err
+		}
+		status = fmt.Sprintf("Exited (%d) %s ago", exitCode, units.HumanDuration(time.Since(finishedTime)))
+	} else if state == define.ContainerStateRunning || state == define.ContainerStatePaused {
+		startedTime, err := l.StartedTime()
+		if err != nil {
+			return nil, err
+		}
+		status = fmt.Sprintf("Up %s", units.HumanDuration(time.Since(startedTime)))
+		if state == define.ContainerStatePaused {
+			status += " (Paused)"
+		}
+	} else if state == define.ContainerStateRemoving {
+		status = "Removal In Progress"
+	} else if state == define.ContainerStateStopping {
+		status = "Stopping"
+	} else {
+		status = "Unknown"
+	}
+
 	if sz {
 		if sizeRW, err = l.RWSize(); err != nil {
 			return nil, err
@@ -294,7 +325,7 @@ func LibpodToContainer(l *libpod.Container, sz bool) (*handlers.Container, error
 		SizeRootFs: sizeRootFs,
 		Labels:     l.Labels(),
 		State:      stateStr,
-		Status:     "",
+		Status:     status,
 		HostConfig: struct {
 			NetworkMode string `json:",omitempty"`
 		}{
diff --git a/pkg/api/handlers/compat/resize.go b/pkg/api/handlers/compat/resize.go
index cc8c6ef0a..a769ae1b5 100644
--- a/pkg/api/handlers/compat/resize.go
+++ b/pkg/api/handlers/compat/resize.go
@@ -84,5 +84,5 @@ func ResizeTTY(w http.ResponseWriter, r *http.Request) {
 		// reasons.
 		status = http.StatusCreated
 	}
-	utils.WriteResponse(w, status, "")
+	w.WriteHeader(status)
 }
diff --git a/pkg/api/handlers/libpod/pods.go b/pkg/api/handlers/libpod/pods.go
index 2409d3a20..2c35dd191 100644
--- a/pkg/api/handlers/libpod/pods.go
+++ b/pkg/api/handlers/libpod/pods.go
@@ -139,19 +139,20 @@ func PodStop(w http.ResponseWriter, r *http.Request) {
 			logrus.Errorf("Error cleaning up pod %s container %s: %v", pod.ID(), id, err)
 		}
 	}
-	var errs []error //nolint
+
+	report := entities.PodStopReport{Id: pod.ID()}
 	for id, err := range responses {
-		errs = append(errs, errors.Wrapf(err, "error stopping container %s", id))
+		report.Errs = append(report.Errs, errors.Wrapf(err, "error stopping container %s", id))
 	}
-	report := entities.PodStopReport{
-		Errs: errs,
-		Id:   pod.ID(),
+
+	code := http.StatusOK
+	if len(report.Errs) > 0 {
+		code = http.StatusConflict
 	}
-	utils.WriteResponse(w, http.StatusOK, report)
+	utils.WriteResponse(w, code, report)
 }
 
 func PodStart(w http.ResponseWriter, r *http.Request) {
-	var errs []error //nolint
 	runtime := r.Context().Value("runtime").(*libpod.Runtime)
 	name := utils.GetName(r)
 	pod, err := runtime.LookupPod(name)
@@ -168,19 +169,23 @@ func PodStart(w http.ResponseWriter, r *http.Request) {
 		utils.WriteResponse(w, http.StatusNotModified, "")
 		return
 	}
+
 	responses, err := pod.Start(r.Context())
 	if err != nil && errors.Cause(err) != define.ErrPodPartialFail {
-		utils.Error(w, "Something went wrong", http.StatusInternalServerError, err)
+		utils.Error(w, "Something went wrong", http.StatusConflict, err)
 		return
 	}
+
+	report := entities.PodStartReport{Id: pod.ID()}
 	for id, err := range responses {
-		errs = append(errs, errors.Wrapf(err, "error starting container %s", id))
+		report.Errs = append(report.Errs, errors.Wrapf(err, "error starting container "+id))
 	}
-	report := entities.PodStartReport{
-		Errs: errs,
-		Id:   pod.ID(),
+
+	code := http.StatusOK
+	if len(report.Errs) > 0 {
+		code = http.StatusConflict
 	}
-	utils.WriteResponse(w, http.StatusOK, report)
+	utils.WriteResponse(w, code, report)
 }
 
 func PodDelete(w http.ResponseWriter, r *http.Request) {
@@ -209,14 +214,11 @@ func PodDelete(w http.ResponseWriter, r *http.Request) {
 		utils.Error(w, "Something went wrong", http.StatusInternalServerError, err)
 		return
 	}
-	report := entities.PodRmReport{
-		Id: pod.ID(),
-	}
+	report := entities.PodRmReport{Id: pod.ID()}
 	utils.WriteResponse(w, http.StatusOK, report)
 }
 
 func PodRestart(w http.ResponseWriter, r *http.Request) {
-	var errs []error //nolint
 	runtime := r.Context().Value("runtime").(*libpod.Runtime)
 	name := utils.GetName(r)
 	pod, err := runtime.LookupPod(name)
@@ -229,14 +231,17 @@ func PodRestart(w http.ResponseWriter, r *http.Request) {
 		utils.Error(w, "Something went wrong", http.StatusInternalServerError, err)
 		return
 	}
+
+	report := entities.PodRestartReport{Id: pod.ID()}
 	for id, err := range responses {
-		errs = append(errs, errors.Wrapf(err, "error restarting container %s", id))
+		report.Errs = append(report.Errs, errors.Wrapf(err, "error restarting container %s", id))
 	}
-	report := entities.PodRestartReport{
-		Errs: errs,
-		Id:   pod.ID(),
+
+	code := http.StatusOK
+	if len(report.Errs) > 0 {
+		code = http.StatusConflict
 	}
-	utils.WriteResponse(w, http.StatusOK, report)
+	utils.WriteResponse(w, code, report)
 }
 
 func PodPrune(w http.ResponseWriter, r *http.Request) {
@@ -267,7 +272,6 @@ func PodPruneHelper(r *http.Request) ([]*entities.PodPruneReport, error) {
 }
 
 func PodPause(w http.ResponseWriter, r *http.Request) {
-	var errs []error //nolint
 	runtime := r.Context().Value("runtime").(*libpod.Runtime)
 	name := utils.GetName(r)
 	pod, err := runtime.LookupPod(name)
@@ -280,18 +284,20 @@ func PodPause(w http.ResponseWriter, r *http.Request) {
 		utils.Error(w, "Something went wrong", http.StatusInternalServerError, err)
 		return
 	}
+
+	report := entities.PodPauseReport{Id: pod.ID()}
 	for id, v := range responses {
-		errs = append(errs, errors.Wrapf(v, "error pausing container %s", id))
+		report.Errs = append(report.Errs, errors.Wrapf(v, "error pausing container %s", id))
 	}
-	report := entities.PodPauseReport{
-		Errs: errs,
-		Id:   pod.ID(),
+
+	code := http.StatusOK
+	if len(report.Errs) > 0 {
+		code = http.StatusConflict
 	}
-	utils.WriteResponse(w, http.StatusOK, report)
+	utils.WriteResponse(w, code, report)
 }
 
 func PodUnpause(w http.ResponseWriter, r *http.Request) {
-	var errs []error //nolint
 	runtime := r.Context().Value("runtime").(*libpod.Runtime)
 	name := utils.GetName(r)
 	pod, err := runtime.LookupPod(name)
@@ -304,14 +310,17 @@ func PodUnpause(w http.ResponseWriter, r *http.Request) {
 		utils.Error(w, "failed to pause pod", http.StatusInternalServerError, err)
 		return
 	}
+
+	report := entities.PodUnpauseReport{Id: pod.ID()}
 	for id, v := range responses {
-		errs = append(errs, errors.Wrapf(v, "error unpausing container %s", id))
+		report.Errs = append(report.Errs, errors.Wrapf(v, "error unpausing container %s", id))
 	}
-	report := entities.PodUnpauseReport{
-		Errs: errs,
-		Id:   pod.ID(),
+
+	code := http.StatusOK
+	if len(report.Errs) > 0 {
+		code = http.StatusConflict
 	}
-	utils.WriteResponse(w, http.StatusOK, &report)
+	utils.WriteResponse(w, code, &report)
 }
 
 func PodTop(w http.ResponseWriter, r *http.Request) {
@@ -361,7 +370,6 @@ func PodKill(w http.ResponseWriter, r *http.Request) {
 		runtime = r.Context().Value("runtime").(*libpod.Runtime)
 		decoder = r.Context().Value("decoder").(*schema.Decoder)
 		signal  = "SIGKILL"
-		errs    []error //nolint
 	)
 	query := struct {
 		Signal string `schema:"signal"`
@@ -413,16 +421,18 @@ func PodKill(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	report := &entities.PodKillReport{Id: pod.ID()}
 	for _, v := range responses {
 		if v != nil {
-			errs = append(errs, v)
+			report.Errs = append(report.Errs, v)
 		}
 	}
-	report := &entities.PodKillReport{
-		Errs: errs,
-		Id:   pod.ID(),
+
+	code := http.StatusOK
+	if len(report.Errs) > 0 {
+		code = http.StatusConflict
 	}
-	utils.WriteResponse(w, http.StatusOK, report)
+	utils.WriteResponse(w, code, report)
 }
 
 func PodExists(w http.ResponseWriter, r *http.Request) {
diff --git a/pkg/api/handlers/utils/handler.go b/pkg/api/handlers/utils/handler.go
index 517dccad0..b3c674788 100644
--- a/pkg/api/handlers/utils/handler.go
+++ b/pkg/api/handlers/utils/handler.go
@@ -1,16 +1,17 @@
 package utils
 
 import (
-	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
 	"net/url"
 	"os"
 	"strings"
+	"unsafe"
 
 	"github.com/blang/semver"
 	"github.com/gorilla/mux"
+	jsoniter "github.com/json-iterator/go"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
@@ -43,8 +44,8 @@ var (
 	//       clients to shop for the Version they wish to support
 	APIVersion = map[VersionTree]map[VersionLevel]semver.Version{
 		LibpodTree: {
-			CurrentAPIVersion: semver.MustParse("2.0.0"),
-			MinimalAPIVersion: semver.MustParse("2.0.0"),
+			CurrentAPIVersion: semver.MustParse("3.0.0"),
+			MinimalAPIVersion: semver.MustParse("3.0.0"),
 		},
 		CompatTree: {
 			CurrentAPIVersion: semver.MustParse("1.40.0"),
@@ -144,6 +145,50 @@ func WriteResponse(w http.ResponseWriter, code int, value interface{}) {
 	}
 }
 
+func init() {
+	jsoniter.RegisterTypeEncoderFunc("error", MarshalErrorJSON, MarshalErrorJSONIsEmpty)
+	jsoniter.RegisterTypeEncoderFunc("[]error", MarshalErrorSliceJSON, MarshalErrorSliceJSONIsEmpty)
+}
+
+var json = jsoniter.ConfigCompatibleWithStandardLibrary
+
+// MarshalErrorJSON writes error to stream as string
+func MarshalErrorJSON(ptr unsafe.Pointer, stream *jsoniter.Stream) {
+	p := *((*error)(ptr))
+	if p == nil {
+		stream.WriteNil()
+	} else {
+		stream.WriteString(p.Error())
+	}
+}
+
+// MarshalErrorSliceJSON writes []error to stream as []string JSON blob
+func MarshalErrorSliceJSON(ptr unsafe.Pointer, stream *jsoniter.Stream) {
+	a := *((*[]error)(ptr))
+	switch {
+	case len(a) == 0:
+		stream.WriteNil()
+	default:
+		stream.WriteArrayStart()
+		for i, e := range a {
+			if i > 0 {
+				stream.WriteMore()
+			}
+			stream.WriteString(e.Error())
+		}
+		stream.WriteArrayEnd()
+	}
+}
+
+func MarshalErrorJSONIsEmpty(_ unsafe.Pointer) bool {
+	return false
+}
+
+func MarshalErrorSliceJSONIsEmpty(_ unsafe.Pointer) bool {
+	return false
+}
+
+// WriteJSON writes an interface value encoded as JSON to w
 func WriteJSON(w http.ResponseWriter, code int, value interface{}) {
 	// FIXME: we don't need to write the header in all/some circumstances.
 	w.Header().Set("Content-Type", "application/json")
diff --git a/pkg/api/server/register_pods.go b/pkg/api/server/register_pods.go
index 105de4ee7..4873eb926 100644
--- a/pkg/api/server/register_pods.go
+++ b/pkg/api/server/register_pods.go
@@ -43,6 +43,11 @@ func (s *APIServer) registerPodsHandlers(r *mux.Router) error {
 	//     $ref: "#/definitions/IdResponse"
 	//   400:
 	//     $ref: "#/responses/BadParamError"
+	//   409:
+	//     description: status conflict
+	//     schema:
+	//       type: string
+	//       description: message describing error
 	//   500:
 	//     $ref: "#/responses/InternalError"
 	r.Handle(VersionedPath("/libpod/pods/create"), s.APIHandler(libpod.PodCreate)).Methods(http.MethodPost)
@@ -149,7 +154,7 @@ func (s *APIServer) registerPodsHandlers(r *mux.Router) error {
 	//   404:
 	//     $ref: "#/responses/NoSuchPod"
 	//   409:
-	//     $ref: "#/responses/ConflictError"
+	//     $ref: "#/responses/PodKillReport"
 	//   500:
 	//     $ref: "#/responses/InternalError"
 	r.Handle(VersionedPath("/libpod/pods/{name}/kill"), s.APIHandler(libpod.PodKill)).Methods(http.MethodPost)
@@ -170,6 +175,8 @@ func (s *APIServer) registerPodsHandlers(r *mux.Router) error {
 	//     $ref: '#/responses/PodPauseReport'
 	//   404:
 	//     $ref: "#/responses/NoSuchPod"
+	//   409:
+	//     $ref: '#/responses/PodPauseReport'
 	//   500:
 	//     $ref: "#/responses/InternalError"
 	r.Handle(VersionedPath("/libpod/pods/{name}/pause"), s.APIHandler(libpod.PodPause)).Methods(http.MethodPost)
@@ -189,6 +196,8 @@ func (s *APIServer) registerPodsHandlers(r *mux.Router) error {
 	//     $ref: '#/responses/PodRestartReport'
 	//   404:
 	//     $ref: "#/responses/NoSuchPod"
+	//   409:
+	//     $ref: "#/responses/PodRestartReport"
 	//   500:
 	//     $ref: "#/responses/InternalError"
 	r.Handle(VersionedPath("/libpod/pods/{name}/restart"), s.APIHandler(libpod.PodRestart)).Methods(http.MethodPost)
@@ -210,6 +219,8 @@ func (s *APIServer) registerPodsHandlers(r *mux.Router) error {
 	//     $ref: "#/responses/PodAlreadyStartedError"
 	//   404:
 	//     $ref: "#/responses/NoSuchPod"
+	//   409:
+	//     $ref: '#/responses/PodStartReport'
 	//   500:
 	//     $ref: "#/responses/InternalError"
 	r.Handle(VersionedPath("/libpod/pods/{name}/start"), s.APIHandler(libpod.PodStart)).Methods(http.MethodPost)
@@ -237,6 +248,8 @@ func (s *APIServer) registerPodsHandlers(r *mux.Router) error {
 	//     $ref: "#/responses/BadParamError"
 	//   404:
 	//     $ref: "#/responses/NoSuchPod"
+	//   409:
+	//     $ref: "#/responses/PodStopReport"
 	//   500:
 	//     $ref: "#/responses/InternalError"
 	r.Handle(VersionedPath("/libpod/pods/{name}/stop"), s.APIHandler(libpod.PodStop)).Methods(http.MethodPost)
@@ -256,6 +269,8 @@ func (s *APIServer) registerPodsHandlers(r *mux.Router) error {
 	//     $ref: '#/responses/PodUnpauseReport'
 	//   404:
 	//     $ref: "#/responses/NoSuchPod"
+	//   409:
+	//     $ref: '#/responses/PodUnpauseReport'
 	//   500:
 	//     $ref: "#/responses/InternalError"
 	r.Handle(VersionedPath("/libpod/pods/{name}/unpause"), s.APIHandler(libpod.PodUnpause)).Methods(http.MethodPost)
diff --git a/pkg/bindings/containers/attach.go b/pkg/bindings/containers/attach.go
index 69ae7a32f..586cdec8c 100644
--- a/pkg/bindings/containers/attach.go
+++ b/pkg/bindings/containers/attach.go
@@ -348,7 +348,7 @@ func attachHandleResize(ctx, winCtx context.Context, winChange chan os.Signal, i
 				resizeErr = ResizeContainerTTY(ctx, id, new(ResizeTTYOptions).WithHeight(h).WithWidth(w))
 			}
 			if resizeErr != nil {
-				logrus.Warnf("failed to resize TTY: %v", err)
+				logrus.Warnf("failed to resize TTY: %v", resizeErr)
 			}
 		}
 	}
diff --git a/pkg/cgroups/cgroups.go b/pkg/cgroups/cgroups.go
index c200dd01a..285fd093a 100644
--- a/pkg/cgroups/cgroups.go
+++ b/pkg/cgroups/cgroups.go
@@ -24,6 +24,7 @@ var (
 	ErrCgroupDeleted = errors.New("cgroup deleted")
 	// ErrCgroupV1Rootless means the cgroup v1 were attempted to be used in rootless environment
 	ErrCgroupV1Rootless = errors.New("no support for CGroups V1 in rootless environments")
+	ErrStatCgroup       = errors.New("no cgroup available for gathering user statistics")
 )
 
 // CgroupControl controls a cgroup hierarchy
@@ -525,10 +526,19 @@ func (c *CgroupControl) AddPid(pid int) error {
 // Stat returns usage statistics for the cgroup
 func (c *CgroupControl) Stat() (*Metrics, error) {
 	m := Metrics{}
+	found := false
 	for _, h := range handlers {
 		if err := h.Stat(c, &m); err != nil {
-			return nil, err
+			if !os.IsNotExist(errors.Cause(err)) {
+				return nil, err
+			}
+			logrus.Warningf("Failed to retrieve cgroup stats: %v", err)
+			continue
 		}
+		found = true
+	}
+	if !found {
+		return nil, ErrStatCgroup
 	}
 	return &m, nil
 }
diff --git a/pkg/cgroups/cgroups_test.go b/pkg/cgroups/cgroups_test.go
new file mode 100644
index 000000000..54315f7be
--- /dev/null
+++ b/pkg/cgroups/cgroups_test.go
@@ -0,0 +1,32 @@
+package cgroups
+
+import (
+	"testing"
+
+	"github.com/containers/podman/v2/pkg/rootless"
+	spec "github.com/opencontainers/runtime-spec/specs-go"
+)
+
+func TestCreated(t *testing.T) {
+	// tests only works in rootless mode
+	if rootless.IsRootless() {
+		return
+	}
+
+	var resources spec.LinuxResources
+	cgr, err := New("machine.slice", &resources)
+	if err != nil {
+		t.Error(err)
+	}
+	if err := cgr.Delete(); err != nil {
+		t.Error(err)
+	}
+
+	cgr, err = NewSystemd("machine.slice")
+	if err != nil {
+		t.Error(err)
+	}
+	if err := cgr.Delete(); err != nil {
+		t.Error(err)
+	}
+}
diff --git a/pkg/specgen/generate/kube/kube.go b/pkg/specgen/generate/kube/kube.go
index e39a700eb..98ab82259 100644
--- a/pkg/specgen/generate/kube/kube.go
+++ b/pkg/specgen/generate/kube/kube.go
@@ -3,6 +3,7 @@ package kube
 import (
 	"context"
 	"fmt"
+	"net"
 	"strings"
 
 	"github.com/containers/common/pkg/parse"
@@ -44,6 +45,31 @@ func ToPodGen(ctx context.Context, podName string, podYAML *v1.PodTemplateSpec)
 	podPorts := getPodPorts(podYAML.Spec.Containers)
 	p.PortMappings = podPorts
 
+	if dnsConfig := podYAML.Spec.DNSConfig; dnsConfig != nil {
+		// name servers
+		if dnsServers := dnsConfig.Nameservers; len(dnsServers) > 0 {
+			servers := make([]net.IP, 0)
+			for _, server := range dnsServers {
+				servers = append(servers, net.ParseIP(server))
+			}
+			p.DNSServer = servers
+		}
+		// search domans
+		if domains := dnsConfig.Searches; len(domains) > 0 {
+			p.DNSSearch = domains
+		}
+		// dns options
+		if options := dnsConfig.Options; len(options) > 0 {
+			dnsOptions := make([]string, 0)
+			for _, opts := range options {
+				d := opts.Name
+				if opts.Value != nil {
+					d += ":" + *opts.Value
+				}
+				dnsOptions = append(dnsOptions, d)
+			}
+		}
+	}
 	return p, nil
 }
 
@@ -256,16 +282,16 @@ func setupSecurityContext(s *specgen.SpecGenerator, containerYAML v1.Container)
 
 	if seopt := containerYAML.SecurityContext.SELinuxOptions; seopt != nil {
 		if seopt.User != "" {
-			s.SelinuxOpts = append(s.SelinuxOpts, fmt.Sprintf("role:%s", seopt.User))
+			s.SelinuxOpts = append(s.SelinuxOpts, fmt.Sprintf("user:%s", seopt.User))
 		}
 		if seopt.Role != "" {
 			s.SelinuxOpts = append(s.SelinuxOpts, fmt.Sprintf("role:%s", seopt.Role))
 		}
 		if seopt.Type != "" {
-			s.SelinuxOpts = append(s.SelinuxOpts, fmt.Sprintf("role:%s", seopt.Type))
+			s.SelinuxOpts = append(s.SelinuxOpts, fmt.Sprintf("type:%s", seopt.Type))
 		}
 		if seopt.Level != "" {
-			s.SelinuxOpts = append(s.SelinuxOpts, fmt.Sprintf("role:%s", seopt.Level))
+			s.SelinuxOpts = append(s.SelinuxOpts, fmt.Sprintf("level:%s", seopt.Level))
 		}
 	}
 	if caps := containerYAML.SecurityContext.Capabilities; caps != nil {
diff --git a/pkg/specgen/generate/pod_create.go b/pkg/specgen/generate/pod_create.go
index 43caf0fe9..645bf7a47 100644
--- a/pkg/specgen/generate/pod_create.go
+++ b/pkg/specgen/generate/pod_create.go
@@ -102,6 +102,9 @@ func createPodOptions(p *specgen.PodSpecGenerator, rt *libpod.Runtime) ([]libpod
 	case specgen.Slirp:
 		logrus.Debugf("Pod will use slirp4netns")
 		options = append(options, libpod.WithPodSlirp4netns(p.NetworkOptions))
+	case specgen.NoNetwork:
+		logrus.Debugf("Pod will not use networking")
+		options = append(options, libpod.WithPodNoNetwork())
 	default:
 		return nil, errors.Errorf("pods presently do not support network mode %s", p.NetNS.NSMode)
 	}
diff --git a/pkg/systemd/generate/common.go b/pkg/systemd/generate/common.go
index de6751a17..e9902319c 100644
--- a/pkg/systemd/generate/common.go
+++ b/pkg/systemd/generate/common.go
@@ -60,13 +60,21 @@ func filterPodFlags(command []string) []string {
 	return processed
 }
 
-// quoteArguments makes sure that all arguments with at least one whitespace
+// escapeSystemdArguments makes sure that all arguments with at least one whitespace
 // are quoted to make sure those are interpreted as one argument instead of
-// multiple ones.
-func quoteArguments(command []string) []string {
+// multiple ones. Also make sure to escape all characters which have a special
+// meaning to systemd -> $,% and \
+// see: https://www.freedesktop.org/software/systemd/man/systemd.service.html#Command%20lines
+func escapeSystemdArguments(command []string) []string {
 	for i := range command {
+		command[i] = strings.ReplaceAll(command[i], "$", "$$")
+		command[i] = strings.ReplaceAll(command[i], "%", "%%")
 		if strings.ContainsAny(command[i], " \t") {
 			command[i] = strconv.Quote(command[i])
+		} else if strings.Contains(command[i], `\`) {
+			// strconv.Quote also escapes backslashes so
+			// we should replace only if strconv.Quote was not used
+			command[i] = strings.ReplaceAll(command[i], `\`, `\\`)
 		}
 	}
 	return command
diff --git a/pkg/systemd/generate/common_test.go b/pkg/systemd/generate/common_test.go
index d0ec5637c..a0691d1ad 100644
--- a/pkg/systemd/generate/common_test.go
+++ b/pkg/systemd/generate/common_test.go
@@ -29,7 +29,7 @@ func TestFilterPodFlags(t *testing.T) {
 	}
 }
 
-func TestQuoteArguments(t *testing.T) {
+func TestEscapeSystemdArguments(t *testing.T) {
 	tests := []struct {
 		input  []string
 		output []string
@@ -46,10 +46,46 @@ func TestQuoteArguments(t *testing.T) {
 			[]string{"foo", "bar=\"arg with\ttab\""},
 			[]string{"foo", "\"bar=\\\"arg with\\ttab\\\"\""},
 		},
+		{
+			[]string{"$"},
+			[]string{"$$"},
+		},
+		{
+			[]string{"foo", "command with dollar sign $"},
+			[]string{"foo", "\"command with dollar sign $$\""},
+		},
+		{
+			[]string{"foo", "command with two dollar signs $$"},
+			[]string{"foo", "\"command with two dollar signs $$$$\""},
+		},
+		{
+			[]string{"%"},
+			[]string{"%%"},
+		},
+		{
+			[]string{"foo", "command with percent sign %"},
+			[]string{"foo", "\"command with percent sign %%\""},
+		},
+		{
+			[]string{"foo", "command with two percent signs %%"},
+			[]string{"foo", "\"command with two percent signs %%%%\""},
+		},
+		{
+			[]string{`\`},
+			[]string{`\\`},
+		},
+		{
+			[]string{"foo", `command with backslash \`},
+			[]string{"foo", `"command with backslash \\"`},
+		},
+		{
+			[]string{"foo", `command with two backslashs \\`},
+			[]string{"foo", `"command with two backslashs \\\\"`},
+		},
 	}
 
 	for _, test := range tests {
-		quoted := quoteArguments(test.input)
+		quoted := escapeSystemdArguments(test.input)
 		assert.Equal(t, test.output, quoted)
 	}
 }
diff --git a/pkg/systemd/generate/containers.go b/pkg/systemd/generate/containers.go
index 5f52b0a77..abe159812 100644
--- a/pkg/systemd/generate/containers.go
+++ b/pkg/systemd/generate/containers.go
@@ -204,7 +204,7 @@ func executeContainerTemplate(info *containerInfo, options entities.GenerateSyst
 		startCommand := []string{info.Executable}
 		if index > 2 {
 			// include root flags
-			info.RootFlags = strings.Join(quoteArguments(info.CreateCommand[1:index-1]), " ")
+			info.RootFlags = strings.Join(escapeSystemdArguments(info.CreateCommand[1:index-1]), " ")
 			startCommand = append(startCommand, info.CreateCommand[1:index-1]...)
 		}
 		startCommand = append(startCommand,
@@ -279,7 +279,7 @@ func executeContainerTemplate(info *containerInfo, options entities.GenerateSyst
 			}
 		}
 		startCommand = append(startCommand, remainingCmd...)
-		startCommand = quoteArguments(startCommand)
+		startCommand = escapeSystemdArguments(startCommand)
 
 		info.ExecStartPre = "/bin/rm -f {{{{.PIDFile}}}} {{{{.ContainerIDFile}}}}"
 		info.ExecStart = strings.Join(startCommand, " ")
diff --git a/pkg/systemd/generate/containers_test.go b/pkg/systemd/generate/containers_test.go
index 96d95644b..be14e4c28 100644
--- a/pkg/systemd/generate/containers_test.go
+++ b/pkg/systemd/generate/containers_test.go
@@ -352,6 +352,30 @@ Type=forking
 [Install]
 WantedBy=multi-user.target default.target
 `
+
+	goodNewWithSpecialChars := `# jadda-jadda.service
+# autogenerated by Podman CI
+
+[Unit]
+Description=Podman jadda-jadda.service
+Documentation=man:podman-generate-systemd(1)
+Wants=network.target
+After=network-online.target
+
+[Service]
+Environment=PODMAN_SYSTEMD_UNIT=%n
+Restart=always
+TimeoutStopSec=70
+ExecStartPre=/bin/rm -f %t/jadda-jadda.pid %t/jadda-jadda.ctr-id
+ExecStart=/usr/bin/podman run --conmon-pidfile %t/jadda-jadda.pid --cidfile %t/jadda-jadda.ctr-id --cgroups=no-conmon -d --replace --name test awesome-image:latest sh -c "kill $$$$ && echo %%\\"
+ExecStop=/usr/bin/podman stop --ignore --cidfile %t/jadda-jadda.ctr-id -t 10
+ExecStopPost=/usr/bin/podman rm --ignore -f --cidfile %t/jadda-jadda.ctr-id
+PIDFile=%t/jadda-jadda.pid
+Type=forking
+
+[Install]
+WantedBy=multi-user.target default.target
+`
 	tests := []struct {
 		name    string
 		info    containerInfo
@@ -647,6 +671,22 @@ WantedBy=multi-user.target default.target
 			true,
 			false,
 		},
+		{"good with special chars",
+			containerInfo{
+				Executable:        "/usr/bin/podman",
+				ServiceName:       "jadda-jadda",
+				ContainerNameOrID: "jadda-jadda",
+				RestartPolicy:     "always",
+				PIDFile:           "/var/run/containers/storage/overlay-containers/639c53578af4d84b8800b4635fa4e680ee80fd67e0e6a2d4eea48d1e3230f401/userdata/conmon.pid",
+				StopTimeout:       10,
+				PodmanVersion:     "CI",
+				CreateCommand:     []string{"I'll get stripped", "create", "--name", "test", "awesome-image:latest", "sh", "-c", "kill $$ && echo %\\"},
+				EnvVariable:       EnvVariable,
+			},
+			goodNewWithSpecialChars,
+			true,
+			false,
+		},
 	}
 	for _, tt := range tests {
 		test := tt
diff --git a/pkg/systemd/generate/pods.go b/pkg/systemd/generate/pods.go
index c7e3aa955..d6ede19af 100644
--- a/pkg/systemd/generate/pods.go
+++ b/pkg/systemd/generate/pods.go
@@ -269,7 +269,7 @@ func executePodTemplate(info *podInfo, options entities.GenerateSystemdOptions)
 				return "", errors.Errorf("pod does not appear to be created via `podman pod create`: %v", info.CreateCommand)
 			}
 			podRootArgs = info.CreateCommand[1 : podCreateIndex-1]
-			info.RootFlags = strings.Join(quoteArguments(podRootArgs), " ")
+			info.RootFlags = strings.Join(escapeSystemdArguments(podRootArgs), " ")
 			podCreateArgs = filterPodFlags(info.CreateCommand[podCreateIndex+1:])
 		}
 		// We're hard-coding the first five arguments and append the
@@ -306,7 +306,7 @@ func executePodTemplate(info *podInfo, options entities.GenerateSystemdOptions)
 		}
 
 		startCommand = append(startCommand, podCreateArgs...)
-		startCommand = quoteArguments(startCommand)
+		startCommand = escapeSystemdArguments(startCommand)
 
 		info.ExecStartPre1 = "/bin/rm -f {{{{.PIDFile}}}} {{{{.PodIDFile}}}}"
 		info.ExecStartPre2 = strings.Join(startCommand, " ")
diff --git a/pkg/terminal/console_windows.go b/pkg/terminal/console_windows.go
index c7691857c..08e66cb3a 100644
--- a/pkg/terminal/console_windows.go
+++ b/pkg/terminal/console_windows.go
@@ -30,7 +30,7 @@ func setConsoleMode(handle windows.Handle, flags uint32) error {
 	if err := windows.SetConsoleMode(handle, mode|flags); err != nil {
 		// In similar code, it is not considered an error if we cannot set the
 		// console mode.  Following same line of thinking here.
-		logrus.WithError(err).Error("Failed to set console mode for cli")
+		logrus.WithError(err).Debug("Failed to set console mode for cli")
 	}
 
 	return nil
diff --git a/pkg/util/mountOpts.go b/pkg/util/mountOpts.go
index 580aaf4f2..b3a38f286 100644
--- a/pkg/util/mountOpts.go
+++ b/pkg/util/mountOpts.go
@@ -86,6 +86,10 @@ func ProcessOptions(options []string, isTmpfs bool, sourcePath string) ([]string
 				return nil, errors.Wrapf(ErrDupeMntOption, "the 'tmpcopyup' or 'notmpcopyup' option can only be set once")
 			}
 			foundCopyUp = true
+		case "consistency":
+			// Often used on MACs and mistakenly on Linux platforms.
+			// Since Docker ignores this option so shall we.
+			continue
 		case "notmpcopyup":
 			if !isTmpfs {
 				return nil, errors.Wrapf(ErrBadMntOption, "the 'notmpcopyup' option is only allowed with tmpfs mounts")