diff options
112 files changed, 3901 insertions, 1117 deletions
diff --git a/cmd/podman/common/create.go b/cmd/podman/common/create.go index fbc8fb8ab..4598e535d 100644 --- a/cmd/podman/common/create.go +++ b/cmd/podman/common/create.go @@ -410,7 +410,7 @@ func DefineCreateFlags(cmd *cobra.Command, cf *entities.ContainerCreateOptions, createFlags.StringVar( &cf.Variant, variantFlagName, "", - "Use _VARIANT_ instead of the running architecture variant for choosing images", + "Use `VARIANT` instead of the running architecture variant for choosing images", ) _ = cmd.RegisterFlagCompletionFunc(variantFlagName, completion.AutocompleteNone) diff --git a/cmd/podman/containers/port.go b/cmd/podman/containers/port.go index f309390c3..0e582ae52 100644 --- a/cmd/podman/containers/port.go +++ b/cmd/podman/containers/port.go @@ -8,7 +8,6 @@ import ( "github.com/containers/podman/v3/cmd/podman/common" "github.com/containers/podman/v3/cmd/podman/registry" "github.com/containers/podman/v3/cmd/podman/validate" - "github.com/containers/podman/v3/libpod/network/types" "github.com/containers/podman/v3/pkg/domain/entities" "github.com/pkg/errors" "github.com/spf13/cobra" @@ -73,7 +72,8 @@ func port(_ *cobra.Command, args []string) error { var ( container string err error - userPort types.OCICNIPortMapping + userPort uint16 + userProto string ) if len(args) == 0 && !portOpts.Latest && !portOpts.All { @@ -101,16 +101,12 @@ func port(_ *cobra.Command, args []string) error { fields = append(fields, "tcp") } - portNum, err := strconv.Atoi(fields[0]) + portNum, err := strconv.ParseUint(fields[0], 10, 16) if err != nil { return err } - userPort = types.OCICNIPortMapping{ - HostPort: 0, - ContainerPort: int32(portNum), - Protocol: fields[1], - HostIP: "", - } + userPort = uint16(portNum) + userProto = fields[1] } reports, err := registry.ContainerEngine().ContainerPort(registry.GetContext(), container, portOpts) @@ -120,24 +116,36 @@ func port(_ *cobra.Command, args []string) error { var found bool // Iterate mappings for _, report := range reports { + allPrefix := "" + if portOpts.All { + allPrefix = report.Id[:12] + "\t" + } for _, v := range report.Ports { hostIP := v.HostIP // Set host IP to 0.0.0.0 if blank if hostIP == "" { hostIP = "0.0.0.0" } - if portOpts.All { - fmt.Printf("%s\t", report.Id[:12]) - } - // If not searching by port or port/proto, then dump what we see - if port == "" { - fmt.Printf("%d/%s -> %s:%d\n", v.ContainerPort, v.Protocol, hostIP, v.HostPort) - continue - } - if v.ContainerPort == userPort.ContainerPort { - fmt.Printf("%s:%d\n", hostIP, v.HostPort) - found = true - break + protocols := strings.Split(v.Protocol, ",") + for _, protocol := range protocols { + // If not searching by port or port/proto, then dump what we see + if port == "" { + for i := uint16(0); i < v.Range; i++ { + fmt.Printf("%s%d/%s -> %s:%d\n", allPrefix, v.ContainerPort+i, protocol, hostIP, v.HostPort+i) + } + continue + } + // check if the proto matches and if the port is in the range + // this is faster than looping over the range for no reason + if v.Protocol == userProto && + v.ContainerPort <= userPort && + v.ContainerPort+v.Range > userPort { + // we have to add the current range to the host port + hostPort := v.HostPort + userPort - v.ContainerPort + fmt.Printf("%s%s:%d\n", allPrefix, hostIP, hostPort) + found = true + break + } } } if !found && port != "" { diff --git a/cmd/podman/containers/ps.go b/cmd/podman/containers/ps.go index 9687cd5bd..712de327c 100644 --- a/cmd/podman/containers/ps.go +++ b/cmd/podman/containers/ps.go @@ -3,8 +3,6 @@ package containers import ( "fmt" "os" - "sort" - "strconv" "strings" "time" @@ -477,174 +475,31 @@ func (l psReporter) UTS() string { // portsToString converts the ports used to a string of the from "port1, port2" // and also groups a continuous list of ports into a readable format. -func portsToString(ports []types.OCICNIPortMapping) string { +// The format is IP:HostPort(-Range)->ContainerPort(-Range)/Proto +func portsToString(ports []types.PortMapping) string { if len(ports) == 0 { return "" } - // Sort the ports, so grouping continuous ports become easy. - sort.Slice(ports, func(i, j int) bool { - return comparePorts(ports[i], ports[j]) - }) - - portGroups := [][]types.OCICNIPortMapping{} - currentGroup := []types.OCICNIPortMapping{} - for i, v := range ports { - var prevPort, nextPort *int32 - if i > 0 { - prevPort = &ports[i-1].ContainerPort - } - if i+1 < len(ports) { - nextPort = &ports[i+1].ContainerPort - } - - port := v.ContainerPort - - // Helper functions - addToCurrentGroup := func(x types.OCICNIPortMapping) { - currentGroup = append(currentGroup, x) - } - - addToPortGroup := func(x types.OCICNIPortMapping) { - portGroups = append(portGroups, []types.OCICNIPortMapping{x}) - } - - finishCurrentGroup := func() { - portGroups = append(portGroups, currentGroup) - currentGroup = []types.OCICNIPortMapping{} - } - - // Single entry slice - if prevPort == nil && nextPort == nil { - addToPortGroup(v) - } - - // Start of the slice with len > 0 - if prevPort == nil && nextPort != nil { - isGroup := *nextPort-1 == port - - if isGroup { - // Start with a group - addToCurrentGroup(v) - } else { - // Start with single item - addToPortGroup(v) - } - - continue - } - - // Middle of the slice with len > 0 - if prevPort != nil && nextPort != nil { - currentIsGroup := *prevPort+1 == port - nextIsGroup := *nextPort-1 == port - - if currentIsGroup { - // Maybe in the middle of a group - addToCurrentGroup(v) - - if !nextIsGroup { - // End of a group - finishCurrentGroup() - } - } else if nextIsGroup { - // Start of a new group - addToCurrentGroup(v) - } else { - // No group at all - addToPortGroup(v) - } - - continue - } - - // End of the slice with len > 0 - if prevPort != nil && nextPort == nil { - isGroup := *prevPort+1 == port - - if isGroup { - // End group - addToCurrentGroup(v) - finishCurrentGroup() - } else { - // End single item - addToPortGroup(v) - } - } - } - - portDisplay := []string{} - for _, group := range portGroups { - if len(group) == 0 { - // Usually should not happen, but better do not crash. - continue - } - - first := group[0] - - hostIP := first.HostIP + sb := &strings.Builder{} + for _, port := range ports { + hostIP := port.HostIP if hostIP == "" { hostIP = "0.0.0.0" } - - // Single mappings - if len(group) == 1 { - portDisplay = append(portDisplay, - fmt.Sprintf( - "%s:%d->%d/%s", - hostIP, first.HostPort, first.ContainerPort, first.Protocol, - ), - ) - continue - } - - // Group mappings - last := group[len(group)-1] - portDisplay = append(portDisplay, formatGroup( - fmt.Sprintf("%s/%s", hostIP, first.Protocol), - first.HostPort, last.HostPort, - first.ContainerPort, last.ContainerPort, - )) - } - return strings.Join(portDisplay, ", ") -} - -func comparePorts(i, j types.OCICNIPortMapping) bool { - if i.ContainerPort != j.ContainerPort { - return i.ContainerPort < j.ContainerPort - } - - if i.HostIP != j.HostIP { - return i.HostIP < j.HostIP - } - - if i.HostPort != j.HostPort { - return i.HostPort < j.HostPort - } - - return i.Protocol < j.Protocol -} - -// formatGroup returns the group in the format: -// <IP:firstHost:lastHost->firstCtr:lastCtr/Proto> -// e.g 0.0.0.0:1000-1006->2000-2006/tcp. -func formatGroup(key string, firstHost, lastHost, firstCtr, lastCtr int32) string { - parts := strings.Split(key, "/") - groupType := parts[0] - var ip string - if len(parts) > 1 { - ip = parts[0] - groupType = parts[1] - } - - group := func(first, last int32) string { - group := strconv.Itoa(int(first)) - if first != last { - group = fmt.Sprintf("%s-%d", group, last) + protocols := strings.Split(port.Protocol, ",") + for _, protocol := range protocols { + if port.Range > 1 { + fmt.Fprintf(sb, "%s:%d-%d->%d-%d/%s, ", + hostIP, port.HostPort, port.HostPort+port.Range-1, + port.ContainerPort, port.ContainerPort+port.Range-1, protocol) + } else { + fmt.Fprintf(sb, "%s:%d->%d/%s, ", + hostIP, port.HostPort, + port.ContainerPort, protocol) + } } - return group } - hostGroup := group(firstHost, lastHost) - ctrGroup := group(firstCtr, lastCtr) - - return fmt.Sprintf("%s:%s->%s/%s", ip, hostGroup, ctrGroup, groupType) + display := sb.String() + // make sure to trim the last ", " of the string + return display[:len(display)-2] } diff --git a/cmd/podman/images/pull.go b/cmd/podman/images/pull.go index a4e3515db..a990d1626 100644 --- a/cmd/podman/images/pull.go +++ b/cmd/podman/images/pull.go @@ -92,7 +92,7 @@ func pullFlags(cmd *cobra.Command) { _ = cmd.RegisterFlagCompletionFunc(osFlagName, completion.AutocompleteOS) variantFlagName := "variant" - flags.StringVar(&pullOptions.Variant, variantFlagName, "", " use VARIANT instead of the running architecture variant for choosing images") + flags.StringVar(&pullOptions.Variant, variantFlagName, "", "Use VARIANT instead of the running architecture variant for choosing images") _ = cmd.RegisterFlagCompletionFunc(variantFlagName, completion.AutocompleteNone) platformFlagName := "platform" diff --git a/cmd/podman/machine/stop.go b/cmd/podman/machine/stop.go index 76ba85601..75666f734 100644 --- a/cmd/podman/machine/stop.go +++ b/cmd/podman/machine/stop.go @@ -3,6 +3,8 @@ package machine import ( + "fmt" + "github.com/containers/podman/v3/cmd/podman/registry" "github.com/containers/podman/v3/pkg/machine" "github.com/containers/podman/v3/pkg/machine/qemu" @@ -46,5 +48,9 @@ func stop(cmd *cobra.Command, args []string) error { if err != nil { return err } - return vm.Stop(vmName, machine.StopOptions{}) + if err := vm.Stop(vmName, machine.StopOptions{}); err != nil { + return err + } + fmt.Printf("Machine %q stopped successfully\n", vmName) + return nil } diff --git a/cmd/podman/system/unshare.go b/cmd/podman/system/unshare.go index 50230609e..9b777dd8f 100644 --- a/cmd/podman/system/unshare.go +++ b/cmd/podman/system/unshare.go @@ -10,6 +10,7 @@ import ( "github.com/containers/podman/v3/pkg/rootless" "github.com/pkg/errors" "github.com/spf13/cobra" + "github.com/spf13/pflag" ) var ( @@ -34,7 +35,14 @@ func init() { }) flags := unshareCommand.Flags() flags.SetInterspersed(false) - flags.BoolVar(&unshareOptions.RootlessCNI, "rootless-cni", false, "Join the rootless network namespace used for CNI networking") + flags.BoolVar(&unshareOptions.RootlessNetNS, "rootless-netns", false, "Join the rootless network namespace used for CNI and netavark networking") + // backwards compat still allow --rootless-cni + flags.SetNormalizeFunc(func(f *pflag.FlagSet, name string) pflag.NormalizedName { + if name == "rootless-cni" { + name = "rootless-netns" + } + return pflag.NormalizedName(name) + }) } func unshare(cmd *cobra.Command, args []string) error { diff --git a/cmd/rootlessport/main.go b/cmd/rootlessport/main.go index feb9f5c06..e691ce2fc 100644 --- a/cmd/rootlessport/main.go +++ b/cmd/rootlessport/main.go @@ -10,6 +10,7 @@ import ( "os" "os/exec" "path/filepath" + "strings" "github.com/containernetworking/plugins/pkg/ns" "github.com/containers/podman/v3/libpod/network/types" @@ -289,25 +290,30 @@ func handler(ctx context.Context, conn io.Reader, pm rkport.Manager) error { return nil } -func exposePorts(pm rkport.Manager, portMappings []types.OCICNIPortMapping, childIP string) error { +func exposePorts(pm rkport.Manager, portMappings []types.PortMapping, childIP string) error { ctx := context.TODO() - for _, i := range portMappings { - hostIP := i.HostIP - if hostIP == "" { - hostIP = "0.0.0.0" - } - spec := rkport.Spec{ - Proto: i.Protocol, - ParentIP: hostIP, - ParentPort: int(i.HostPort), - ChildPort: int(i.ContainerPort), - ChildIP: childIP, - } - if err := rkportutil.ValidatePortSpec(spec, nil); err != nil { - return err - } - if _, err := pm.AddPort(ctx, spec); err != nil { - return err + for _, port := range portMappings { + protocols := strings.Split(port.Protocol, ",") + for _, protocol := range protocols { + hostIP := port.HostIP + if hostIP == "" { + hostIP = "0.0.0.0" + } + for i := uint16(0); i < port.Range; i++ { + spec := rkport.Spec{ + Proto: protocol, + ParentIP: hostIP, + ParentPort: int(port.HostPort + i), + ChildPort: int(port.ContainerPort + i), + ChildIP: childIP, + } + if err := rkportutil.ValidatePortSpec(spec, nil); err != nil { + return err + } + if _, err := pm.AddPort(ctx, spec); err != nil { + return err + } + } } } return nil diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh index 9a7bfba8f..9b7c613f5 100644 --- a/contrib/cirrus/lib.sh +++ b/contrib/cirrus/lib.sh @@ -143,6 +143,8 @@ setup_rootless() { local rootless_uid local rootless_gid local env_var_val + local akfilepath + local sshcmd # Only do this once; established by setup_environment.sh # shellcheck disable=SC2154 @@ -169,24 +171,25 @@ setup_rootless() { ssh-keygen -P "" -f "$HOME/.ssh/id_rsa" msg "Allowing ssh key for $ROOTLESS_USER" + akfilepath="/home/$ROOTLESS_USER/.ssh/authorized_keys" (umask 077 && mkdir "/home/$ROOTLESS_USER/.ssh") chown -R $ROOTLESS_USER:$ROOTLESS_USER "/home/$ROOTLESS_USER/.ssh" install -o $ROOTLESS_USER -g $ROOTLESS_USER -m 0600 \ - "$HOME/.ssh/id_rsa.pub" "/home/$ROOTLESS_USER/.ssh/authorized_keys" + "$HOME/.ssh/id_rsa.pub" "$akfilepath" # Makes debugging easier - cat /root/.ssh/authorized_keys >> "/home/$ROOTLESS_USER/.ssh/authorized_keys" - - msg "Configuring subuid and subgid" - grep -q "${ROOTLESS_USER}" /etc/subuid || \ - echo "${ROOTLESS_USER}:$[rootless_uid * 100]:65536" | \ - tee -a /etc/subuid >> /etc/subgid + cat /root/.ssh/authorized_keys >> "$akfilepath" msg "Ensure the ssh daemon is up and running within 5 minutes" systemctl start sshd - lilto ssh $ROOTLESS_USER@localhost \ - -o UserKnownHostsFile=/dev/null \ - -o StrictHostKeyChecking=no \ - -o CheckHostIP=no true + sshcmd="ssh $ROOTLESS_USER@localhost + -o UserKnownHostsFile=/dev/null + -o StrictHostKeyChecking=no + -o CheckHostIP=no" + lilto $sshcmd true # retry until sshd is up + + msg "Configuring rootless user self-access to ssh to localhost" + $sshcmd ssh-keygen -P '""' -f "/home/$ROOTLESS_USER/.ssh/id_rsa" + cat "/home/$ROOTLESS_USER/.ssh/id_rsa" >> "$akfilepath" } install_test_configs() { diff --git a/docs/source/markdown/podman-build.1.md b/docs/source/markdown/podman-build.1.md index 5a867c574..835df7693 100644 --- a/docs/source/markdown/podman-build.1.md +++ b/docs/source/markdown/podman-build.1.md @@ -774,6 +774,14 @@ content label. Shared volume labels allow all containers to read/write content. The `Z` option tells Podman to label the content with a private unshared label. Only the current container can use a private volume. +Note: Do not relabel system files and directories. Relabeling system content +might cause other confined services on your machine to fail. For these types +of containers, disabling SELinux separation is recommended. The option +`--security-opt label=disable` disables SELinux separation for the container. +For example, if a user wanted to volume mount their entire home directory into the build containers, they need to disable SELinux separation. + + $ podman build --security-opt label=disable -v $HOME:/home/user . + `Overlay Volume Mounts` The `:O` flag tells Podman to mount the directory from the host as a diff --git a/docs/source/markdown/podman-create.1.md b/docs/source/markdown/podman-create.1.md index 3ff736adb..9a37a1dd0 100644 --- a/docs/source/markdown/podman-create.1.md +++ b/docs/source/markdown/podman-create.1.md @@ -1249,6 +1249,15 @@ content label. Shared volume labels allow all containers to read/write content. The `Z` option tells Podman to label the content with a private unshared label. Only the current container can use a private volume. +Note: Do not relabel system files and directories. Relabeling system content +might cause other confined services on your machine to fail. For these types +of containers we recommend that disable SELinux separation. The option +`--security-opt label=disable` disables SELinux separation for containers used in the build. +For example if a user wanted to volume mount their entire home directory into a +container, they need to disable SELinux separation. + + $ podman create --security-opt label=disable -v $HOME:/home/user fedora touch /home/user/file + `Overlay Volume Mounts` The `:O` flag tells Podman to mount the directory from the host as a diff --git a/docs/source/markdown/podman-run.1.md b/docs/source/markdown/podman-run.1.md index a1170253f..ce1e86afe 100644 --- a/docs/source/markdown/podman-run.1.md +++ b/docs/source/markdown/podman-run.1.md @@ -1314,6 +1314,15 @@ share the volume content. As a result, Podman labels the content with a shared content label. Shared volume labels allow all containers to read/write content. The **Z** option tells Podman to label the content with a private unshared label. +Note: Do not relabel system files and directories. Relabeling system content +might cause other confined services on your machine to fail. For these types +of containers we recommend that disable SELinux separation. The option +`--security-opt label=disable` disables SELinux separation for the container. +For example if a user wanted to volume mount their entire home directory into a +container, they need to disable SELinux separation. + + $ podman run --security-opt label=disable -v $HOME:/home/user fedora touch /home/user/file + `Overlay Volume Mounts` The `:O` flag tells Podman to mount the directory from the host as a diff --git a/docs/source/markdown/podman-unshare.1.md b/docs/source/markdown/podman-unshare.1.md index 72821b6e5..fa5259ae1 100644 --- a/docs/source/markdown/podman-unshare.1.md +++ b/docs/source/markdown/podman-unshare.1.md @@ -30,10 +30,10 @@ The unshare session defines two environment variables: Print usage statement -#### **--rootless-cni** +#### **--rootless-netns** -Join the rootless network namespace used for CNI networking. It can be used to -connect to a rootless container via IP address (CNI networking). This is otherwise +Join the rootless network namespace used for CNI and netavark networking. It can be used to +connect to a rootless container via IP address (bridge networking). This is otherwise not possible from the host network namespace. _Note: Using this option with more than one unshare session can have unexpected results._ @@ -78,7 +78,7 @@ $ podman unshare cat /proc/self/uid_map /proc/self/gid_map 0 1000 1 1 10000 65536 -$ podman unshare --rootless-cni ip addr +$ podman unshare --rootless-netns ip addr 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo @@ -7,7 +7,7 @@ require ( github.com/blang/semver v3.5.1+incompatible github.com/buger/goterm v0.0.0-20181115115552-c206103e1f37 github.com/checkpoint-restore/checkpointctl v0.0.0-20210922093614-c31748bec9f2 - github.com/checkpoint-restore/go-criu/v5 v5.1.0 + github.com/checkpoint-restore/go-criu/v5 v5.2.0 github.com/container-orchestrated-devices/container-device-interface v0.0.0-20210325223243-f99e8b6c10b9 github.com/containernetworking/cni v1.0.1 github.com/containernetworking/plugins v1.0.1 @@ -139,8 +139,8 @@ github.com/checkpoint-restore/checkpointctl v0.0.0-20210922093614-c31748bec9f2/g github.com/checkpoint-restore/go-criu/v4 v4.0.2/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw= github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw= github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M= -github.com/checkpoint-restore/go-criu/v5 v5.1.0 h1:BkVH17kcwgmKMnFArsvLrxuBbMwfvPNYRB7mfJ0lzyI= -github.com/checkpoint-restore/go-criu/v5 v5.1.0/go.mod h1:iaS8bb7p6zKJanp1Qe8mpl7+bnkYBR500psJR6mwma0= +github.com/checkpoint-restore/go-criu/v5 v5.2.0 h1:QwsRK9EdBr2kQr44DqSdBrP4dULp2+4EkqounYQOnF8= +github.com/checkpoint-restore/go-criu/v5 v5.2.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E= github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8= @@ -1449,8 +1449,9 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/libpod/boltdb_state_internal.go b/libpod/boltdb_state_internal.go index 3e3c17a9e..e71d82736 100644 --- a/libpod/boltdb_state_internal.go +++ b/libpod/boltdb_state_internal.go @@ -384,6 +384,15 @@ func (s *BoltState) getContainerConfigFromDB(id []byte, config *ContainerConfig, return errors.Wrapf(err, "error unmarshalling container %s config", string(id)) } + // convert ports to the new format if needed + if len(config.ContainerNetworkConfig.OldPortMappings) > 0 && len(config.ContainerNetworkConfig.PortMappings) == 0 { + config.ContainerNetworkConfig.PortMappings = ocicniPortsToNetTypesPorts(config.ContainerNetworkConfig.OldPortMappings) + // keep the OldPortMappings in case an user has to downgrade podman + + // indicate the the config was modified and should be written back to the db when possible + config.rewrite = true + } + return nil } diff --git a/libpod/common_test.go b/libpod/common_test.go index 4662a33bd..67e29c265 100644 --- a/libpod/common_test.go +++ b/libpod/common_test.go @@ -41,18 +41,20 @@ func getTestContainer(id, name string, manager lock.Manager) (*Container, error) ContainerNetworkConfig: ContainerNetworkConfig{ DNSServer: []net.IP{net.ParseIP("192.168.1.1"), net.ParseIP("192.168.2.2")}, DNSSearch: []string{"example.com", "example.example.com"}, - PortMappings: []types.OCICNIPortMapping{ + PortMappings: []types.PortMapping{ { HostPort: 80, ContainerPort: 90, Protocol: "tcp", HostIP: "192.168.3.3", + Range: 1, }, { HostPort: 100, ContainerPort: 110, Protocol: "udp", HostIP: "192.168.4.4", + Range: 1, }, }, }, diff --git a/libpod/container.go b/libpod/container.go index 4d15c04c5..86989a02f 100644 --- a/libpod/container.go +++ b/libpod/container.go @@ -465,7 +465,7 @@ func (c *Container) NewNetNS() bool { // PortMappings returns the ports that will be mapped into a container if // a new network namespace is created // If NewNetNS() is false, this value is unused -func (c *Container) PortMappings() ([]types.OCICNIPortMapping, error) { +func (c *Container) PortMappings() ([]types.PortMapping, error) { // First check if the container belongs to a network namespace (like a pod) if len(c.config.NetNsCtr) > 0 { netNsCtr, err := c.runtime.GetContainer(c.config.NetNsCtr) diff --git a/libpod/container_config.go b/libpod/container_config.go index 33ea731fd..412be835f 100644 --- a/libpod/container_config.go +++ b/libpod/container_config.go @@ -78,6 +78,11 @@ type ContainerConfig struct { // These containers must be started before this container is started. Dependencies []string + // rewrite is an internal bool to indicate that the config was modified after + // a read from the db, e.g. to migrate config fields after an upgrade. + // This field should never be written to the db, the json tag ensures this. + rewrite bool `json:"-"` + // embedded sub-configs ContainerRootFSConfig ContainerSecurityConfig @@ -228,11 +233,16 @@ type ContainerNetworkConfig struct { // StaticMAC is a static MAC to request for the container. // This cannot be set unless CreateNetNS is set. // If not set, the container will be dynamically assigned a MAC by CNI. - StaticMAC net.HardwareAddr `json:"staticMAC"` + StaticMAC types.HardwareAddr `json:"staticMAC"` // PortMappings are the ports forwarded to the container's network // namespace // These are not used unless CreateNetNS is true - PortMappings []types.OCICNIPortMapping `json:"portMappings,omitempty"` + PortMappings []types.PortMapping `json:"newPortMappings,omitempty"` + // OldPortMappings are the ports forwarded to the container's network + // namespace. As of podman 4.0 this field is deprecated, use PortMappings + // instead. The db will convert the old ports to the new structure for you. + // These are not used unless CreateNetNS is true + OldPortMappings []types.OCICNIPortMapping `json:"portMappings,omitempty"` // ExposedPorts are the ports which are exposed but not forwarded // into the container. // The map key is the port and the string slice contains the protocols, diff --git a/libpod/container_internal.go b/libpod/container_internal.go index b9805faa3..fbc2c1f38 100644 --- a/libpod/container_internal.go +++ b/libpod/container_internal.go @@ -496,9 +496,27 @@ func (c *Container) setupStorage(ctx context.Context) error { c.setupStorageMapping(&options.IDMappingOptions, &c.config.IDMappings) - containerInfo, err := c.runtime.storageService.CreateContainerStorage(ctx, c.runtime.imageContext, c.config.RootfsImageName, c.config.RootfsImageID, c.config.Name, c.config.ID, options) - if err != nil { - return errors.Wrapf(err, "error creating container storage") + // Unless the user has specified a name, use a randomly generated one. + // Note that name conflicts may occur (see #11735), so we need to loop. + generateName := c.config.Name == "" + var containerInfo ContainerInfo + var containerInfoErr error + for { + if generateName { + name, err := c.runtime.generateName() + if err != nil { + return err + } + c.config.Name = name + } + containerInfo, containerInfoErr = c.runtime.storageService.CreateContainerStorage(ctx, c.runtime.imageContext, c.config.RootfsImageName, c.config.RootfsImageID, c.config.Name, c.config.ID, options) + + if !generateName || errors.Cause(containerInfoErr) != storage.ErrDuplicateName { + break + } + } + if containerInfoErr != nil { + return errors.Wrapf(containerInfoErr, "error creating container storage") } // only reconfig IDMappings if layer was mounted from storage @@ -667,6 +685,19 @@ func (c *Container) refresh() error { c.state.NetworkStatus = nil c.state.NetworkStatusOld = nil + // Rewrite the config if necessary. + // Podman 4.0 uses a new port format in the config. + // getContainerConfigFromDB() already converted the old ports to the new one + // but it did not write the config to the db back for performance reasons. + // If a rewrite must happen the config.rewrite field is set to true. + if c.config.rewrite { + // SafeRewriteContainerConfig must be used with care. Make sure to not change config fields by accident. + if err := c.runtime.state.SafeRewriteContainerConfig(c, "", "", c.config); err != nil { + return errors.Wrapf(err, "failed to rewrite the config for container %s", c.config.ID) + } + c.config.rewrite = false + } + if err := c.save(); err != nil { return errors.Wrapf(err, "error refreshing state for container %s", c.ID()) } diff --git a/libpod/container_internal_linux.go b/libpod/container_internal_linux.go index 2fd519990..3187724ca 100644 --- a/libpod/container_internal_linux.go +++ b/libpod/container_internal_linux.go @@ -21,6 +21,7 @@ import ( "time" metadata "github.com/checkpoint-restore/checkpointctl/lib" + "github.com/checkpoint-restore/go-criu/v5/stats" cdi "github.com/container-orchestrated-devices/container-device-interface/pkg" "github.com/containernetworking/plugins/pkg/ns" "github.com/containers/buildah/pkg/chrootuser" @@ -1013,6 +1014,7 @@ func (c *Container) exportCheckpoint(options ContainerCheckpointOptions) error { metadata.ConfigDumpFile, metadata.SpecDumpFile, metadata.NetworkStatusFile, + stats.StatsDump, } if c.LogDriver() == define.KubernetesLogging || @@ -1197,7 +1199,7 @@ func (c *Container) checkpoint(ctx context.Context, options ContainerCheckpointO if !options.Keep && !options.PreCheckPoint { cleanup := []string{ "dump.log", - "stats-dump", + stats.StatsDump, metadata.ConfigDumpFile, metadata.SpecDumpFile, } @@ -1564,8 +1566,8 @@ func (c *Container) restore(ctx context.Context, options ContainerCheckpointOpti cleanup := [...]string{ "restore.log", "dump.log", - "stats-dump", - "stats-restore", + stats.StatsDump, + stats.StatsRestore, metadata.NetworkStatusFile, metadata.RootFsDiffTar, metadata.DeletedFilesFile, diff --git a/libpod/kube.go b/libpod/kube.go index ad5acea78..e0ed911af 100644 --- a/libpod/kube.go +++ b/libpod/kube.go @@ -75,7 +75,7 @@ func (p *Pod) GenerateForKube(ctx context.Context) (*v1.Pod, []v1.ServicePort, e Hostnames: []string{hostSli[0]}, }) } - ports, err = ocicniPortMappingToContainerPort(infraContainer.config.PortMappings) + ports, err = portMappingToContainerPort(infraContainer.config.PortMappings) if err != nil { return nil, servicePorts, err } @@ -452,7 +452,7 @@ func containerToV1Container(ctx context.Context, c *Container) (v1.Container, [] if err != nil { return kubeContainer, kubeVolumes, nil, annotations, err } - ports, err := ocicniPortMappingToContainerPort(portmappings) + ports, err := portMappingToContainerPort(portmappings) if err != nil { return kubeContainer, kubeVolumes, nil, annotations, err } @@ -588,29 +588,36 @@ func containerToV1Container(ctx context.Context, c *Container) (v1.Container, [] return kubeContainer, kubeVolumes, &dns, annotations, nil } -// ocicniPortMappingToContainerPort takes an ocicni portmapping and converts +// portMappingToContainerPort takes an portmapping and converts // it to a v1.ContainerPort format for kube output -func ocicniPortMappingToContainerPort(portMappings []types.OCICNIPortMapping) ([]v1.ContainerPort, error) { +func portMappingToContainerPort(portMappings []types.PortMapping) ([]v1.ContainerPort, error) { containerPorts := make([]v1.ContainerPort, 0, len(portMappings)) for _, p := range portMappings { - var protocol v1.Protocol - switch strings.ToUpper(p.Protocol) { - case "TCP": - // do nothing as it is the default protocol in k8s, there is no need to explicitly - // add it to the generated yaml - case "UDP": - protocol = v1.ProtocolUDP - default: - return containerPorts, errors.Errorf("unknown network protocol %s", p.Protocol) - } - cp := v1.ContainerPort{ - // Name will not be supported - HostPort: p.HostPort, - HostIP: p.HostIP, - ContainerPort: p.ContainerPort, - Protocol: protocol, - } - containerPorts = append(containerPorts, cp) + protocols := strings.Split(p.Protocol, ",") + for _, proto := range protocols { + var protocol v1.Protocol + switch strings.ToUpper(proto) { + case "TCP": + // do nothing as it is the default protocol in k8s, there is no need to explicitly + // add it to the generated yaml + case "UDP": + protocol = v1.ProtocolUDP + case "SCTP": + protocol = v1.ProtocolSCTP + default: + return containerPorts, errors.Errorf("unknown network protocol %s", p.Protocol) + } + for i := uint16(0); i < p.Range; i++ { + cp := v1.ContainerPort{ + // Name will not be supported + HostPort: int32(p.HostPort + i), + HostIP: p.HostIP, + ContainerPort: int32(p.ContainerPort + i), + Protocol: protocol, + } + containerPorts = append(containerPorts, cp) + } + } } return containerPorts, nil } diff --git a/libpod/network/cni/run.go b/libpod/network/cni/run.go index 99b2adce5..7795dfeeb 100644 --- a/libpod/network/cni/run.go +++ b/libpod/network/cni/run.go @@ -160,7 +160,7 @@ func CNIResultToStatus(res cnitypes.Result) (types.StatusBlock, error) { return result, err } interfaces[cniInt.Name] = types.NetInterface{ - MacAddress: mac, + MacAddress: types.HardwareAddr(mac), Networks: []types.NetAddress{{ Subnet: types.IPNet{IPNet: ip.Address}, Gateway: ip.Gateway, diff --git a/libpod/network/cni/run_test.go b/libpod/network/cni/run_test.go index 965203c2a..3169cd0eb 100644 --- a/libpod/network/cni/run_test.go +++ b/libpod/network/cni/run_test.go @@ -398,7 +398,7 @@ var _ = Describe("run CNI", func() { i, err := net.InterfaceByName(intName1) Expect(err).To(BeNil()) Expect(i.Name).To(Equal(intName1)) - Expect(i.HardwareAddr).To(Equal(macInt1)) + Expect(i.HardwareAddr).To(Equal((net.HardwareAddr)(macInt1))) addrs, err := i.Addrs() Expect(err).To(BeNil()) subnet := &net.IPNet{ @@ -448,7 +448,7 @@ var _ = Describe("run CNI", func() { i, err := net.InterfaceByName(intName1) Expect(err).To(BeNil()) Expect(i.Name).To(Equal(intName1)) - Expect(i.HardwareAddr).To(Equal(macInt1)) + Expect(i.HardwareAddr).To(Equal(net.HardwareAddr(macInt1))) addrs, err := i.Addrs() Expect(err).To(BeNil()) subnet := &net.IPNet{ @@ -460,7 +460,7 @@ var _ = Describe("run CNI", func() { i, err = net.InterfaceByName(intName2) Expect(err).To(BeNil()) Expect(i.Name).To(Equal(intName2)) - Expect(i.HardwareAddr).To(Equal(macInt2)) + Expect(i.HardwareAddr).To(Equal(net.HardwareAddr(macInt2))) addrs, err = i.Addrs() Expect(err).To(BeNil()) subnet = &net.IPNet{ @@ -600,7 +600,7 @@ var _ = Describe("run CNI", func() { i, err := net.InterfaceByName(intName1) Expect(err).To(BeNil()) Expect(i.Name).To(Equal(intName1)) - Expect(i.HardwareAddr).To(Equal(macInt1)) + Expect(i.HardwareAddr).To(Equal(net.HardwareAddr(macInt1))) addrs, err := i.Addrs() Expect(err).To(BeNil()) subnet := &net.IPNet{ @@ -612,7 +612,7 @@ var _ = Describe("run CNI", func() { i, err = net.InterfaceByName(intName2) Expect(err).To(BeNil()) Expect(i.Name).To(Equal(intName2)) - Expect(i.HardwareAddr).To(Equal(macInt2)) + Expect(i.HardwareAddr).To(Equal(net.HardwareAddr(macInt2))) addrs, err = i.Addrs() Expect(err).To(BeNil()) subnet = &net.IPNet{ @@ -690,7 +690,7 @@ var _ = Describe("run CNI", func() { netName: { InterfaceName: interfaceName, StaticIPs: []net.IP{ip1, ip2}, - StaticMAC: mac, + StaticMAC: types.HardwareAddr(mac), }, }, }, @@ -708,7 +708,7 @@ var _ = Describe("run CNI", func() { Expect(res[netName].Interfaces[interfaceName].Networks[1].Subnet.IP.String()).To(Equal(ip2.String())) Expect(res[netName].Interfaces[interfaceName].Networks[1].Subnet.Mask).To(Equal(subnet2.Mask)) Expect(res[netName].Interfaces[interfaceName].Networks[1].Gateway).To(Equal(net.ParseIP("fd41:0a75:2ca0:48a9::1"))) - Expect(res[netName].Interfaces[interfaceName].MacAddress).To(Equal(mac)) + Expect(res[netName].Interfaces[interfaceName].MacAddress).To(Equal(types.HardwareAddr(mac))) // default network has no dns Expect(res[netName].DNSServerIPs).To(BeEmpty()) Expect(res[netName].DNSSearchDomains).To(BeEmpty()) diff --git a/libpod/network/types/network.go b/libpod/network/types/network.go index 657c1ca6a..ba5e018fd 100644 --- a/libpod/network/types/network.go +++ b/libpod/network/types/network.go @@ -1,6 +1,7 @@ package types import ( + "encoding/json" "net" "time" ) @@ -94,6 +95,51 @@ func (n *IPNet) UnmarshalText(text []byte) error { return nil } +// HardwareAddr is the same as net.HardwareAddr except +// that it adds the json marshal/unmarshal methods. +// This allows us to read the mac from a json string +// and a byte array. +// swagger:model MacAddress +type HardwareAddr net.HardwareAddr + +func (h *HardwareAddr) String() string { + return (*net.HardwareAddr)(h).String() +} + +func (h *HardwareAddr) MarshalText() ([]byte, error) { + return []byte((*net.HardwareAddr)(h).String()), nil +} + +func (h *HardwareAddr) UnmarshalJSON(text []byte) error { + if len(text) == 0 { + *h = nil + return nil + } + + // if the json string start with a quote we got a string + // unmarshal the string and parse the mac from this string + if string(text[0]) == `"` { + var macString string + err := json.Unmarshal(text, &macString) + if err == nil { + mac, err := net.ParseMAC(macString) + if err == nil { + *h = HardwareAddr(mac) + return nil + } + } + } + // not a string or got an error fallback to the normal parsing + mac := make(net.HardwareAddr, 0, 6) + // use the standard json unmarshal for backwards compat + err := json.Unmarshal(text, &mac) + if err != nil { + return err + } + *h = HardwareAddr(mac) + return nil +} + type Subnet struct { // Subnet for this Network in CIDR form. // swagger:strfmt string @@ -134,7 +180,7 @@ type NetInterface struct { // Networks list of assigned subnets with their gateway. Networks []NetAddress `json:"networks,omitempty"` // MacAddress for this Interface. - MacAddress net.HardwareAddr `json:"mac_address"` + MacAddress HardwareAddr `json:"mac_address"` } // NetAddress contains the subnet and gateway. @@ -157,7 +203,7 @@ type PerNetworkOptions struct { // Optional. Aliases []string `json:"aliases,omitempty"` // StaticMac for this container. Optional. - StaticMAC net.HardwareAddr `json:"static_mac,omitempty"` + StaticMAC HardwareAddr `json:"static_mac,omitempty"` // InterfaceName for this container. Required. InterfaceName string `json:"interface_name"` } diff --git a/libpod/network/types/network_test.go b/libpod/network/types/network_test.go new file mode 100644 index 000000000..91ee93692 --- /dev/null +++ b/libpod/network/types/network_test.go @@ -0,0 +1,82 @@ +package types_test + +import ( + "encoding/json" + "reflect" + "testing" + + "github.com/containers/podman/v3/libpod/network/types" +) + +func TestUnmarshalMacAddress(t *testing.T) { + tests := []struct { + name string + json string + want types.HardwareAddr + wantErr bool + }{ + { + name: "mac as string with colon", + json: `"52:54:00:1c:2e:46"`, + want: types.HardwareAddr{0x52, 0x54, 0x00, 0x1c, 0x2e, 0x46}, + }, + { + name: "mac as string with dash", + json: `"52-54-00-1c-2e-46"`, + want: types.HardwareAddr{0x52, 0x54, 0x00, 0x1c, 0x2e, 0x46}, + }, + { + name: "mac as byte array", + json: `[82, 84, 0, 28, 46, 70]`, + want: types.HardwareAddr{0x52, 0x54, 0x00, 0x1c, 0x2e, 0x46}, + }, + { + name: "null value", + json: `null`, + want: nil, + }, + { + name: "mac as base64", + json: `"qrvM3e7/"`, + want: types.HardwareAddr{0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff}, + }, + { + name: "invalid string", + json: `"52:54:00:1c:2e`, + wantErr: true, + }, + { + name: "invalid array", + json: `[82, 84, 0, 28, 46`, + wantErr: true, + }, + + { + name: "invalid value", + json: `ab`, + wantErr: true, + }, + { + name: "invalid object", + json: `{}`, + wantErr: true, + }, + } + for _, tt := range tests { + test := tt + t.Run(test.name, func(t *testing.T) { + mac := types.HardwareAddr{} + err := json.Unmarshal([]byte(test.json), &mac) + if (err != nil) != test.wantErr { + t.Errorf("types.HardwareAddress Unmarshal() error = %v, wantErr %v", err, test.wantErr) + return + } + if test.wantErr { + return + } + if !reflect.DeepEqual(mac, test.want) { + t.Errorf("types.HardwareAddress Unmarshal() got = %v, want %v", mac, test.want) + } + }) + } +} diff --git a/libpod/networking_linux.go b/libpod/networking_linux.go index e792a410c..ef261a438 100644 --- a/libpod/networking_linux.go +++ b/libpod/networking_linux.go @@ -11,6 +11,7 @@ import ( "os/exec" "path/filepath" "regexp" + "sort" "strconv" "strings" "syscall" @@ -41,8 +42,11 @@ const ( // default slirp4ns subnet defaultSlirp4netnsSubnet = "10.0.2.0/24" - // rootlessCNINSName is the file name for the rootless network namespace bind mount - rootlessCNINSName = "rootless-cni-ns" + // rootlessNetNsName is the file name for the rootless network namespace bind mount + rootlessNetNsName = "rootless-netns" + + // rootlessNetNsSilrp4netnsPidFile is the name of the rootless netns slirp4netns pid file + rootlessNetNsSilrp4netnsPidFile = "rootless-netns-slirp4netns.pid" // persistentCNIDir is the directory where the CNI files are stored persistentCNIDir = "/var/lib/cni" @@ -88,10 +92,7 @@ func (c *Container) getNetworkOptions() (types.NetworkOptions, error) { ContainerID: c.config.ID, ContainerName: getCNIPodName(c), } - // TODO remove ocicni PortMappings from container config and store as types PortMappings - if len(c.config.PortMappings) > 0 { - opts.PortMappings = ocicniPortsToNetTypesPorts(c.config.PortMappings) - } + opts.PortMappings = c.config.PortMappings networks, _, err := c.networks() if err != nil { return opts, err @@ -136,21 +137,21 @@ func (c *Container) getNetworkOptions() (types.NetworkOptions, error) { return opts, nil } -type RootlessCNI struct { +type RootlessNetNS struct { ns ns.NetNS dir string Lock lockfile.Locker } -// getPath will join the given path to the rootless cni dir -func (r *RootlessCNI) getPath(path string) string { +// getPath will join the given path to the rootless netns dir +func (r *RootlessNetNS) getPath(path string) string { return filepath.Join(r.dir, path) } -// Do - run the given function in the rootless cni ns. +// Do - run the given function in the rootless netns. // It does not lock the rootlessCNI lock, the caller // should only lock when needed, e.g. for cni operations. -func (r *RootlessCNI) Do(toRun func() error) error { +func (r *RootlessNetNS) Do(toRun func() error) error { err := r.ns.Do(func(_ ns.NetNS) error { // Before we can run the given function, // we have to setup all mounts correctly. @@ -161,11 +162,11 @@ func (r *RootlessCNI) Do(toRun func() error) error { // Because the plugins also need access to XDG_RUNTIME_DIR/netns some special setup is needed. // The following bind mounts are needed - // 1. XDG_RUNTIME_DIR/netns -> XDG_RUNTIME_DIR/rootless-cni/XDG_RUNTIME_DIR/netns - // 2. /run/systemd -> XDG_RUNTIME_DIR/rootless-cni/run/systemd (only if it exists) - // 3. XDG_RUNTIME_DIR/rootless-cni/resolv.conf -> /etc/resolv.conf or XDG_RUNTIME_DIR/rootless-cni/run/symlink/target - // 4. XDG_RUNTIME_DIR/rootless-cni/var/lib/cni -> /var/lib/cni (if /var/lib/cni does not exists use the parent dir) - // 5. XDG_RUNTIME_DIR/rootless-cni/run -> /run + // 1. XDG_RUNTIME_DIR -> XDG_RUNTIME_DIR/rootless-netns/XDG_RUNTIME_DIR + // 2. /run/systemd -> XDG_RUNTIME_DIR/rootless-netns/run/systemd (only if it exists) + // 3. XDG_RUNTIME_DIR/rootless-netns/resolv.conf -> /etc/resolv.conf or XDG_RUNTIME_DIR/rootless-netns/run/symlink/target + // 4. XDG_RUNTIME_DIR/rootless-netns/var/lib/cni -> /var/lib/cni (if /var/lib/cni does not exists use the parent dir) + // 5. XDG_RUNTIME_DIR/rootless-netns/run -> /run // Create a new mount namespace, // this must happen inside the netns thread. @@ -174,16 +175,16 @@ func (r *RootlessCNI) Do(toRun func() error) error { return errors.Wrapf(err, "cannot create a new mount namespace") } - netNsDir, err := netns.GetNSRunDir() + xdgRuntimeDir, err := util.GetRuntimeDir() if err != nil { - return errors.Wrap(err, "could not get network namespace directory") + return errors.Wrap(err, "could not get runtime directory") } - newNetNsDir := r.getPath(netNsDir) + newXDGRuntimeDir := r.getPath(xdgRuntimeDir) // 1. Mount the netns into the new run to keep them accessible. // Otherwise cni setup will fail because it cannot access the netns files. - err = unix.Mount(netNsDir, newNetNsDir, "none", unix.MS_BIND|unix.MS_SHARED|unix.MS_REC, "") + err = unix.Mount(xdgRuntimeDir, newXDGRuntimeDir, "none", unix.MS_BIND|unix.MS_SHARED|unix.MS_REC, "") if err != nil { - return errors.Wrap(err, "failed to mount netns directory for rootless cni") + return errors.Wrap(err, "failed to mount runtime directory for rootless netns") } // 2. Also keep /run/systemd if it exists. @@ -194,7 +195,7 @@ func (r *RootlessCNI) Do(toRun func() error) error { newRunSystemd := r.getPath(runSystemd) err = unix.Mount(runSystemd, newRunSystemd, "none", unix.MS_BIND|unix.MS_REC, "") if err != nil { - return errors.Wrap(err, "failed to mount /run/systemd directory for rootless cni") + return errors.Wrap(err, "failed to mount /run/systemd directory for rootless netns") } } @@ -242,25 +243,25 @@ func (r *RootlessCNI) Do(toRun func() error) error { rsr := r.getPath("/run/systemd/resolve") err = unix.Mount("", rsr, "tmpfs", unix.MS_NOEXEC|unix.MS_NOSUID|unix.MS_NODEV, "") if err != nil { - return errors.Wrapf(err, "failed to mount tmpfs on %q for rootless cni", rsr) + return errors.Wrapf(err, "failed to mount tmpfs on %q for rootless netns", rsr) } } if strings.HasPrefix(resolvePath, "/run/") { resolvePath = r.getPath(resolvePath) err = os.MkdirAll(filepath.Dir(resolvePath), 0700) if err != nil { - return errors.Wrap(err, "failed to create rootless-cni resolv.conf directory") + return errors.Wrap(err, "failed to create rootless-netns resolv.conf directory") } // we want to bind mount on this file so we have to create the file first _, err = os.OpenFile(resolvePath, os.O_CREATE|os.O_RDONLY, 0700) if err != nil { - return errors.Wrap(err, "failed to create rootless-cni resolv.conf file") + return errors.Wrap(err, "failed to create rootless-netns resolv.conf file") } } // mount resolv.conf to make use of the host dns err = unix.Mount(r.getPath("resolv.conf"), resolvePath, "none", unix.MS_BIND, "") if err != nil { - return errors.Wrap(err, "failed to mount resolv.conf for rootless cni") + return errors.Wrap(err, "failed to mount resolv.conf for rootless netns") } // 4. CNI plugins need access to /var/lib/cni and /run @@ -285,14 +286,14 @@ func (r *RootlessCNI) Do(toRun func() error) error { // make sure to mount var first err = unix.Mount(varDir, varTarget, "none", unix.MS_BIND, "") if err != nil { - return errors.Wrapf(err, "failed to mount %s for rootless cni", varTarget) + return errors.Wrapf(err, "failed to mount %s for rootless netns", varTarget) } // 5. Mount the new prepared run dir to /run, it has to be recursive to keep the other bind mounts. runDir := r.getPath("run") err = unix.Mount(runDir, "/run", "none", unix.MS_BIND|unix.MS_REC, "") if err != nil { - return errors.Wrap(err, "failed to mount /run for rootless cni") + return errors.Wrap(err, "failed to mount /run for rootless netns") } // run the given function in the correct namespace @@ -302,10 +303,11 @@ func (r *RootlessCNI) Do(toRun func() error) error { return err } -// Cleanup the rootless cni namespace if needed. +// Cleanup the rootless network namespace if needed. // It checks if we have running containers with the bridge network mode. -// Cleanup() will try to lock RootlessCNI, therefore you have to call it with an unlocked -func (r *RootlessCNI) Cleanup(runtime *Runtime) error { +// Cleanup() will try to lock RootlessNetNS, therefore you have to call +// it with an unlocked lock. +func (r *RootlessNetNS) Cleanup(runtime *Runtime) error { _, err := os.Stat(r.dir) if os.IsNotExist(err) { // the directory does not exists no need for cleanup @@ -314,8 +316,25 @@ func (r *RootlessCNI) Cleanup(runtime *Runtime) error { r.Lock.Lock() defer r.Lock.Unlock() running := func(c *Container) bool { + // no bridge => no need to check + if !c.config.NetMode.IsBridge() { + return false + } + // we cannot use c.state() because it will try to lock the container - // using c.state.State directly should be good enough for this use case + // locking is a problem because cleanup is called after net teardown + // at this stage the container is already locked. + // also do not try to lock only containers which are not currently in net + // teardown because this will result in an ABBA deadlock between the rootless + // cni lock and the container lock + // because we need to get the state we have to sync otherwise this will not + // work because the state is empty by default + // I do not like this but I do not see a better way at moment + err := c.syncContainer() + if err != nil { + return false + } + state := c.state.State return state == define.ContainerStateRunning } @@ -323,101 +342,89 @@ func (r *RootlessCNI) Cleanup(runtime *Runtime) error { if err != nil { return err } - cleanup := true - for _, ctr := range ctrs { - if ctr.config.NetMode.IsBridge() { - cleanup = false - } + // no cleanup if we found containers + if len(ctrs) > 0 { + return nil } - if cleanup { - // make sure the the cni results (cache) dir is empty - // libpod instances with another root dir are not covered by the check above - // this allows several libpod instances to use the same rootless cni ns - contents, err := ioutil.ReadDir(r.getPath("var/lib/cni/results")) - if (err == nil && len(contents) == 0) || os.IsNotExist(err) { - logrus.Debug("Cleaning up rootless cni namespace") - err = netns.UnmountNS(r.ns) - if err != nil { - return err - } - // make the following errors not fatal - err = r.ns.Close() - if err != nil { - logrus.Error(err) - } - b, err := ioutil.ReadFile(r.getPath("rootless-cni-slirp4netns.pid")) - if err == nil { - var i int - i, err = strconv.Atoi(string(b)) - if err == nil { - // kill the slirp process so we do not leak it - err = syscall.Kill(i, syscall.SIGTERM) - } - } - if err != nil { - logrus.Errorf("Failed to kill slirp4netns process: %s", err) - } - err = os.RemoveAll(r.dir) - if err != nil { - logrus.Error(err) - } - } else if err != nil && !os.IsNotExist(err) { - logrus.Errorf("Could not read rootless cni directory, skipping cleanup: %s", err) + logrus.Debug("Cleaning up rootless network namespace") + err = netns.UnmountNS(r.ns) + if err != nil { + return err + } + // make the following errors not fatal + err = r.ns.Close() + if err != nil { + logrus.Error(err) + } + b, err := ioutil.ReadFile(r.getPath(rootlessNetNsSilrp4netnsPidFile)) + if err == nil { + var i int + i, err = strconv.Atoi(string(b)) + if err == nil { + // kill the slirp process so we do not leak it + err = syscall.Kill(i, syscall.SIGTERM) } } + if err != nil { + logrus.Errorf("Failed to kill slirp4netns process: %s", err) + } + err = os.RemoveAll(r.dir) + if err != nil { + logrus.Error(err) + } return nil } -// GetRootlessCNINetNs returns the rootless cni object. If create is set to true -// the rootless cni namespace will be created if it does not exists already. +// GetRootlessNetNs returns the rootless netns object. If create is set to true +// the rootless network namespace will be created if it does not exists already. // If called as root it returns always nil. // On success the returned RootlessCNI lock is locked and must be unlocked by the caller. -func (r *Runtime) GetRootlessCNINetNs(new bool) (*RootlessCNI, error) { +func (r *Runtime) GetRootlessNetNs(new bool) (*RootlessNetNS, error) { if !rootless.IsRootless() { return nil, nil } - var rootlessCNINS *RootlessCNI + var rootlessNetNS *RootlessNetNS runDir, err := util.GetRuntimeDir() if err != nil { return nil, err } - lfile := filepath.Join(runDir, "rootless-cni.lock") + lfile := filepath.Join(runDir, "rootless-netns.lock") lock, err := lockfile.GetLockfile(lfile) if err != nil { - return nil, errors.Wrap(err, "failed to get rootless-cni lockfile") + return nil, errors.Wrap(err, "failed to get rootless-netns lockfile") } lock.Lock() defer func() { - // In case of an error (early exit) rootlessCNINS will be nil. + // In case of an error (early exit) rootlessNetNS will be nil. // Make sure to unlock otherwise we could deadlock. - if rootlessCNINS == nil { + if rootlessNetNS == nil { lock.Unlock() } }() - cniDir := filepath.Join(runDir, "rootless-cni") - err = os.MkdirAll(cniDir, 0700) + rootlessNetNsDir := filepath.Join(runDir, rootlessNetNsName) + err = os.MkdirAll(rootlessNetNsDir, 0700) if err != nil { - return nil, errors.Wrap(err, "could not create rootless-cni directory") + return nil, errors.Wrap(err, "could not create rootless-netns directory") } nsDir, err := netns.GetNSRunDir() if err != nil { return nil, err } - path := filepath.Join(nsDir, rootlessCNINSName) + path := filepath.Join(nsDir, rootlessNetNsName) ns, err := ns.GetNS(path) if err != nil { if !new { // return a error if we could not get the namespace and should no create one - return nil, errors.Wrap(err, "error getting rootless cni network namespace") + return nil, errors.Wrap(err, "error getting rootless network namespace") } // create a new namespace - logrus.Debug("creating rootless cni network namespace") - ns, err = netns.NewNSWithName(rootlessCNINSName) + logrus.Debug("creating rootless network namespace") + ns, err = netns.NewNSWithName(rootlessNetNsName) if err != nil { - return nil, errors.Wrap(err, "error creating rootless cni network namespace") + return nil, errors.Wrap(err, "error creating rootless network namespace") } // setup slirp4netns here path := r.config.Engine.NetworkCmdPath @@ -467,7 +474,7 @@ func (r *Runtime) GetRootlessCNINetNs(new bool) (*RootlessCNI, error) { // Leak one end of the pipe in slirp4netns cmd.ExtraFiles = append(cmd.ExtraFiles, syncW) - logPath := filepath.Join(r.config.Engine.TmpDir, "slirp4netns-rootless-cni.log") + logPath := filepath.Join(r.config.Engine.TmpDir, "slirp4netns-rootless-netns.log") logFile, err := os.Create(logPath) if err != nil { return nil, errors.Wrapf(err, "failed to open slirp4netns log file %s", logPath) @@ -486,9 +493,9 @@ func (r *Runtime) GetRootlessCNINetNs(new bool) (*RootlessCNI, error) { // create pid file for the slirp4netns process // this is need to kill the process in the cleanup pid := strconv.Itoa(cmd.Process.Pid) - err = ioutil.WriteFile(filepath.Join(cniDir, "rootless-cni-slirp4netns.pid"), []byte(pid), 0700) + err = ioutil.WriteFile(filepath.Join(rootlessNetNsDir, rootlessNetNsSilrp4netnsPidFile), []byte(pid), 0700) if err != nil { - errors.Wrap(err, "unable to write rootless-cni slirp4netns pid file") + errors.Wrap(err, "unable to write rootless-netns slirp4netns pid file") } defer func() { @@ -529,43 +536,43 @@ func (r *Runtime) GetRootlessCNINetNs(new bool) (*RootlessCNI, error) { dnsOptions := resolvconf.GetOptions(conf.Content) nameServers := resolvconf.GetNameservers(conf.Content) - _, err = resolvconf.Build(filepath.Join(cniDir, "resolv.conf"), append([]string{resolveIP.String()}, nameServers...), searchDomains, dnsOptions) + _, err = resolvconf.Build(filepath.Join(rootlessNetNsDir, "resolv.conf"), append([]string{resolveIP.String()}, nameServers...), searchDomains, dnsOptions) if err != nil { - return nil, errors.Wrap(err, "failed to create rootless cni resolv.conf") + return nil, errors.Wrap(err, "failed to create rootless netns resolv.conf") } // create cni directories to store files // they will be bind mounted to the correct location in a extra mount ns - err = os.MkdirAll(filepath.Join(cniDir, strings.TrimPrefix(persistentCNIDir, "/")), 0700) + err = os.MkdirAll(filepath.Join(rootlessNetNsDir, persistentCNIDir), 0700) if err != nil { - return nil, errors.Wrap(err, "could not create rootless-cni var directory") + return nil, errors.Wrap(err, "could not create rootless-netns var directory") } - runDir := filepath.Join(cniDir, "run") + runDir := filepath.Join(rootlessNetNsDir, "run") err = os.MkdirAll(runDir, 0700) if err != nil { - return nil, errors.Wrap(err, "could not create rootless-cni run directory") + return nil, errors.Wrap(err, "could not create rootless-netns run directory") } // relabel the new run directory to the iptables /run label // this is important, otherwise the iptables command will fail err = label.Relabel(runDir, "system_u:object_r:iptables_var_run_t:s0", false) if err != nil { - return nil, errors.Wrap(err, "could not create relabel rootless-cni run directory") + return nil, errors.Wrap(err, "could not create relabel rootless-netns run directory") } // create systemd run directory err = os.MkdirAll(filepath.Join(runDir, "systemd"), 0700) if err != nil { - return nil, errors.Wrap(err, "could not create rootless-cni systemd directory") + return nil, errors.Wrap(err, "could not create rootless-netns systemd directory") } // create the directory for the netns files at the same location - // relative to the rootless-cni location - err = os.MkdirAll(filepath.Join(cniDir, nsDir), 0700) + // relative to the rootless-netns location + err = os.MkdirAll(filepath.Join(rootlessNetNsDir, nsDir), 0700) if err != nil { - return nil, errors.Wrap(err, "could not create rootless-cni netns directory") + return nil, errors.Wrap(err, "could not create rootless-netns netns directory") } } - // The CNI plugins need access to iptables in $PATH. As it turns out debian doesn't put - // /usr/sbin in $PATH for rootless users. This will break rootless cni completely. + // The CNI plugins and netavark need access to iptables in $PATH. As it turns out debian doesn't put + // /usr/sbin in $PATH for rootless users. This will break rootless networking completely. // We might break existing users and we cannot expect everyone to change their $PATH so // lets add /usr/sbin to $PATH ourselves. path = os.Getenv("PATH") @@ -574,14 +581,14 @@ func (r *Runtime) GetRootlessCNINetNs(new bool) (*RootlessCNI, error) { os.Setenv("PATH", path) } - // Important set rootlessCNINS as last step. + // Important set rootlessNetNS as last step. // Do not return any errors after this. - rootlessCNINS = &RootlessCNI{ + rootlessNetNS = &RootlessNetNS{ ns: ns, - dir: cniDir, + dir: rootlessNetNsDir, Lock: lock, } - return rootlessCNINS, nil + return rootlessNetNS, nil } // setPrimaryMachineIP is used for podman-machine and it sets @@ -603,14 +610,14 @@ func setPrimaryMachineIP() error { } // setUpNetwork will set up the the networks, on error it will also tear down the cni -// networks. If rootless it will join/create the rootless cni namespace. +// networks. If rootless it will join/create the rootless network namespace. func (r *Runtime) setUpNetwork(ns string, opts types.NetworkOptions) (map[string]types.StatusBlock, error) { if r.config.MachineEnabled() { if err := setPrimaryMachineIP(); err != nil { return nil, err } } - rootlessCNINS, err := r.GetRootlessCNINetNs(true) + rootlessNetNS, err := r.GetRootlessNetNs(true) if err != nil { return nil, err } @@ -619,11 +626,11 @@ func (r *Runtime) setUpNetwork(ns string, opts types.NetworkOptions) (map[string results, err = r.network.Setup(ns, types.SetupOptions{NetworkOptions: opts}) return err } - // rootlessCNINS is nil if we are root - if rootlessCNINS != nil { - // execute the cni setup in the rootless net ns - err = rootlessCNINS.Do(setUpPod) - rootlessCNINS.Lock.Unlock() + // rootlessNetNS is nil if we are root + if rootlessNetNS != nil { + // execute the setup in the rootless net ns + err = rootlessNetNS.Do(setUpPod) + rootlessNetNS.Lock.Unlock() } else { err = setUpPod() } @@ -697,10 +704,10 @@ func (r *Runtime) setupRootlessNetNS(ctr *Container) error { return err } if len(networks) > 0 && len(ctr.config.PortMappings) > 0 { - // set up port forwarder for CNI-in-slirp4netns + // set up port forwarder for rootless netns netnsPath := ctr.state.NetNS.Path() // TODO: support slirp4netns port forwarder as well - // make sure to fix this container.handleRestartPolicy() as well + // make sure to fix this in container.handleRestartPolicy() as well return r.setupRootlessPortMappingViaRLK(ctr, netnsPath) } return nil @@ -719,7 +726,7 @@ func (r *Runtime) setupNetNS(ctr *Container) error { if err != nil { return err } - nsPath = filepath.Join(nsPath, fmt.Sprintf("cni-%x-%x-%x-%x-%x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:])) + nsPath = filepath.Join(nsPath, fmt.Sprintf("netns-%x-%x-%x-%x-%x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:])) if err := os.MkdirAll(filepath.Dir(nsPath), 0711); err != nil { return err @@ -777,10 +784,10 @@ func (r *Runtime) closeNetNS(ctr *Container) error { return nil } -// Tear down a container's CNI network configuration and joins the +// Tear down a container's network configuration and joins the // rootless net ns as rootless user func (r *Runtime) teardownNetwork(ns string, opts types.NetworkOptions) error { - rootlessCNINS, err := r.GetRootlessCNINetNs(false) + rootlessNetNS, err := r.GetRootlessNetNs(false) if err != nil { return err } @@ -789,13 +796,13 @@ func (r *Runtime) teardownNetwork(ns string, opts types.NetworkOptions) error { return errors.Wrapf(err, "error tearing down network namespace configuration for container %s", opts.ContainerID) } - // rootlessCNINS is nil if we are root - if rootlessCNINS != nil { + // rootlessNetNS is nil if we are root + if rootlessNetNS != nil { // execute the cni setup in the rootless net ns - err = rootlessCNINS.Do(tearDownPod) - rootlessCNINS.Lock.Unlock() + err = rootlessNetNS.Do(tearDownPod) + rootlessNetNS.Lock.Unlock() if err == nil { - err = rootlessCNINS.Cleanup(r) + err = rootlessNetNS.Cleanup(r) } } else { err = tearDownPod() @@ -1200,9 +1207,7 @@ func (c *Container) NetworkDisconnect(nameOrID, netName string, force bool) erro ContainerID: c.config.ID, ContainerName: getCNIPodName(c), } - if len(c.config.PortMappings) > 0 { - opts.PortMappings = ocicniPortsToNetTypesPorts(c.config.PortMappings) - } + opts.PortMappings = c.config.PortMappings eth, exists := c.state.NetInterfaceDescriptions.getInterfaceByName(netName) if !exists { return errors.Errorf("no network interface name for container %s on network %s", c.config.ID, netName) @@ -1294,9 +1299,7 @@ func (c *Container) NetworkConnect(nameOrID, netName string, aliases []string) e ContainerID: c.config.ID, ContainerName: getCNIPodName(c), } - if len(c.config.PortMappings) > 0 { - opts.PortMappings = ocicniPortsToNetTypesPorts(c.config.PortMappings) - } + opts.PortMappings = c.config.PortMappings eth, exists := c.state.NetInterfaceDescriptions.getInterfaceByName(netName) if !exists { return errors.Errorf("no network interface name for container %s on network %s", c.config.ID, netName) @@ -1364,16 +1367,67 @@ func (r *Runtime) normalizeNetworkName(nameOrID string) (string, error) { return net.Name, nil } +// ocicniPortsToNetTypesPorts convert the old port format to the new one +// while deduplicating ports into ranges func ocicniPortsToNetTypesPorts(ports []types.OCICNIPortMapping) []types.PortMapping { + if len(ports) == 0 { + return nil + } + newPorts := make([]types.PortMapping, 0, len(ports)) - for _, port := range ports { - newPorts = append(newPorts, types.PortMapping{ - HostIP: port.HostIP, - HostPort: uint16(port.HostPort), - ContainerPort: uint16(port.ContainerPort), - Protocol: port.Protocol, - Range: 1, - }) + + // first sort the ports + sort.Slice(ports, func(i, j int) bool { + return compareOCICNIPorts(ports[i], ports[j]) + }) + + // we already check if the slice is empty so we can use the first element + currentPort := types.PortMapping{ + HostIP: ports[0].HostIP, + HostPort: uint16(ports[0].HostPort), + ContainerPort: uint16(ports[0].ContainerPort), + Protocol: ports[0].Protocol, + Range: 1, + } + + for i := 1; i < len(ports); i++ { + if ports[i].HostIP == currentPort.HostIP && + ports[i].Protocol == currentPort.Protocol && + ports[i].HostPort-int32(currentPort.Range) == int32(currentPort.HostPort) && + ports[i].ContainerPort-int32(currentPort.Range) == int32(currentPort.ContainerPort) { + currentPort.Range = currentPort.Range + 1 + } else { + newPorts = append(newPorts, currentPort) + currentPort = types.PortMapping{ + HostIP: ports[i].HostIP, + HostPort: uint16(ports[i].HostPort), + ContainerPort: uint16(ports[i].ContainerPort), + Protocol: ports[i].Protocol, + Range: 1, + } + } } + newPorts = append(newPorts, currentPort) return newPorts } + +// compareOCICNIPorts will sort the ocicni ports by +// 1) host ip +// 2) protocol +// 3) hostPort +// 4) container port +func compareOCICNIPorts(i, j types.OCICNIPortMapping) bool { + if i.HostIP != j.HostIP { + return i.HostIP < j.HostIP + } + + if i.Protocol != j.Protocol { + return i.Protocol < j.Protocol + } + + if i.HostPort != j.HostPort { + return i.HostPort < j.HostPort + } + + return i.ContainerPort < j.ContainerPort +} diff --git a/libpod/networking_linux_test.go b/libpod/networking_linux_test.go new file mode 100644 index 000000000..06bf05723 --- /dev/null +++ b/libpod/networking_linux_test.go @@ -0,0 +1,323 @@ +package libpod + +import ( + "fmt" + "testing" + + "github.com/containers/podman/v3/libpod/network/types" + "github.com/stretchr/testify/assert" +) + +func Test_ocicniPortsToNetTypesPorts(t *testing.T) { + tests := []struct { + name string + arg []types.OCICNIPortMapping + want []types.PortMapping + }{ + { + name: "no ports", + arg: nil, + want: nil, + }, + { + name: "empty ports", + arg: []types.OCICNIPortMapping{}, + want: nil, + }, + { + name: "single port", + arg: []types.OCICNIPortMapping{ + { + HostPort: 8080, + ContainerPort: 80, + Protocol: "tcp", + }, + }, + want: []types.PortMapping{ + { + HostPort: 8080, + ContainerPort: 80, + Protocol: "tcp", + Range: 1, + }, + }, + }, + { + name: "two separate ports", + arg: []types.OCICNIPortMapping{ + { + HostPort: 8080, + ContainerPort: 80, + Protocol: "tcp", + }, + { + HostPort: 9000, + ContainerPort: 90, + Protocol: "tcp", + }, + }, + want: []types.PortMapping{ + { + HostPort: 8080, + ContainerPort: 80, + Protocol: "tcp", + Range: 1, + }, + { + HostPort: 9000, + ContainerPort: 90, + Protocol: "tcp", + Range: 1, + }, + }, + }, + { + name: "two ports joined", + arg: []types.OCICNIPortMapping{ + { + HostPort: 8080, + ContainerPort: 80, + Protocol: "tcp", + }, + { + HostPort: 8081, + ContainerPort: 81, + Protocol: "tcp", + }, + }, + want: []types.PortMapping{ + { + HostPort: 8080, + ContainerPort: 80, + Protocol: "tcp", + Range: 2, + }, + }, + }, + { + name: "three ports with different container port are not joined", + arg: []types.OCICNIPortMapping{ + { + HostPort: 8080, + ContainerPort: 80, + Protocol: "tcp", + }, + { + HostPort: 8081, + ContainerPort: 79, + Protocol: "tcp", + }, + { + HostPort: 8082, + ContainerPort: 82, + Protocol: "tcp", + }, + }, + want: []types.PortMapping{ + { + HostPort: 8080, + ContainerPort: 80, + Protocol: "tcp", + Range: 1, + }, + { + HostPort: 8081, + ContainerPort: 79, + Protocol: "tcp", + Range: 1, + }, + { + HostPort: 8082, + ContainerPort: 82, + Protocol: "tcp", + Range: 1, + }, + }, + }, + { + name: "three ports joined (not sorted)", + arg: []types.OCICNIPortMapping{ + { + HostPort: 8081, + ContainerPort: 81, + Protocol: "tcp", + }, + { + HostPort: 8080, + ContainerPort: 80, + Protocol: "tcp", + }, + { + HostPort: 8082, + ContainerPort: 82, + Protocol: "tcp", + }, + }, + want: []types.PortMapping{ + { + HostPort: 8080, + ContainerPort: 80, + Protocol: "tcp", + Range: 3, + }, + }, + }, + { + name: "different protocols ports are not joined", + arg: []types.OCICNIPortMapping{ + { + HostPort: 8080, + ContainerPort: 80, + Protocol: "tcp", + }, + { + HostPort: 8081, + ContainerPort: 81, + Protocol: "udp", + }, + }, + want: []types.PortMapping{ + { + HostPort: 8080, + ContainerPort: 80, + Protocol: "tcp", + Range: 1, + }, + { + HostPort: 8081, + ContainerPort: 81, + Protocol: "udp", + Range: 1, + }, + }, + }, + { + name: "different host ip ports are not joined", + arg: []types.OCICNIPortMapping{ + { + HostPort: 8080, + ContainerPort: 80, + Protocol: "tcp", + HostIP: "192.168.1.1", + }, + { + HostPort: 8081, + ContainerPort: 81, + Protocol: "tcp", + HostIP: "192.168.1.2", + }, + }, + want: []types.PortMapping{ + { + HostPort: 8080, + ContainerPort: 80, + Protocol: "tcp", + Range: 1, + HostIP: "192.168.1.1", + }, + { + HostPort: 8081, + ContainerPort: 81, + Protocol: "tcp", + Range: 1, + HostIP: "192.168.1.2", + }, + }, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + result := ocicniPortsToNetTypesPorts(tt.arg) + assert.Equal(t, tt.want, result, "ports do not match") + }) + } +} + +func benchmarkOCICNIPortsToNetTypesPorts(b *testing.B, ports []types.OCICNIPortMapping) { + for n := 0; n < b.N; n++ { + ocicniPortsToNetTypesPorts(ports) + } +} + +func Benchmark_ocicniPortsToNetTypesPortsNoPorts(b *testing.B) { + benchmarkOCICNIPortsToNetTypesPorts(b, nil) +} + +func Benchmark_ocicniPortsToNetTypesPorts1(b *testing.B) { + benchmarkOCICNIPortsToNetTypesPorts(b, []types.OCICNIPortMapping{ + { + HostPort: 8080, + ContainerPort: 80, + Protocol: "tcp", + }, + }) +} + +func Benchmark_ocicniPortsToNetTypesPorts10(b *testing.B) { + ports := make([]types.OCICNIPortMapping, 0, 10) + for i := int32(8080); i < 8090; i++ { + ports = append(ports, types.OCICNIPortMapping{ + HostPort: i, + ContainerPort: i, + Protocol: "tcp", + }) + } + b.ResetTimer() + benchmarkOCICNIPortsToNetTypesPorts(b, ports) +} + +func Benchmark_ocicniPortsToNetTypesPorts100(b *testing.B) { + ports := make([]types.OCICNIPortMapping, 0, 100) + for i := int32(8080); i < 8180; i++ { + ports = append(ports, types.OCICNIPortMapping{ + HostPort: i, + ContainerPort: i, + Protocol: "tcp", + }) + } + b.ResetTimer() + benchmarkOCICNIPortsToNetTypesPorts(b, ports) +} + +func Benchmark_ocicniPortsToNetTypesPorts1k(b *testing.B) { + ports := make([]types.OCICNIPortMapping, 0, 1000) + for i := int32(8080); i < 9080; i++ { + ports = append(ports, types.OCICNIPortMapping{ + HostPort: i, + ContainerPort: i, + Protocol: "tcp", + }) + } + b.ResetTimer() + benchmarkOCICNIPortsToNetTypesPorts(b, ports) +} + +func Benchmark_ocicniPortsToNetTypesPorts10k(b *testing.B) { + ports := make([]types.OCICNIPortMapping, 0, 30000) + for i := int32(8080); i < 18080; i++ { + ports = append(ports, types.OCICNIPortMapping{ + HostPort: i, + ContainerPort: i, + Protocol: "tcp", + }) + } + b.ResetTimer() + benchmarkOCICNIPortsToNetTypesPorts(b, ports) +} + +func Benchmark_ocicniPortsToNetTypesPorts1m(b *testing.B) { + ports := make([]types.OCICNIPortMapping, 0, 1000000) + for j := 0; j < 20; j++ { + for i := int32(1); i <= 50000; i++ { + ports = append(ports, types.OCICNIPortMapping{ + HostPort: i, + ContainerPort: i, + Protocol: "tcp", + HostIP: fmt.Sprintf("192.168.1.%d", j), + }) + } + } + b.ResetTimer() + benchmarkOCICNIPortsToNetTypesPorts(b, ports) +} diff --git a/libpod/networking_slirp4netns.go b/libpod/networking_slirp4netns.go index 56e8eca99..760427f22 100644 --- a/libpod/networking_slirp4netns.go +++ b/libpod/networking_slirp4netns.go @@ -38,9 +38,9 @@ type slirpFeatures struct { type slirp4netnsCmdArg struct { Proto string `json:"proto,omitempty"` HostAddr string `json:"host_addr"` - HostPort int32 `json:"host_port"` + HostPort uint16 `json:"host_port"` GuestAddr string `json:"guest_addr"` - GuestPort int32 `json:"guest_port"` + GuestPort uint16 `json:"guest_port"` } type slirp4netnsCmd struct { diff --git a/libpod/oci_util.go b/libpod/oci_util.go index c1afc0d20..6d99d5836 100644 --- a/libpod/oci_util.go +++ b/libpod/oci_util.go @@ -32,93 +32,108 @@ func createUnitName(prefix string, name string) string { } // Bind ports to keep them closed on the host -func bindPorts(ports []types.OCICNIPortMapping) ([]*os.File, error) { +func bindPorts(ports []types.PortMapping) ([]*os.File, error) { var files []*os.File - notifySCTP := false - for _, i := range ports { - isV6 := net.ParseIP(i.HostIP).To4() == nil - if i.HostIP == "" { + sctpWarning := true + for _, port := range ports { + isV6 := net.ParseIP(port.HostIP).To4() == nil + if port.HostIP == "" { isV6 = false } - switch i.Protocol { - case "udp": - var ( - addr *net.UDPAddr - err error - ) - if isV6 { - addr, err = net.ResolveUDPAddr("udp6", fmt.Sprintf("[%s]:%d", i.HostIP, i.HostPort)) - } else { - addr, err = net.ResolveUDPAddr("udp4", fmt.Sprintf("%s:%d", i.HostIP, i.HostPort)) - } - if err != nil { - return nil, errors.Wrapf(err, "cannot resolve the UDP address") + protocols := strings.Split(port.Protocol, ",") + for _, protocol := range protocols { + for i := uint16(0); i < port.Range; i++ { + f, err := bindPort(protocol, port.HostIP, port.HostPort+i, isV6, &sctpWarning) + if err != nil { + return files, err + } + if f != nil { + files = append(files, f) + } } + } + } + return files, nil +} - proto := "udp4" - if isV6 { - proto = "udp6" - } - server, err := net.ListenUDP(proto, addr) - if err != nil { - return nil, errors.Wrapf(err, "cannot listen on the UDP port") - } - f, err := server.File() - if err != nil { - return nil, errors.Wrapf(err, "cannot get file for UDP socket") - } - files = append(files, f) - // close the listener - // note that this does not affect the fd, see the godoc for server.File() - err = server.Close() - if err != nil { - logrus.Warnf("Failed to close connection: %v", err) - } +func bindPort(protocol, hostIP string, port uint16, isV6 bool, sctpWarning *bool) (*os.File, error) { + var file *os.File + switch protocol { + case "udp": + var ( + addr *net.UDPAddr + err error + ) + if isV6 { + addr, err = net.ResolveUDPAddr("udp6", fmt.Sprintf("[%s]:%d", hostIP, port)) + } else { + addr, err = net.ResolveUDPAddr("udp4", fmt.Sprintf("%s:%d", hostIP, port)) + } + if err != nil { + return nil, errors.Wrapf(err, "cannot resolve the UDP address") + } - case "tcp": - var ( - addr *net.TCPAddr - err error - ) - if isV6 { - addr, err = net.ResolveTCPAddr("tcp6", fmt.Sprintf("[%s]:%d", i.HostIP, i.HostPort)) - } else { - addr, err = net.ResolveTCPAddr("tcp4", fmt.Sprintf("%s:%d", i.HostIP, i.HostPort)) - } - if err != nil { - return nil, errors.Wrapf(err, "cannot resolve the TCP address") - } + proto := "udp4" + if isV6 { + proto = "udp6" + } + server, err := net.ListenUDP(proto, addr) + if err != nil { + return nil, errors.Wrapf(err, "cannot listen on the UDP port") + } + file, err = server.File() + if err != nil { + return nil, errors.Wrapf(err, "cannot get file for UDP socket") + } + // close the listener + // note that this does not affect the fd, see the godoc for server.File() + err = server.Close() + if err != nil { + logrus.Warnf("Failed to close connection: %v", err) + } - proto := "tcp4" - if isV6 { - proto = "tcp6" - } - server, err := net.ListenTCP(proto, addr) - if err != nil { - return nil, errors.Wrapf(err, "cannot listen on the TCP port") - } - f, err := server.File() - if err != nil { - return nil, errors.Wrapf(err, "cannot get file for TCP socket") - } - files = append(files, f) - // close the listener - // note that this does not affect the fd, see the godoc for server.File() - err = server.Close() - if err != nil { - logrus.Warnf("Failed to close connection: %v", err) - } + case "tcp": + var ( + addr *net.TCPAddr + err error + ) + if isV6 { + addr, err = net.ResolveTCPAddr("tcp6", fmt.Sprintf("[%s]:%d", hostIP, port)) + } else { + addr, err = net.ResolveTCPAddr("tcp4", fmt.Sprintf("%s:%d", hostIP, port)) + } + if err != nil { + return nil, errors.Wrapf(err, "cannot resolve the TCP address") + } - case "sctp": - if !notifySCTP { - notifySCTP = true - logrus.Warnf("Port reservation for SCTP is not supported") - } - default: - return nil, fmt.Errorf("unknown protocol %s", i.Protocol) + proto := "tcp4" + if isV6 { + proto = "tcp6" + } + server, err := net.ListenTCP(proto, addr) + if err != nil { + return nil, errors.Wrapf(err, "cannot listen on the TCP port") + } + file, err = server.File() + if err != nil { + return nil, errors.Wrapf(err, "cannot get file for TCP socket") + } + // close the listener + // note that this does not affect the fd, see the godoc for server.File() + err = server.Close() + if err != nil { + logrus.Warnf("Failed to close connection: %v", err) } + + case "sctp": + if *sctpWarning { + logrus.Info("Port reservation for SCTP is not supported") + *sctpWarning = false + } + default: + return nil, fmt.Errorf("unknown protocol %s", protocol) } - return files, nil + return file, nil } func getOCIRuntimeError(runtimeMsg string) error { diff --git a/libpod/options.go b/libpod/options.go index 135b2f363..250b16556 100644 --- a/libpod/options.go +++ b/libpod/options.go @@ -1051,7 +1051,7 @@ func WithDependencyCtrs(ctrs []*Container) CtrCreateOption { // namespace with a minimal configuration. // An optional array of port mappings can be provided. // Conflicts with WithNetNSFrom(). -func WithNetNS(portMappings []nettypes.OCICNIPortMapping, exposedPorts map[uint16][]string, postConfigureNetNS bool, netmode string, networks []string) CtrCreateOption { +func WithNetNS(portMappings []nettypes.PortMapping, exposedPorts map[uint16][]string, postConfigureNetNS bool, netmode string, networks []string) CtrCreateOption { return func(ctr *Container) error { if ctr.valid { return define.ErrCtrFinalized @@ -1104,7 +1104,7 @@ func WithNetworkOptions(options map[string][]string) CtrCreateOption { // It cannot be set unless WithNetNS has already been passed. // Further, it cannot be set if additional CNI networks to join have been // specified. -func WithStaticMAC(mac net.HardwareAddr) CtrCreateOption { +func WithStaticMAC(mac nettypes.HardwareAddr) CtrCreateOption { return func(ctr *Container) error { if ctr.valid { return define.ErrCtrFinalized @@ -2084,21 +2084,6 @@ func WithInfraContainer() PodCreateOption { } } -// WithInfraContainerPorts tells the pod to add port bindings to the pause container -func WithInfraContainerPorts(bindings []nettypes.OCICNIPortMapping, infraSpec *specgen.SpecGenerator) []nettypes.PortMapping { - bindingSpec := []nettypes.PortMapping{} - for _, bind := range bindings { - currBind := nettypes.PortMapping{} - currBind.ContainerPort = uint16(bind.ContainerPort) - currBind.HostIP = bind.HostIP - currBind.HostPort = uint16(bind.HostPort) - currBind.Protocol = bind.Protocol - bindingSpec = append(bindingSpec, currBind) - } - infraSpec.PortMappings = bindingSpec - return infraSpec.PortMappings -} - // WithVolatile sets the volatile flag for the container storage. // The option can potentially cause data loss when used on a container that must survive a machine reboot. func WithVolatile() CtrCreateOption { diff --git a/libpod/runtime_ctr.go b/libpod/runtime_ctr.go index 0a7db33f1..114bf9315 100644 --- a/libpod/runtime_ctr.go +++ b/libpod/runtime_ctr.go @@ -326,15 +326,6 @@ func (r *Runtime) setupContainer(ctx context.Context, ctr *Container) (_ *Contai } } - if ctr.config.Name == "" { - name, err := r.generateName() - if err != nil { - return nil, err - } - - ctr.config.Name = name - } - // Check CGroup parent sanity, and set it if it was not set. // Only if we're actually configuring CGroups. if !ctr.config.NoCgroups { diff --git a/libpod/runtime_pod_linux.go b/libpod/runtime_pod_linux.go index 7d7fef4d1..15050ef48 100644 --- a/libpod/runtime_pod_linux.go +++ b/libpod/runtime_pod_linux.go @@ -43,18 +43,6 @@ func (r *Runtime) NewPod(ctx context.Context, p specgen.PodSpecGenerator, option } } - if pod.config.Name == "" { - name, err := r.generateName() - if err != nil { - return nil, err - } - pod.config.Name = name - } - - if p.InfraContainerSpec != nil && p.InfraContainerSpec.Hostname == "" { - p.InfraContainerSpec.Hostname = pod.config.Name - } - // Allocate a lock for the pod lock, err := r.lockManager.AllocateLock() if err != nil { @@ -131,9 +119,33 @@ func (r *Runtime) NewPod(ctx context.Context, p specgen.PodSpecGenerator, option logrus.Infof("Pod has an infra container, but shares no namespaces") } - if err := r.state.AddPod(pod); err != nil { - return nil, errors.Wrapf(err, "error adding pod to state") + // Unless the user has specified a name, use a randomly generated one. + // Note that name conflicts may occur (see #11735), so we need to loop. + generateName := pod.config.Name == "" + var addPodErr error + for { + if generateName { + name, err := r.generateName() + if err != nil { + return nil, err + } + pod.config.Name = name + } + + if p.InfraContainerSpec != nil && p.InfraContainerSpec.Hostname == "" { + p.InfraContainerSpec.Hostname = pod.config.Name + } + if addPodErr = r.state.AddPod(pod); addPodErr == nil { + return pod, nil + } + if !generateName || (errors.Cause(addPodErr) != define.ErrPodExists && errors.Cause(addPodErr) != define.ErrCtrExists) { + break + } + } + if addPodErr != nil { + return nil, errors.Wrapf(addPodErr, "error adding pod to state") } + return pod, nil } diff --git a/libpod/state_test.go b/libpod/state_test.go index 4799d7b8d..5c3b0d7f7 100644 --- a/libpod/state_test.go +++ b/libpod/state_test.go @@ -11,6 +11,7 @@ import ( "github.com/containers/common/pkg/config" "github.com/containers/podman/v3/libpod/define" "github.com/containers/podman/v3/libpod/lock" + "github.com/containers/podman/v3/libpod/network/types" "github.com/containers/storage" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -3705,3 +3706,42 @@ func TestGetContainerConfigNonExistentIDFails(t *testing.T) { assert.Error(t, err) }) } + +// Test that the state will convert the ports to the new format +func TestConvertPortMapping(t *testing.T) { + runForAllStates(t, func(t *testing.T, state State, manager lock.Manager) { + testCtr, err := getTestCtr1(manager) + assert.NoError(t, err) + + ports := testCtr.config.PortMappings + + oldPorts := []types.OCICNIPortMapping{ + { + HostPort: 80, + ContainerPort: 90, + Protocol: "tcp", + HostIP: "192.168.3.3", + }, + { + HostPort: 100, + ContainerPort: 110, + Protocol: "udp", + HostIP: "192.168.4.4", + }, + } + + testCtr.config.OldPortMappings = oldPorts + testCtr.config.PortMappings = nil + + err = state.AddContainer(testCtr) + assert.NoError(t, err) + + retrievedCtr, err := state.Container(testCtr.ID()) + assert.NoError(t, err) + + // set values to expected ones + testCtr.config.PortMappings = ports + + testContainersEqual(t, retrievedCtr, testCtr, true) + }) +} diff --git a/libpod/util.go b/libpod/util.go index 8f8303ff2..5154a261e 100644 --- a/libpod/util.go +++ b/libpod/util.go @@ -295,19 +295,21 @@ func writeHijackHeader(r *http.Request, conn io.Writer) { } // Convert OCICNI port bindings into Inspect-formatted port bindings. -func makeInspectPortBindings(bindings []types.OCICNIPortMapping, expose map[uint16][]string) map[string][]define.InspectHostPort { +func makeInspectPortBindings(bindings []types.PortMapping, expose map[uint16][]string) map[string][]define.InspectHostPort { portBindings := make(map[string][]define.InspectHostPort) for _, port := range bindings { - key := fmt.Sprintf("%d/%s", port.ContainerPort, port.Protocol) - hostPorts := portBindings[key] - if hostPorts == nil { - hostPorts = []define.InspectHostPort{} + protocols := strings.Split(port.Protocol, ",") + for _, protocol := range protocols { + for i := uint16(0); i < port.Range; i++ { + key := fmt.Sprintf("%d/%s", port.ContainerPort+i, protocol) + hostPorts := portBindings[key] + hostPorts = append(hostPorts, define.InspectHostPort{ + HostIP: port.HostIP, + HostPort: fmt.Sprintf("%d", port.HostPort+i), + }) + portBindings[key] = hostPorts + } } - hostPorts = append(hostPorts, define.InspectHostPort{ - HostIP: port.HostIP, - HostPort: fmt.Sprintf("%d", port.HostPort), - }) - portBindings[key] = hostPorts } // add exposed ports without host port information to match docker for port, protocols := range expose { diff --git a/pkg/api/handlers/compat/containers_top.go b/pkg/api/handlers/compat/containers_top.go index b5debd37d..545320ad9 100644 --- a/pkg/api/handlers/compat/containers_top.go +++ b/pkg/api/handlers/compat/containers_top.go @@ -1,8 +1,11 @@ package compat import ( + "encoding/json" + "fmt" "net/http" "strings" + "time" "github.com/containers/podman/v3/libpod" "github.com/containers/podman/v3/pkg/api/handlers" @@ -10,20 +13,24 @@ import ( api "github.com/containers/podman/v3/pkg/api/types" "github.com/gorilla/schema" "github.com/pkg/errors" + "github.com/sirupsen/logrus" ) func TopContainer(w http.ResponseWriter, r *http.Request) { runtime := r.Context().Value(api.RuntimeKey).(*libpod.Runtime) decoder := r.Context().Value(api.DecoderKey).(*schema.Decoder) - defaultValue := "-ef" + psArgs := "-ef" if utils.IsLibpodRequest(r) { - defaultValue = "" + psArgs = "" } query := struct { + Delay int `schema:"delay"` PsArgs string `schema:"ps_args"` + Stream bool `schema:"stream"` }{ - PsArgs: defaultValue, + Delay: 5, + PsArgs: psArgs, } if err := decoder.Decode(&query, r.URL.Query()); err != nil { utils.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest, @@ -31,6 +38,12 @@ func TopContainer(w http.ResponseWriter, r *http.Request) { return } + if query.Delay < 1 { + utils.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest, + fmt.Errorf("\"delay\" parameter of value %d < 1", query.Delay)) + return + } + name := utils.GetName(r) c, err := runtime.LookupContainer(name) if err != nil { @@ -38,26 +51,56 @@ func TopContainer(w http.ResponseWriter, r *http.Request) { return } - output, err := c.Top([]string{query.PsArgs}) - if err != nil { - utils.InternalServerError(w, err) - return + // We are committed now - all errors logged but not reported to client, ship has sailed + w.WriteHeader(http.StatusOK) + w.Header().Set("Content-Type", "application/json") + if f, ok := w.(http.Flusher); ok { + f.Flush() } - var body = handlers.ContainerTopOKBody{} - if len(output) > 0 { - body.Titles = strings.Split(output[0], "\t") - for i := range body.Titles { - body.Titles[i] = strings.TrimSpace(body.Titles[i]) - } + encoder := json.NewEncoder(w) + +loop: // break out of for/select infinite` loop + for { + select { + case <-r.Context().Done(): + break loop + default: + output, err := c.Top([]string{query.PsArgs}) + if err != nil { + logrus.Infof("Error from %s %q : %v", r.Method, r.URL, err) + break loop + } + + if len(output) > 0 { + body := handlers.ContainerTopOKBody{} + body.Titles = strings.Split(output[0], "\t") + for i := range body.Titles { + body.Titles[i] = strings.TrimSpace(body.Titles[i]) + } + + for _, line := range output[1:] { + process := strings.Split(line, "\t") + for i := range process { + process[i] = strings.TrimSpace(process[i]) + } + body.Processes = append(body.Processes, process) + } + + if err := encoder.Encode(body); err != nil { + logrus.Infof("Error from %s %q : %v", r.Method, r.URL, err) + break loop + } + if f, ok := w.(http.Flusher); ok { + f.Flush() + } + } - for _, line := range output[1:] { - process := strings.Split(line, "\t") - for i := range process { - process[i] = strings.TrimSpace(process[i]) + if query.Stream { + time.Sleep(time.Duration(query.Delay) * time.Second) + } else { + break loop } - body.Processes = append(body.Processes, process) } } - utils.WriteJSON(w, http.StatusOK, body) } diff --git a/pkg/api/handlers/libpod/pods.go b/pkg/api/handlers/libpod/pods.go index 1e64de0ee..2ba292579 100644 --- a/pkg/api/handlers/libpod/pods.go +++ b/pkg/api/handlers/libpod/pods.go @@ -5,6 +5,7 @@ import ( "fmt" "net/http" "strings" + "time" "github.com/containers/common/pkg/config" "github.com/containers/podman/v3/libpod" @@ -363,10 +364,17 @@ func PodTop(w http.ResponseWriter, r *http.Request) { runtime := r.Context().Value(api.RuntimeKey).(*libpod.Runtime) decoder := r.Context().Value(api.DecoderKey).(*schema.Decoder) + psArgs := "-ef" + if utils.IsLibpodRequest(r) { + psArgs = "" + } query := struct { + Delay int `schema:"delay"` PsArgs string `schema:"ps_args"` + Stream bool `schema:"stream"` }{ - PsArgs: "", + Delay: 5, + PsArgs: psArgs, } if err := decoder.Decode(&query, r.URL.Query()); err != nil { utils.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest, @@ -374,31 +382,71 @@ func PodTop(w http.ResponseWriter, r *http.Request) { return } - name := utils.GetName(r) - pod, err := runtime.LookupPod(name) - if err != nil { - utils.PodNotFound(w, name, err) + if query.Delay < 1 { + utils.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest, + fmt.Errorf("\"delay\" parameter of value %d < 1", query.Delay)) return } - args := []string{} - if query.PsArgs != "" { - args = append(args, query.PsArgs) - } - output, err := pod.GetPodPidInformation(args) + name := utils.GetName(r) + pod, err := runtime.LookupPod(name) if err != nil { - utils.InternalServerError(w, err) + utils.PodNotFound(w, name, err) return } - var body = handlers.PodTopOKBody{} - if len(output) > 0 { - body.Titles = strings.Split(output[0], "\t") - for _, line := range output[1:] { - body.Processes = append(body.Processes, strings.Split(line, "\t")) + // We are committed now - all errors logged but not reported to client, ship has sailed + w.WriteHeader(http.StatusOK) + w.Header().Set("Content-Type", "application/json") + if f, ok := w.(http.Flusher); ok { + f.Flush() + } + + encoder := json.NewEncoder(w) + +loop: // break out of for/select infinite` loop + for { + select { + case <-r.Context().Done(): + break loop + default: + output, err := pod.GetPodPidInformation([]string{query.PsArgs}) + if err != nil { + logrus.Infof("Error from %s %q : %v", r.Method, r.URL, err) + break loop + } + + if len(output) > 0 { + var body = handlers.PodTopOKBody{} + body.Titles = strings.Split(output[0], "\t") + for i := range body.Titles { + body.Titles[i] = strings.TrimSpace(body.Titles[i]) + } + + for _, line := range output[1:] { + process := strings.Split(line, "\t") + for i := range process { + process[i] = strings.TrimSpace(process[i]) + } + body.Processes = append(body.Processes, process) + } + + if err := encoder.Encode(body); err != nil { + logrus.Infof("Error from %s %q : %v", r.Method, r.URL, err) + break loop + } + if f, ok := w.(http.Flusher); ok { + f.Flush() + } + } + + if query.Stream { + time.Sleep(time.Duration(query.Delay) * time.Second) + } else { + break loop + } } } - utils.WriteJSON(w, http.StatusOK, body) } func PodKill(w http.ResponseWriter, r *http.Request) { diff --git a/pkg/api/handlers/types.go b/pkg/api/handlers/types.go index b90154e30..35120a1a5 100644 --- a/pkg/api/handlers/types.go +++ b/pkg/api/handlers/types.go @@ -42,9 +42,15 @@ type ContainersPruneReport struct { } type LibpodContainersPruneReport struct { - ID string `json:"id"` - SpaceReclaimed int64 `json:"space"` - PruneError string `json:"error"` + ID string `json:"Id"` + SpaceReclaimed int64 `json:"Size"` + // Error which occurred during prune operation (if any). + // This field is optional and may be omitted if no error occurred. + // + // Extensions: + // x-omitempty: true + // x-nullable: true + PruneError string `json:"Err,omitempty"` } type Info struct { diff --git a/pkg/api/handlers/utils/handler.go b/pkg/api/handlers/utils/handler.go index 29139a98e..96b7a957c 100644 --- a/pkg/api/handlers/utils/handler.go +++ b/pkg/api/handlers/utils/handler.go @@ -145,12 +145,12 @@ func MarshalErrorSliceJSON(ptr unsafe.Pointer, stream *jsoniter.Stream) { } } -func MarshalErrorJSONIsEmpty(_ unsafe.Pointer) bool { - return false +func MarshalErrorJSONIsEmpty(ptr unsafe.Pointer) bool { + return *((*error)(ptr)) == nil } -func MarshalErrorSliceJSONIsEmpty(_ unsafe.Pointer) bool { - return false +func MarshalErrorSliceJSONIsEmpty(ptr unsafe.Pointer) bool { + return len(*((*[]error)(ptr))) <= 0 } // WriteJSON writes an interface value encoded as JSON to w diff --git a/pkg/api/handlers/utils/handler_test.go b/pkg/api/handlers/utils/handler_test.go index 18a1d2678..5957e7d74 100644 --- a/pkg/api/handlers/utils/handler_test.go +++ b/pkg/api/handlers/utils/handler_test.go @@ -138,3 +138,51 @@ func TestEqualVersion(t *testing.T) { rr.Body.String(), expected) } } + +func TestErrorEncoderFuncOmit(t *testing.T) { + data, err := json.Marshal(struct { + Err error `json:"err,omitempty"` + Errs []error `json:"errs,omitempty"` + }{}) + if err != nil { + t.Fatal(err) + } + + dataAsMap := make(map[string]interface{}) + err = json.Unmarshal(data, &dataAsMap) + if err != nil { + t.Fatal(err) + } + + _, ok := dataAsMap["err"] + if ok { + t.Errorf("the `err` field should have been omitted") + } + _, ok = dataAsMap["errs"] + if ok { + t.Errorf("the `errs` field should have been omitted") + } + + dataAsMap = make(map[string]interface{}) + data, err = json.Marshal(struct { + Err error `json:"err"` + Errs []error `json:"errs"` + }{}) + if err != nil { + t.Fatal(err) + } + + err = json.Unmarshal(data, &dataAsMap) + if err != nil { + t.Fatal(err) + } + + _, ok = dataAsMap["err"] + if !ok { + t.Errorf("the `err` field shouldn't have been omitted") + } + _, ok = dataAsMap["errs"] + if !ok { + t.Errorf("the `errs` field shouldn't have been omitted") + } +} diff --git a/pkg/api/server/handler_rid.go b/pkg/api/server/handler_rid.go index b624b99a6..7dcf436f7 100644 --- a/pkg/api/server/handler_rid.go +++ b/pkg/api/server/handler_rid.go @@ -2,6 +2,7 @@ package server import ( "fmt" + "io/ioutil" "net/http" "github.com/containers/podman/v3/pkg/api/types" @@ -15,7 +16,13 @@ import ( // and Apache style request logging func referenceIDHandler() mux.MiddlewareFunc { return func(h http.Handler) http.Handler { - return handlers.CombinedLoggingHandler(logrus.StandardLogger().Out, + // Only log Apache access_log-like entries at Info level or below + out := ioutil.Discard + if logrus.IsLevelEnabled(logrus.InfoLevel) { + out = logrus.StandardLogger().Out + } + + return handlers.CombinedLoggingHandler(out, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { rid := r.Header.Get("X-Reference-Id") if rid == "" { diff --git a/pkg/api/server/register_containers.go b/pkg/api/server/register_containers.go index 8dcea1301..c4919182b 100644 --- a/pkg/api/server/register_containers.go +++ b/pkg/api/server/register_containers.go @@ -442,6 +442,7 @@ func (s *APIServer) registerContainersHandlers(r *mux.Router) error { // - in: query // name: ps_args // type: string + // default: -ef // description: arguments to pass to ps such as aux. Requires ps(1) to be installed in the container if no ps(1) compatible AIX descriptors are used. // produces: // - application/json @@ -1142,19 +1143,23 @@ func (s *APIServer) registerContainersHandlers(r *mux.Router) error { // name: name // type: string // required: true - // description: | - // Name of container to query for processes - // (As of version 1.xx) + // description: Name of container to query for processes (As of version 1.xx) // - in: query // name: stream // type: boolean - // default: true - // description: Stream the output + // description: when true, repeatedly stream the latest output (As of version 4.0) + // - in: query + // name: delay + // type: integer + // description: if streaming, delay in seconds between updates. Must be >1. (As of version 4.0) + // default: 5 // - in: query // name: ps_args // type: string // default: -ef - // description: arguments to pass to ps such as aux. Requires ps(1) to be installed in the container if no ps(1) compatible AIX descriptors are used. + // description: | + // arguments to pass to ps such as aux. + // Requires ps(1) to be installed in the container if no ps(1) compatible AIX descriptors are used. // produces: // - application/json // responses: diff --git a/pkg/api/server/register_images.go b/pkg/api/server/register_images.go index 95a8b4939..38ceea271 100644 --- a/pkg/api/server/register_images.go +++ b/pkg/api/server/register_images.go @@ -103,7 +103,7 @@ func (s *APIServer) registerImagesHandlers(r *mux.Router) error { // - application/json // responses: // 200: - // $ref: "#/responses/DockerImageSummary" + // $ref: "#/responses/DockerImageSummaryResponse" // 500: // $ref: '#/responses/InternalError' r.Handle(VersionedPath("/images/json"), s.APIHandler(compat.GetImages)).Methods(http.MethodGet) @@ -837,7 +837,7 @@ func (s *APIServer) registerImagesHandlers(r *mux.Router) error { // - application/json // responses: // 200: - // $ref: "#/responses/DockerImageSummary" + // $ref: "#/responses/LibpodImageSummaryResponse" // 500: // $ref: '#/responses/InternalError' r.Handle(VersionedPath("/libpod/images/json"), s.APIHandler(libpod.GetImages)).Methods(http.MethodGet) @@ -967,7 +967,7 @@ func (s *APIServer) registerImagesHandlers(r *mux.Router) error { // - application/json // responses: // 200: - // $ref: "#/responses/DocsImageDeleteResponse" + // $ref: "#/responses/DocsLibpodImagesRemoveResponse" // 400: // $ref: "#/responses/BadParamError" // 404: @@ -1069,7 +1069,7 @@ func (s *APIServer) registerImagesHandlers(r *mux.Router) error { // - application/json // responses: // 200: - // $ref: "#/responses/DocsImageDeleteResponse" + // $ref: "#/responses/DocsLibpodPruneResponse" // 500: // $ref: '#/responses/InternalError' r.Handle(VersionedPath("/libpod/images/prune"), s.APIHandler(libpod.PruneImages)).Methods(http.MethodPost) diff --git a/pkg/api/server/register_pods.go b/pkg/api/server/register_pods.go index de3669a0a..16a7bbb4c 100644 --- a/pkg/api/server/register_pods.go +++ b/pkg/api/server/register_pods.go @@ -296,18 +296,23 @@ func (s *APIServer) registerPodsHandlers(r *mux.Router) error { // name: name // type: string // required: true - // description: | - // Name of pod to query for processes + // description: Name of pod to query for processes // - in: query // name: stream // type: boolean - // default: true - // description: Stream the output + // description: when true, repeatedly stream the latest output (As of version 4.0) + // - in: query + // name: delay + // type: integer + // description: if streaming, delay in seconds between updates. Must be >1. (As of version 4.0) + // default: 5 // - in: query // name: ps_args // type: string // default: -ef - // description: arguments to pass to ps such as aux. Requires ps(1) to be installed in the container if no ps(1) compatible AIX descriptors are used. + // description: | + // arguments to pass to ps such as aux. + // Requires ps(1) to be installed in the container if no ps(1) compatible AIX descriptors are used. // responses: // 200: // $ref: "#/responses/DocsPodTopResponse" diff --git a/pkg/api/server/swagger.go b/pkg/api/server/swagger.go index 0fd66652e..3f8f6f9c5 100644 --- a/pkg/api/server/swagger.go +++ b/pkg/api/server/swagger.go @@ -6,6 +6,7 @@ import ( "github.com/containers/podman/v3/pkg/domain/entities" "github.com/containers/podman/v3/pkg/domain/entities/reports" "github.com/containers/podman/v3/pkg/errorhandling" + docker "github.com/docker/docker/api/types" ) // No such image @@ -134,9 +135,16 @@ type swagPodAlreadyStopped struct { } } -// Image summary -// swagger:response DockerImageSummary -type swagImageSummary struct { +// Image summary for compat API +// swagger:response DockerImageSummaryResponse +type swagDockerImageSummaryResponse struct { + // in:body + Body []docker.ImageSummary +} + +// Image summary for libpod API +// swagger:response LibpodImageSummaryResponse +type swagLibpodImageSummaryResponse struct { // in:body Body []entities.ImageSummary } diff --git a/pkg/bindings/connection.go b/pkg/bindings/connection.go index dc75dac5a..a2be44ab4 100644 --- a/pkg/bindings/connection.go +++ b/pkg/bindings/connection.go @@ -390,6 +390,11 @@ func (h *APIResponse) IsClientError() bool { return h.Response.StatusCode/100 == 4 } +// IsConflictError returns true if the response code is 409 +func (h *APIResponse) IsConflictError() bool { + return h.Response.StatusCode == 409 +} + // IsServerError returns true if the response code is 5xx func (h *APIResponse) IsServerError() bool { return h.Response.StatusCode/100 == 5 diff --git a/pkg/bindings/containers/attach.go b/pkg/bindings/containers/attach.go index c5f54c1af..47de89b33 100644 --- a/pkg/bindings/containers/attach.go +++ b/pkg/bindings/containers/attach.go @@ -214,7 +214,7 @@ func Attach(ctx context.Context, nameOrID string, stdin io.Reader, stdout io.Wri // Read multiplexed channels and write to appropriate stream fd, l, err := DemuxHeader(socket, buffer) if err != nil { - if errors.Is(err, io.EOF) { + if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) { return nil } return err @@ -531,7 +531,7 @@ func ExecStartAndAttach(ctx context.Context, sessionID string, options *ExecStar // Read multiplexed channels and write to appropriate stream fd, l, err := DemuxHeader(socket, buffer) if err != nil { - if errors.Is(err, io.EOF) { + if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) { return nil } return err diff --git a/pkg/bindings/containers/logs.go b/pkg/bindings/containers/logs.go index 67db94487..37ffdf0a5 100644 --- a/pkg/bindings/containers/logs.go +++ b/pkg/bindings/containers/logs.go @@ -39,7 +39,7 @@ func Logs(ctx context.Context, nameOrID string, options *LogOptions, stdoutChan, for { fd, l, err := DemuxHeader(response.Body, buffer) if err != nil { - if errors.Is(err, io.EOF) { + if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) { return nil } return err diff --git a/pkg/bindings/errors.go b/pkg/bindings/errors.go index 9c311d912..ec837b39c 100644 --- a/pkg/bindings/errors.go +++ b/pkg/bindings/errors.go @@ -12,17 +12,22 @@ var ( ErrNotImplemented = errors.New("function not implemented") ) -func handleError(data []byte) error { - e := errorhandling.ErrorModel{} - if err := json.Unmarshal(data, &e); err != nil { +func handleError(data []byte, unmarshalErrorInto interface{}) error { + if err := json.Unmarshal(data, unmarshalErrorInto); err != nil { return err } - return e + return unmarshalErrorInto.(error) } // Process drains the response body, and processes the HTTP status code // Note: Closing the response.Body is left to the caller func (h APIResponse) Process(unmarshalInto interface{}) error { + return h.ProcessWithError(unmarshalInto, &errorhandling.ErrorModel{}) +} + +// Process drains the response body, and processes the HTTP status code +// Note: Closing the response.Body is left to the caller +func (h APIResponse) ProcessWithError(unmarshalInto interface{}, unmarshalErrorInto interface{}) error { data, err := ioutil.ReadAll(h.Response.Body) if err != nil { return errors.Wrap(err, "unable to process API response") @@ -33,14 +38,22 @@ func (h APIResponse) Process(unmarshalInto interface{}) error { } return nil } + + if h.IsConflictError() { + return handleError(data, unmarshalErrorInto) + } + // TODO should we add a debug here with the response code? - return handleError(data) + return handleError(data, &errorhandling.ErrorModel{}) } func CheckResponseCode(inError error) (int, error) { - e, ok := inError.(errorhandling.ErrorModel) - if !ok { + switch e := inError.(type) { + case *errorhandling.ErrorModel: + return e.Code(), nil + case *errorhandling.PodConflictErrorModel: + return e.Code(), nil + default: return -1, errors.New("error is not type ErrorModel") } - return e.Code(), nil } diff --git a/pkg/bindings/pods/pods.go b/pkg/bindings/pods/pods.go index a1a431a3b..3b5832373 100644 --- a/pkg/bindings/pods/pods.go +++ b/pkg/bindings/pods/pods.go @@ -9,6 +9,7 @@ import ( "github.com/containers/podman/v3/pkg/api/handlers" "github.com/containers/podman/v3/pkg/bindings" "github.com/containers/podman/v3/pkg/domain/entities" + "github.com/containers/podman/v3/pkg/errorhandling" jsoniter "github.com/json-iterator/go" ) @@ -97,7 +98,7 @@ func Kill(ctx context.Context, nameOrID string, options *KillOptions) (*entities } defer response.Body.Close() - return &report, response.Process(&report) + return &report, response.ProcessWithError(&report, &errorhandling.PodConflictErrorModel{}) } // Pause pauses all running containers in a given pod. @@ -117,7 +118,7 @@ func Pause(ctx context.Context, nameOrID string, options *PauseOptions) (*entiti } defer response.Body.Close() - return &report, response.Process(&report) + return &report, response.ProcessWithError(&report, &errorhandling.PodConflictErrorModel{}) } // Prune by default removes all non-running pods in local storage. @@ -184,7 +185,7 @@ func Restart(ctx context.Context, nameOrID string, options *RestartOptions) (*en } defer response.Body.Close() - return &report, response.Process(&report) + return &report, response.ProcessWithError(&report, &errorhandling.PodConflictErrorModel{}) } // Remove deletes a Pod from from local storage. The optional force parameter denotes @@ -232,7 +233,8 @@ func Start(ctx context.Context, nameOrID string, options *StartOptions) (*entiti report.Id = nameOrID return &report, nil } - return &report, response.Process(&report) + + return &report, response.ProcessWithError(&report, &errorhandling.PodConflictErrorModel{}) } // Stop stops all containers in a Pod. The optional timeout parameter can be @@ -260,7 +262,7 @@ func Stop(ctx context.Context, nameOrID string, options *StopOptions) (*entities report.Id = nameOrID return &report, nil } - return &report, response.Process(&report) + return &report, response.ProcessWithError(&report, &errorhandling.PodConflictErrorModel{}) } // Top gathers statistics about the running processes in a pod. The nameOrID can be a pod name @@ -316,7 +318,7 @@ func Unpause(ctx context.Context, nameOrID string, options *UnpauseOptions) (*en } defer response.Body.Close() - return &report, response.Process(&report) + return &report, response.ProcessWithError(&report, &errorhandling.PodConflictErrorModel{}) } // Stats display resource-usage statistics of one or more pods. diff --git a/pkg/bindings/test/common_test.go b/pkg/bindings/test/common_test.go index 91ebe21fc..d996595bf 100644 --- a/pkg/bindings/test/common_test.go +++ b/pkg/bindings/test/common_test.go @@ -225,12 +225,23 @@ func (b *bindingTest) RunTopContainer(containerName *string, podName *string) (s // This method creates a pod with the given pod name. // Podname is an optional parameter func (b *bindingTest) Podcreate(name *string) { + b.PodcreateAndExpose(name, nil) +} + +// This method creates a pod with the given pod name and publish port. +// Podname is an optional parameter +// port is an optional parameter +func (b *bindingTest) PodcreateAndExpose(name *string, port *string) { + command := []string{"pod", "create"} if name != nil { podname := *name - b.runPodman([]string{"pod", "create", "--name", podname}).Wait(45) - } else { - b.runPodman([]string{"pod", "create"}).Wait(45) + command = append(command, "--name", podname) + } + if port != nil { + podport := *port + command = append(command, "--publish", podport) } + b.runPodman(command).Wait(45) } // StringInSlice returns a boolean based on whether a given diff --git a/pkg/bindings/test/containers_test.go b/pkg/bindings/test/containers_test.go index b9ed67255..0f535bc31 100644 --- a/pkg/bindings/test/containers_test.go +++ b/pkg/bindings/test/containers_test.go @@ -259,6 +259,7 @@ var _ = Describe("Podman containers ", func() { _, err = bt.RunTopContainer(&name, nil) Expect(err).To(BeNil()) go func() { + defer GinkgoRecover() exitCode, err = containers.Wait(bt.conn, name, nil) errChan <- err close(errChan) @@ -281,6 +282,7 @@ var _ = Describe("Podman containers ", func() { _, err := bt.RunTopContainer(&name, nil) Expect(err).To(BeNil()) go func() { + defer GinkgoRecover() exitCode, err = containers.Wait(bt.conn, name, new(containers.WaitOptions).WithCondition([]define.ContainerStatus{pause})) errChan <- err close(errChan) @@ -366,7 +368,10 @@ var _ = Describe("Podman containers ", func() { opts := new(containers.LogOptions).WithStdout(true).WithFollow(true) go func() { - containers.Logs(bt.conn, r.ID, opts, stdoutChan, nil) + defer GinkgoRecover() + err := containers.Logs(bt.conn, r.ID, opts, stdoutChan, nil) + close(stdoutChan) + Expect(err).ShouldNot(HaveOccurred()) }() o := <-stdoutChan o = strings.TrimSpace(o) diff --git a/pkg/bindings/test/pods_test.go b/pkg/bindings/test/pods_test.go index 5331cf439..879d4d00d 100644 --- a/pkg/bindings/test/pods_test.go +++ b/pkg/bindings/test/pods_test.go @@ -1,6 +1,7 @@ package test_bindings import ( + "fmt" "net/http" "strings" "time" @@ -9,7 +10,9 @@ import ( "github.com/containers/podman/v3/pkg/bindings" "github.com/containers/podman/v3/pkg/bindings/pods" "github.com/containers/podman/v3/pkg/domain/entities" + "github.com/containers/podman/v3/pkg/errorhandling" "github.com/containers/podman/v3/pkg/specgen" + "github.com/containers/podman/v3/utils" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/onsi/gomega/gexec" @@ -208,6 +211,29 @@ var _ = Describe("Podman pods", func() { } }) + It("start pod with port conflict", func() { + randomport, err := utils.GetRandomPort() + Expect(err).To(BeNil()) + + portPublish := fmt.Sprintf("%d:%d", randomport, randomport) + var podwithport string = "newpodwithport" + bt.PodcreateAndExpose(&podwithport, &portPublish) + + // Start pod and expose port 12345 + _, err = pods.Start(bt.conn, podwithport, nil) + Expect(err).To(BeNil()) + + // Start another pod and expose same port 12345 + var podwithport2 string = "newpodwithport2" + bt.PodcreateAndExpose(&podwithport2, &portPublish) + + _, err = pods.Start(bt.conn, podwithport2, nil) + Expect(err).ToNot(BeNil()) + code, _ := bindings.CheckResponseCode(err) + Expect(code).To(BeNumerically("==", http.StatusConflict)) + Expect(err).To(BeAssignableToTypeOf(&errorhandling.PodConflictErrorModel{})) + }) + It("start stop restart pod", func() { // Start an invalid pod _, err = pods.Start(bt.conn, "dummyName", nil) diff --git a/pkg/checkpoint/checkpoint_restore.go b/pkg/checkpoint/checkpoint_restore.go index f53e31f9b..da82c9745 100644 --- a/pkg/checkpoint/checkpoint_restore.go +++ b/pkg/checkpoint/checkpoint_restore.go @@ -6,6 +6,7 @@ import ( "os" metadata "github.com/checkpoint-restore/checkpointctl/lib" + "github.com/checkpoint-restore/go-criu/v5/stats" "github.com/containers/common/libimage" "github.com/containers/common/pkg/config" "github.com/containers/podman/v3/libpod" @@ -39,6 +40,7 @@ func CRImportCheckpoint(ctx context.Context, runtime *libpod.Runtime, restoreOpt "volumes", "ctr.log", "artifacts", + stats.StatsDump, metadata.RootFsDiffTar, metadata.DeletedFilesFile, metadata.NetworkStatusFile, @@ -193,7 +195,7 @@ func CRImportCheckpoint(ctx context.Context, runtime *libpod.Runtime, restoreOpt } if len(restoreOptions.PublishPorts) > 0 { - ports, _, _, err := generate.ParsePortMapping(restoreOptions.PublishPorts) + ports, err := generate.ParsePortMapping(restoreOptions.PublishPorts, nil) if err != nil { return nil, err } diff --git a/pkg/domain/entities/container_ps.go b/pkg/domain/entities/container_ps.go index 58f231a2f..d018d373f 100644 --- a/pkg/domain/entities/container_ps.go +++ b/pkg/domain/entities/container_ps.go @@ -54,7 +54,7 @@ type ListContainer struct { // boolean to be set PodName string // Port mappings - Ports []types.OCICNIPortMapping + Ports []types.PortMapping // Size of the container rootfs. Requires the size boolean to be true Size *define.ContainerSize // Time when container started diff --git a/pkg/domain/entities/containers.go b/pkg/domain/entities/containers.go index deae85fe1..869c616ea 100644 --- a/pkg/domain/entities/containers.go +++ b/pkg/domain/entities/containers.go @@ -422,7 +422,7 @@ type ContainerPortOptions struct { // the CLI to output ports type ContainerPortReport struct { Id string //nolint - Ports []nettypes.OCICNIPortMapping + Ports []nettypes.PortMapping } // ContainerCpOptions describes input options for cp. diff --git a/pkg/domain/entities/images.go b/pkg/domain/entities/images.go index 7ac07a8b3..7583ce442 100644 --- a/pkg/domain/entities/images.go +++ b/pkg/domain/entities/images.go @@ -50,6 +50,7 @@ func (i *Image) Id() string { // nolint return i.ID } +// swagger:model LibpodImageSummary type ImageSummary struct { ID string `json:"Id"` ParentId string // nolint diff --git a/pkg/domain/entities/pods.go b/pkg/domain/entities/pods.go index 3d8579acf..1df18be58 100644 --- a/pkg/domain/entities/pods.go +++ b/pkg/domain/entities/pods.go @@ -7,6 +7,7 @@ import ( commonFlag "github.com/containers/common/pkg/flag" "github.com/containers/podman/v3/libpod/define" + "github.com/containers/podman/v3/libpod/network/types" "github.com/containers/podman/v3/pkg/specgen" "github.com/containers/podman/v3/pkg/util" "github.com/opencontainers/runtime-spec/specs-go" @@ -318,7 +319,8 @@ func ToPodSpecGen(s specgen.PodSpecGenerator, p *PodCreateOptions) (*specgen.Pod if p.Net != nil { s.NetNS = p.Net.Network s.StaticIP = p.Net.StaticIP - s.StaticMAC = p.Net.StaticMAC + // type cast to types.HardwareAddr + s.StaticMAC = (*types.HardwareAddr)(p.Net.StaticMAC) s.PortMappings = p.Net.PublishPorts s.CNINetworks = p.Net.CNINetworks s.NetworkOptions = p.Net.NetworkOptions diff --git a/pkg/domain/entities/reports/prune.go b/pkg/domain/entities/reports/prune.go index 5494ac3ae..219e35b67 100644 --- a/pkg/domain/entities/reports/prune.go +++ b/pkg/domain/entities/reports/prune.go @@ -1,9 +1,9 @@ package reports type PruneReport struct { - Id string //nolint - Err error - Size uint64 + Id string `json:"Id"` //nolint + Err error `json:"Err,omitempty"` + Size uint64 `json:"Size"` } func PruneReportsIds(r []*PruneReport) []string { diff --git a/pkg/domain/entities/system.go b/pkg/domain/entities/system.go index fe041dec8..49f0c2323 100644 --- a/pkg/domain/entities/system.go +++ b/pkg/domain/entities/system.go @@ -100,7 +100,7 @@ type SystemVersionReport struct { // SystemUnshareOptions describes the options for the unshare command type SystemUnshareOptions struct { - RootlessCNI bool + RootlessNetNS bool } type ComponentVersion struct { diff --git a/pkg/domain/infra/abi/images.go b/pkg/domain/infra/abi/images.go index e9528acf1..5c0227986 100644 --- a/pkg/domain/infra/abi/images.go +++ b/pkg/domain/infra/abi/images.go @@ -61,7 +61,7 @@ func (ir *ImageEngine) Prune(ctx context.Context, opts entities.ImagePruneOption pruneOptions.Filters = append(pruneOptions.Filters, "containers=false") } - var pruneReports []*reports.PruneReport + pruneReports := make([]*reports.PruneReport, 0) // Now prune all images until we converge. numPreviouslyRemovedImages := 1 diff --git a/pkg/domain/infra/abi/system.go b/pkg/domain/infra/abi/system.go index e326f26a8..7da7754f2 100644 --- a/pkg/domain/infra/abi/system.go +++ b/pkg/domain/infra/abi/system.go @@ -360,15 +360,15 @@ func (ic *ContainerEngine) Unshare(ctx context.Context, args []string, options e return cmd.Run() } - if options.RootlessCNI { - rootlesscni, err := ic.Libpod.GetRootlessCNINetNs(true) + if options.RootlessNetNS { + rootlessNetNS, err := ic.Libpod.GetRootlessNetNs(true) if err != nil { return err } // make sure to unlock, unshare can run for a long time - rootlesscni.Lock.Unlock() - defer rootlesscni.Cleanup(ic.Libpod) - return rootlesscni.Do(unshare) + rootlessNetNS.Lock.Unlock() + defer rootlessNetNS.Cleanup(ic.Libpod) + return rootlessNetNS.Do(unshare) } return unshare() } diff --git a/pkg/domain/infra/tunnel/containers.go b/pkg/domain/infra/tunnel/containers.go index 3f78ba7bc..5b5a1912c 100644 --- a/pkg/domain/infra/tunnel/containers.go +++ b/pkg/domain/infra/tunnel/containers.go @@ -228,7 +228,7 @@ func (ic *ContainerEngine) ContainerInspect(ctx context.Context, namesOrIds []st for _, name := range namesOrIds { inspect, err := containers.Inspect(ic.ClientCtx, name, options) if err != nil { - errModel, ok := err.(errorhandling.ErrorModel) + errModel, ok := err.(*errorhandling.ErrorModel) if !ok { return nil, nil, err } diff --git a/pkg/domain/infra/tunnel/images.go b/pkg/domain/infra/tunnel/images.go index ed0c35de4..fde57972f 100644 --- a/pkg/domain/infra/tunnel/images.go +++ b/pkg/domain/infra/tunnel/images.go @@ -193,7 +193,7 @@ func (ir *ImageEngine) Inspect(ctx context.Context, namesOrIDs []string, opts en for _, i := range namesOrIDs { r, err := images.GetImage(ir.ClientCtx, i, options) if err != nil { - errModel, ok := err.(errorhandling.ErrorModel) + errModel, ok := err.(*errorhandling.ErrorModel) if !ok { return nil, nil, err } diff --git a/pkg/domain/infra/tunnel/network.go b/pkg/domain/infra/tunnel/network.go index 79fba1943..069982d30 100644 --- a/pkg/domain/infra/tunnel/network.go +++ b/pkg/domain/infra/tunnel/network.go @@ -25,7 +25,7 @@ func (ic *ContainerEngine) NetworkInspect(ctx context.Context, namesOrIds []stri for _, name := range namesOrIds { report, err := network.Inspect(ic.ClientCtx, name, options) if err != nil { - errModel, ok := err.(errorhandling.ErrorModel) + errModel, ok := err.(*errorhandling.ErrorModel) if !ok { return nil, nil, err } diff --git a/pkg/domain/infra/tunnel/secrets.go b/pkg/domain/infra/tunnel/secrets.go index 6337c7fbe..e5fa200bd 100644 --- a/pkg/domain/infra/tunnel/secrets.go +++ b/pkg/domain/infra/tunnel/secrets.go @@ -28,7 +28,7 @@ func (ic *ContainerEngine) SecretInspect(ctx context.Context, nameOrIDs []string for _, name := range nameOrIDs { inspected, err := secrets.Inspect(ic.ClientCtx, name, nil) if err != nil { - errModel, ok := err.(errorhandling.ErrorModel) + errModel, ok := err.(*errorhandling.ErrorModel) if !ok { return nil, nil, err } @@ -67,7 +67,7 @@ func (ic *ContainerEngine) SecretRm(ctx context.Context, nameOrIDs []string, opt for _, name := range nameOrIDs { secret, err := secrets.Inspect(ic.ClientCtx, name, nil) if err != nil { - errModel, ok := err.(errorhandling.ErrorModel) + errModel, ok := err.(*errorhandling.ErrorModel) if !ok { return nil, err } diff --git a/pkg/domain/infra/tunnel/volumes.go b/pkg/domain/infra/tunnel/volumes.go index cfd1574c3..ccb363935 100644 --- a/pkg/domain/infra/tunnel/volumes.go +++ b/pkg/domain/infra/tunnel/volumes.go @@ -59,7 +59,7 @@ func (ic *ContainerEngine) VolumeInspect(ctx context.Context, namesOrIds []strin for _, id := range namesOrIds { data, err := volumes.Inspect(ic.ClientCtx, id, nil) if err != nil { - errModel, ok := err.(errorhandling.ErrorModel) + errModel, ok := err.(*errorhandling.ErrorModel) if !ok { return nil, nil, err } diff --git a/pkg/errorhandling/errorhandling.go b/pkg/errorhandling/errorhandling.go index 44a0c3efd..04110b62a 100644 --- a/pkg/errorhandling/errorhandling.go +++ b/pkg/errorhandling/errorhandling.go @@ -83,6 +83,12 @@ func Contains(err error, sub error) bool { return strings.Contains(err.Error(), sub.Error()) } +// PodConflictErrorModel is used in remote connections with podman +type PodConflictErrorModel struct { + Errs []string + Id string //nolint +} + // ErrorModel is used in remote connections with podman type ErrorModel struct { // API root cause formatted for automated parsing @@ -106,3 +112,11 @@ func (e ErrorModel) Cause() error { func (e ErrorModel) Code() int { return e.ResponseCode } + +func (e PodConflictErrorModel) Error() string { + return strings.Join(e.Errs, ",") +} + +func (e PodConflictErrorModel) Code() int { + return 409 +} diff --git a/pkg/machine/qemu/machine.go b/pkg/machine/qemu/machine.go index 4d8242e39..a7174aac3 100644 --- a/pkg/machine/qemu/machine.go +++ b/pkg/machine/qemu/machine.go @@ -398,7 +398,6 @@ func (v *MachineVM) Stop(name string, _ machine.StopOptions) error { return err } - fmt.Printf("Successfully stopped machine: %s", name) return nil } diff --git a/pkg/rootlessport/rootlessport_linux.go b/pkg/rootlessport/rootlessport_linux.go index 7b9e5bbfa..7e6075789 100644 --- a/pkg/rootlessport/rootlessport_linux.go +++ b/pkg/rootlessport/rootlessport_linux.go @@ -23,7 +23,7 @@ const ( // Config needs to be provided to the process via stdin as a JSON string. // stdin needs to be closed after the message has been written. type Config struct { - Mappings []types.OCICNIPortMapping + Mappings []types.PortMapping NetNSPath string ExitFD int ReadyFD int diff --git a/pkg/specgen/generate/pod_create.go b/pkg/specgen/generate/pod_create.go index 88ebc7ae3..501bce05d 100644 --- a/pkg/specgen/generate/pod_create.go +++ b/pkg/specgen/generate/pod_create.go @@ -204,11 +204,11 @@ func createPodOptions(p *specgen.PodSpecGenerator, rt *libpod.Runtime, infraSpec // replacing necessary values with those specified in pod creation func MapSpec(p *specgen.PodSpecGenerator) (*specgen.SpecGenerator, error) { if len(p.PortMappings) > 0 { - ports, _, _, err := ParsePortMapping(p.PortMappings) + ports, err := ParsePortMapping(p.PortMappings, nil) if err != nil { return nil, err } - p.InfraContainerSpec.PortMappings = libpod.WithInfraContainerPorts(ports, p.InfraContainerSpec) + p.InfraContainerSpec.PortMappings = ports } switch p.NetNS.NSMode { case specgen.Default, "": diff --git a/pkg/specgen/generate/ports.go b/pkg/specgen/generate/ports.go index 992b4a8e9..53a5e5697 100644 --- a/pkg/specgen/generate/ports.go +++ b/pkg/specgen/generate/ports.go @@ -2,7 +2,9 @@ package generate import ( "context" + "fmt" "net" + "sort" "strconv" "strings" @@ -11,6 +13,7 @@ import ( "github.com/containers/podman/v3/utils" "github.com/containers/podman/v3/pkg/specgen" + "github.com/containers/podman/v3/pkg/util" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -21,252 +24,323 @@ const ( protoSCTP = "sctp" ) -// Parse port maps to OCICNI port mappings. -// Returns a set of OCICNI port mappings, and maps of utilized container and +// joinTwoPortsToRangePortIfPossible will expect two ports the previous port one must have a lower or equal hostPort than the current port. +func joinTwoPortsToRangePortIfPossible(ports *[]types.PortMapping, allHostPorts, allContainerPorts, currentHostPorts *[65536]bool, + previousPort *types.PortMapping, port types.PortMapping) (*types.PortMapping, error) { + // no previous port just return the current one + if previousPort == nil { + return &port, nil + } + if previousPort.HostPort+previousPort.Range >= port.HostPort { + // check if the port range matches the host and container ports + portDiff := port.HostPort - previousPort.HostPort + if portDiff == port.ContainerPort-previousPort.ContainerPort { + // calc the new range use the old range and add the difference between the ports + newRange := port.Range + portDiff + // if the newRange is greater than the old range use it + // this is important otherwise we would could lower the range + if newRange > previousPort.Range { + previousPort.Range = newRange + } + return previousPort, nil + } + // if both host port ranges overlap and the container port range did not match + // we have to error because we cannot assign the same host port to more than one container port + if previousPort.HostPort+previousPort.Range-1 > port.HostPort { + return nil, errors.Errorf("conflicting port mappings for host port %d (protocol %s)", port.HostPort, port.Protocol) + } + } + // we could not join the ports so we append the old one to the list + // and return the current port as previous port + addPortToUsedPorts(ports, allHostPorts, allContainerPorts, currentHostPorts, previousPort) + return &port, nil +} + +// joinTwoContainerPortsToRangePortIfPossible will expect two ports with both no host port set, +// the previous port one must have a lower or equal containerPort than the current port. +func joinTwoContainerPortsToRangePortIfPossible(ports *[]types.PortMapping, allHostPorts, allContainerPorts, currentHostPorts *[65536]bool, + previousPort *types.PortMapping, port types.PortMapping) (*types.PortMapping, error) { + // no previous port just return the current one + if previousPort == nil { + return &port, nil + } + if previousPort.ContainerPort+previousPort.Range > port.ContainerPort { + // calc the new range use the old range and add the difference between the ports + newRange := port.ContainerPort - previousPort.ContainerPort + port.Range + // if the newRange is greater than the old range use it + // this is important otherwise we would could lower the range + if newRange > previousPort.Range { + previousPort.Range = newRange + } + return previousPort, nil + } + // we could not join the ports so we append the old one to the list + // and return the current port as previous port + newPort, err := getRandomHostPort(currentHostPorts, *previousPort) + if err != nil { + return nil, err + } + addPortToUsedPorts(ports, allHostPorts, allContainerPorts, currentHostPorts, &newPort) + return &port, nil +} + +func addPortToUsedPorts(ports *[]types.PortMapping, allHostPorts, allContainerPorts, currentHostPorts *[65536]bool, port *types.PortMapping) { + for i := uint16(0); i < port.Range; i++ { + h := port.HostPort + i + allHostPorts[h] = true + currentHostPorts[h] = true + c := port.ContainerPort + i + allContainerPorts[c] = true + } + *ports = append(*ports, *port) +} + +// getRandomHostPort get a random host port mapping for the given port +// the caller has to supply a array with he already used ports +func getRandomHostPort(hostPorts *[65536]bool, port types.PortMapping) (types.PortMapping, error) { +outer: + for i := 0; i < 15; i++ { + ranPort, err := utils.GetRandomPort() + if err != nil { + return port, err + } + + // if port range is exceeds max port we cannot use it + if ranPort+int(port.Range) > 65535 { + continue + } + + // check if there is a port in the range which is used + for j := 0; j < int(port.Range); j++ { + // port already used + if hostPorts[ranPort+j] { + continue outer + } + } + + port.HostPort = uint16(ranPort) + return port, nil + } + + // add range to error message if needed + rangePort := "" + if port.Range > 1 { + rangePort = fmt.Sprintf("with range %d ", port.Range) + } + + return port, errors.Errorf("failed to find an open port to expose container port %d %son the host", port.ContainerPort, rangePort) +} + +// Parse port maps to port mappings. +// Returns a set of port mappings, and maps of utilized container and // host ports. -func ParsePortMapping(portMappings []types.PortMapping) ([]types.OCICNIPortMapping, map[string]map[string]map[uint16]uint16, map[string]map[string]map[uint16]uint16, error) { - // First, we need to validate the ports passed in the specgen, and then - // convert them into CNI port mappings. - type tempMapping struct { - mapping types.OCICNIPortMapping - startOfRange bool - isInRange bool +func ParsePortMapping(portMappings []types.PortMapping, exposePorts map[uint16][]string) ([]types.PortMapping, error) { + if len(portMappings) == 0 && len(exposePorts) == 0 { + return nil, nil } - tempMappings := []tempMapping{} - - // To validate, we need two maps: one for host ports, one for container - // ports. - // Each is a map of protocol to map of IP address to map of port to - // port (for hostPortValidate, it's host port to container port; - // for containerPortValidate, container port to host port. - // These will ensure no collisions. - hostPortValidate := make(map[string]map[string]map[uint16]uint16) - containerPortValidate := make(map[string]map[string]map[uint16]uint16) - - // Initialize the first level of maps (we can't really guess keys for - // the rest). - for _, proto := range []string{protoTCP, protoUDP, protoSCTP} { - hostPortValidate[proto] = make(map[string]map[uint16]uint16) - containerPortValidate[proto] = make(map[string]map[uint16]uint16) + + // tempMapping stores the ports without ip and protocol + type tempMapping struct { + hostPort uint16 + containerPort uint16 + rangePort uint16 } - postAssignHostPort := false + // portMap is a temporary structure to sort all ports + // the map is hostIp -> protocol -> array of mappings + portMap := make(map[string]map[string][]tempMapping) + + // allUsedContainerPorts stores all used ports for each protocol + // the key is the protocol and the array is 65536 elements long for each port. + allUsedContainerPortsMap := make(map[string][65536]bool) + allUsedHostPortsMap := make(map[string][65536]bool) - // Iterate through all port mappings, generating OCICNI PortMapping - // structs and validating there is no overlap. + // First, we need to validate the ports passed in the specgen for _, port := range portMappings { // First, check proto protocols, err := checkProtocol(port.Protocol, true) if err != nil { - return nil, nil, nil, err - } - - // Validate host IP - hostIP := port.HostIP - if hostIP == "" { - hostIP = "0.0.0.0" + return nil, err } - if ip := net.ParseIP(hostIP); ip == nil { - return nil, nil, nil, errors.Errorf("invalid IP address %s in port mapping", port.HostIP) + if port.HostIP != "" { + if ip := net.ParseIP(port.HostIP); ip == nil { + return nil, errors.Errorf("invalid IP address %q in port mapping", port.HostIP) + } } // Validate port numbers and range. - len := port.Range - if len == 0 { - len = 1 + portRange := port.Range + if portRange == 0 { + portRange = 1 } containerPort := port.ContainerPort if containerPort == 0 { - return nil, nil, nil, errors.Errorf("container port number must be non-0") + return nil, errors.Errorf("container port number must be non-0") } hostPort := port.HostPort - if uint32(len-1)+uint32(containerPort) > 65535 { - return nil, nil, nil, errors.Errorf("container port range exceeds maximum allowable port number") + if uint32(portRange-1)+uint32(containerPort) > 65535 { + return nil, errors.Errorf("container port range exceeds maximum allowable port number") } - if uint32(len-1)+uint32(hostPort) > 65536 { - return nil, nil, nil, errors.Errorf("host port range exceeds maximum allowable port number") + if uint32(portRange-1)+uint32(hostPort) > 65535 { + return nil, errors.Errorf("host port range exceeds maximum allowable port number") } - // Iterate through ports, populating maps to check for conflicts - // and generating CNI port mappings. - for _, p := range protocols { - hostIPMap := hostPortValidate[p] - ctrIPMap := containerPortValidate[p] - - hostPortMap, ok := hostIPMap[hostIP] - if !ok { - hostPortMap = make(map[uint16]uint16) - hostIPMap[hostIP] = hostPortMap + hostProtoMap, ok := portMap[port.HostIP] + if !ok { + hostProtoMap = make(map[string][]tempMapping) + for _, proto := range []string{protoTCP, protoUDP, protoSCTP} { + hostProtoMap[proto] = make([]tempMapping, 0) } - ctrPortMap, ok := ctrIPMap[hostIP] - if !ok { - ctrPortMap = make(map[uint16]uint16) - ctrIPMap[hostIP] = ctrPortMap - } - - // Iterate through all port numbers in the requested - // range. - var index uint16 - for index = 0; index < len; index++ { - cPort := containerPort + index - hPort := hostPort - // Only increment host port if it's not 0. - if hostPort != 0 { - hPort += index - } - - if cPort == 0 { - return nil, nil, nil, errors.Errorf("container port cannot be 0") - } + portMap[port.HostIP] = hostProtoMap + } - // Host port is allowed to be 0. If it is, we - // select a random port on the host. - // This will happen *after* all other ports are - // placed, to ensure we don't accidentally - // select a port that a later mapping wanted. - if hPort == 0 { - // If we already have a host port - // assigned to their container port - - // just use that. - if ctrPortMap[cPort] != 0 { - hPort = ctrPortMap[cPort] - } else { - postAssignHostPort = true - } - } else { - testHPort := hostPortMap[hPort] - if testHPort != 0 && testHPort != cPort { - return nil, nil, nil, errors.Errorf("conflicting port mappings for host port %d (protocol %s)", hPort, p) - } - hostPortMap[hPort] = cPort - - // Mapping a container port to multiple - // host ports is allowed. - // We only store the latest of these in - // the container port map - we don't - // need to know all of them, just one. - testCPort := ctrPortMap[cPort] - ctrPortMap[cPort] = hPort - - // If we have an exact duplicate, just continue - if testCPort == hPort && testHPort == cPort { - continue - } - } + p := tempMapping{ + hostPort: port.HostPort, + containerPort: port.ContainerPort, + rangePort: portRange, + } - // We appear to be clear. Make an OCICNI port - // struct. - // Don't use hostIP - we want to preserve the - // empty string hostIP by default for compat. - cniPort := types.OCICNIPortMapping{ - HostPort: int32(hPort), - ContainerPort: int32(cPort), - Protocol: p, - HostIP: port.HostIP, - } - tempMappings = append( - tempMappings, - tempMapping{ - mapping: cniPort, - startOfRange: port.Range > 1 && index == 0, - isInRange: port.Range > 1, - }, - ) - } + for _, proto := range protocols { + hostProtoMap[proto] = append(hostProtoMap[proto], p) } } - // Handle any 0 host ports now by setting random container ports. - if postAssignHostPort { - remadeMappings := make([]types.OCICNIPortMapping, 0, len(tempMappings)) - - var ( - candidate int - err error - ) - - // Iterate over all - for _, tmp := range tempMappings { - p := tmp.mapping + // we do no longer need the original port mappings + // set it to 0 length so we can resuse it to populate + // the slice again while keeping the underlying capacity + portMappings = portMappings[:0] - if p.HostPort != 0 { - remadeMappings = append(remadeMappings, p) + for hostIP, protoMap := range portMap { + for protocol, ports := range protoMap { + ports := ports + if len(ports) == 0 { continue } - - hostIPMap := hostPortValidate[p.Protocol] - ctrIPMap := containerPortValidate[p.Protocol] - - hostPortMap, ok := hostIPMap[p.HostIP] - if !ok { - hostPortMap = make(map[uint16]uint16) - hostIPMap[p.HostIP] = hostPortMap + // 1. sort the ports by host port + // use a small hack to make sure ports with host port 0 are sorted last + sort.Slice(ports, func(i, j int) bool { + if ports[i].hostPort == ports[j].hostPort { + return ports[i].containerPort < ports[j].containerPort + } + if ports[i].hostPort == 0 { + return false + } + if ports[j].hostPort == 0 { + return true + } + return ports[i].hostPort < ports[j].hostPort + }) + + allUsedContainerPorts := allUsedContainerPortsMap[protocol] + allUsedHostPorts := allUsedHostPortsMap[protocol] + var usedHostPorts [65536]bool + + var previousPort *types.PortMapping + var i int + for i = 0; i < len(ports); i++ { + if ports[i].hostPort == 0 { + // because the ports are sorted and host port 0 is last + // we can break when we hit 0 + // we will fit them in afterwards + break + } + p := types.PortMapping{ + HostIP: hostIP, + Protocol: protocol, + HostPort: ports[i].hostPort, + ContainerPort: ports[i].containerPort, + Range: ports[i].rangePort, + } + var err error + previousPort, err = joinTwoPortsToRangePortIfPossible(&portMappings, &allUsedHostPorts, + &allUsedContainerPorts, &usedHostPorts, previousPort, p) + if err != nil { + return nil, err + } } - ctrPortMap, ok := ctrIPMap[p.HostIP] - if !ok { - ctrPortMap = make(map[uint16]uint16) - ctrIPMap[p.HostIP] = ctrPortMap + if previousPort != nil { + addPortToUsedPorts(&portMappings, &allUsedHostPorts, + &allUsedContainerPorts, &usedHostPorts, previousPort) } - // See if container port has been used elsewhere - if ctrPortMap[uint16(p.ContainerPort)] != 0 { - // Duplicate definition. Let's not bother - // including it. - continue + // now take care of the hostPort = 0 ports + previousPort = nil + for i < len(ports) { + p := types.PortMapping{ + HostIP: hostIP, + Protocol: protocol, + ContainerPort: ports[i].containerPort, + Range: ports[i].rangePort, + } + var err error + previousPort, err = joinTwoContainerPortsToRangePortIfPossible(&portMappings, &allUsedHostPorts, + &allUsedContainerPorts, &usedHostPorts, previousPort, p) + if err != nil { + return nil, err + } + i++ + } + if previousPort != nil { + newPort, err := getRandomHostPort(&usedHostPorts, *previousPort) + if err != nil { + return nil, err + } + addPortToUsedPorts(&portMappings, &allUsedHostPorts, + &allUsedContainerPorts, &usedHostPorts, &newPort) } - // Max retries to ensure we don't loop forever. - for i := 0; i < 15; i++ { - // Only get a random candidate for single entries or the start - // of a range. Otherwise we just increment the candidate. - if !tmp.isInRange || tmp.startOfRange { - candidate, err = utils.GetRandomPort() + allUsedContainerPortsMap[protocol] = allUsedContainerPorts + allUsedHostPortsMap[protocol] = allUsedHostPorts + } + } + + if len(exposePorts) > 0 { + logrus.Debugf("Adding exposed ports") + + for port, protocols := range exposePorts { + newProtocols := make([]string, 0, len(protocols)) + for _, protocol := range protocols { + if !allUsedContainerPortsMap[protocol][port] { + p := types.PortMapping{ + ContainerPort: port, + Protocol: protocol, + Range: 1, + } + allPorts := allUsedContainerPortsMap[protocol] + p, err := getRandomHostPort(&allPorts, p) if err != nil { - return nil, nil, nil, errors.Wrapf(err, "error getting candidate host port for container port %d", p.ContainerPort) + return nil, err } + portMappings = append(portMappings, p) } else { - candidate++ - } - - if hostPortMap[uint16(candidate)] == 0 { - logrus.Debugf("Successfully assigned container port %d to host port %d (IP %s Protocol %s)", p.ContainerPort, candidate, p.HostIP, p.Protocol) - hostPortMap[uint16(candidate)] = uint16(p.ContainerPort) - ctrPortMap[uint16(p.ContainerPort)] = uint16(candidate) - p.HostPort = int32(candidate) - break + newProtocols = append(newProtocols, protocol) } } - if p.HostPort == 0 { - return nil, nil, nil, errors.Errorf("could not find open host port to map container port %d to", p.ContainerPort) + // make sure to delete the key from the map if there are no protocols left + if len(newProtocols) == 0 { + delete(exposePorts, port) + } else { + exposePorts[port] = newProtocols } - remadeMappings = append(remadeMappings, p) } - return remadeMappings, containerPortValidate, hostPortValidate, nil } + return portMappings, nil +} - finalMappings := []types.OCICNIPortMapping{} - for _, m := range tempMappings { - finalMappings = append(finalMappings, m.mapping) +func appendProtocolsNoDuplicates(slice []string, protocols []string) []string { + for _, proto := range protocols { + if util.StringInSlice(proto, slice) { + continue + } + slice = append(slice, proto) } - - return finalMappings, containerPortValidate, hostPortValidate, nil + return slice } // Make final port mappings for the container -func createPortMappings(ctx context.Context, s *specgen.SpecGenerator, imageData *libimage.ImageData) ([]types.OCICNIPortMapping, map[uint16][]string, error) { - finalMappings, containerPortValidate, hostPortValidate, err := ParsePortMapping(s.PortMappings) - if err != nil { - return nil, nil, err - } - - // No exposed ports so return the port mappings we've made so far. - if len(s.Expose) == 0 && imageData == nil { - return finalMappings, nil, nil - } - - logrus.Debugf("Adding exposed ports") - +func createPortMappings(ctx context.Context, s *specgen.SpecGenerator, imageData *libimage.ImageData) ([]types.PortMapping, map[uint16][]string, error) { expose := make(map[uint16]string) + var err error if imageData != nil { expose, err = GenExposedPorts(imageData.Config.ExposedPorts) if err != nil { @@ -274,103 +348,30 @@ func createPortMappings(ctx context.Context, s *specgen.SpecGenerator, imageData } } - // We need to merge s.Expose into image exposed ports - for k, v := range s.Expose { - expose[k] = v - } - // There's been a request to expose some ports. Let's do that. - // Start by figuring out what needs to be exposed. - // This is a map of container port number to protocols to expose. - toExpose := make(map[uint16][]string) - for port, proto := range expose { - // Validate protocol first - protocols, err := checkProtocol(proto, false) - if err != nil { - return nil, nil, errors.Wrapf(err, "error validating protocols for exposed port %d", port) - } - - if port == 0 { - return nil, nil, errors.Errorf("cannot expose 0 as it is not a valid port number") - } - - // Check to see if the port is already present in existing - // mappings. - for _, p := range protocols { - ctrPortMap, ok := containerPortValidate[p]["0.0.0.0"] - if !ok { - ctrPortMap = make(map[uint16]uint16) - containerPortValidate[p]["0.0.0.0"] = ctrPortMap + toExpose := make(map[uint16][]string, len(s.Expose)+len(expose)) + for _, expose := range []map[uint16]string{expose, s.Expose} { + for port, proto := range expose { + if port == 0 { + return nil, nil, errors.Errorf("cannot expose 0 as it is not a valid port number") } - - if portNum := ctrPortMap[port]; portNum == 0 { - // We want to expose this port for this protocol - exposeProto, ok := toExpose[port] - if !ok { - exposeProto = []string{} - } - exposeProto = append(exposeProto, p) - toExpose[port] = exposeProto + protocols, err := checkProtocol(proto, false) + if err != nil { + return nil, nil, errors.Wrapf(err, "error validating protocols for exposed port %d", port) } + toExpose[port] = appendProtocolsNoDuplicates(toExpose[port], protocols) } } - // If not publishing exposed ports return mappings and exposed ports. + publishPorts := toExpose if !s.PublishExposedPorts { - return finalMappings, toExpose, nil + publishPorts = nil } - // We now have a final list of ports that we want exposed. - // Let's find empty, unallocated host ports for them. - for port, protocols := range toExpose { - for _, p := range protocols { - // Find an open port on the host. - // I see a faint possibility that this will infinite - // loop trying to find a valid open port, so I've - // included a max-tries counter. - hostPort := 0 - tries := 15 - for hostPort == 0 && tries > 0 { - // We can't select a specific protocol, which is - // unfortunate for the UDP case. - candidate, err := utils.GetRandomPort() - if err != nil { - return nil, nil, err - } - - // Check if the host port is already bound - hostPortMap, ok := hostPortValidate[p]["0.0.0.0"] - if !ok { - hostPortMap = make(map[uint16]uint16) - hostPortValidate[p]["0.0.0.0"] = hostPortMap - } - - if checkPort := hostPortMap[uint16(candidate)]; checkPort != 0 { - // Host port is already allocated, try again - tries-- - continue - } - - hostPortMap[uint16(candidate)] = port - hostPort = candidate - logrus.Debugf("Mapping exposed port %d/%s to host port %d", port, p, hostPort) - - // Make a CNI port mapping - cniPort := types.OCICNIPortMapping{ - HostPort: int32(candidate), - ContainerPort: int32(port), - Protocol: p, - HostIP: "", - } - finalMappings = append(finalMappings, cniPort) - } - if tries == 0 && hostPort == 0 { - // We failed to find an open port. - return nil, nil, errors.Errorf("failed to find an open port to expose container port %d on the host", port) - } - } + finalMappings, err := ParsePortMapping(s.PortMappings, publishPorts) + if err != nil { + return nil, nil, err } - - return finalMappings, nil, nil + return finalMappings, toExpose, nil } // Check a string to ensure it is a comma-separated set of valid protocols @@ -409,7 +410,7 @@ func checkProtocol(protocol string, allowSCTP bool) ([]string, error) { } func GenExposedPorts(exposedPorts map[string]struct{}) (map[uint16]string, error) { - expose := make(map[uint16]string) + expose := make(map[uint16]string, len(exposedPorts)) for imgExpose := range exposedPorts { // Expose format is portNumber[/protocol] splitExpose := strings.SplitN(imgExpose, "/", 2) @@ -420,12 +421,20 @@ func GenExposedPorts(exposedPorts map[string]struct{}) (map[uint16]string, error if num > 65535 || num < 1 { return nil, errors.Errorf("%d from image EXPOSE statement %q is not a valid port number", num, imgExpose) } - // No need to validate protocol, we'll do it below. - if len(splitExpose) == 1 { - expose[uint16(num)] = "tcp" + + // No need to validate protocol, we'll do it later. + newProto := "tcp" + if len(splitExpose) == 2 { + newProto = splitExpose[1] + } + + proto := expose[uint16(num)] + if len(proto) > 1 { + proto = proto + "," + newProto } else { - expose[uint16(num)] = splitExpose[1] + proto = newProto } + expose[uint16(num)] = proto } return expose, nil } diff --git a/pkg/specgen/generate/ports_bench_test.go b/pkg/specgen/generate/ports_bench_test.go new file mode 100644 index 000000000..06f02acda --- /dev/null +++ b/pkg/specgen/generate/ports_bench_test.go @@ -0,0 +1,197 @@ +package generate + +import ( + "fmt" + "testing" + + "github.com/containers/podman/v3/libpod/network/types" +) + +func benchmarkParsePortMapping(b *testing.B, ports []types.PortMapping) { + for n := 0; n < b.N; n++ { + ParsePortMapping(ports, nil) + } +} + +func BenchmarkParsePortMappingNoPorts(b *testing.B) { + benchmarkParsePortMapping(b, nil) +} + +func BenchmarkParsePortMapping1(b *testing.B) { + benchmarkParsePortMapping(b, []types.PortMapping{ + { + HostPort: 8080, + ContainerPort: 80, + Protocol: "tcp", + }, + }) +} + +func BenchmarkParsePortMapping100(b *testing.B) { + ports := make([]types.PortMapping, 0, 100) + for i := uint16(8080); i < 8180; i++ { + ports = append(ports, types.PortMapping{ + HostPort: i, + ContainerPort: i, + Protocol: "tcp", + }) + } + b.ResetTimer() + benchmarkParsePortMapping(b, ports) +} + +func BenchmarkParsePortMapping1k(b *testing.B) { + ports := make([]types.PortMapping, 0, 1000) + for i := uint16(8080); i < 9080; i++ { + ports = append(ports, types.PortMapping{ + HostPort: i, + ContainerPort: i, + Protocol: "tcp", + }) + } + b.ResetTimer() + benchmarkParsePortMapping(b, ports) +} + +func BenchmarkParsePortMapping10k(b *testing.B) { + ports := make([]types.PortMapping, 0, 30000) + for i := uint16(8080); i < 18080; i++ { + ports = append(ports, types.PortMapping{ + HostPort: i, + ContainerPort: i, + Protocol: "tcp", + }) + } + b.ResetTimer() + benchmarkParsePortMapping(b, ports) +} + +func BenchmarkParsePortMapping1m(b *testing.B) { + ports := make([]types.PortMapping, 0, 1000000) + for j := 0; j < 20; j++ { + for i := uint16(1); i <= 50000; i++ { + ports = append(ports, types.PortMapping{ + HostPort: i, + ContainerPort: i, + Protocol: "tcp", + HostIP: fmt.Sprintf("192.168.1.%d", j), + }) + } + } + b.ResetTimer() + benchmarkParsePortMapping(b, ports) +} + +func BenchmarkParsePortMappingReverse100(b *testing.B) { + ports := make([]types.PortMapping, 0, 100) + for i := uint16(8180); i > 8080; i-- { + ports = append(ports, types.PortMapping{ + HostPort: i, + ContainerPort: i, + Protocol: "tcp", + }) + } + b.ResetTimer() + benchmarkParsePortMapping(b, ports) +} + +func BenchmarkParsePortMappingReverse1k(b *testing.B) { + ports := make([]types.PortMapping, 0, 1000) + for i := uint16(9080); i > 8080; i-- { + ports = append(ports, types.PortMapping{ + HostPort: i, + ContainerPort: i, + Protocol: "tcp", + }) + } + b.ResetTimer() + benchmarkParsePortMapping(b, ports) +} + +func BenchmarkParsePortMappingReverse10k(b *testing.B) { + ports := make([]types.PortMapping, 0, 30000) + for i := uint16(18080); i > 8080; i-- { + ports = append(ports, types.PortMapping{ + HostPort: i, + ContainerPort: i, + Protocol: "tcp", + }) + } + b.ResetTimer() + benchmarkParsePortMapping(b, ports) +} + +func BenchmarkParsePortMappingReverse1m(b *testing.B) { + ports := make([]types.PortMapping, 0, 1000000) + for j := 0; j < 20; j++ { + for i := uint16(50000); i > 0; i-- { + ports = append(ports, types.PortMapping{ + HostPort: i, + ContainerPort: i, + Protocol: "tcp", + HostIP: fmt.Sprintf("192.168.1.%d", j), + }) + } + } + b.ResetTimer() + benchmarkParsePortMapping(b, ports) +} + +func BenchmarkParsePortMappingRange1(b *testing.B) { + benchmarkParsePortMapping(b, []types.PortMapping{ + { + HostPort: 8080, + ContainerPort: 80, + Protocol: "tcp", + Range: 1, + }, + }) +} + +func BenchmarkParsePortMappingRange100(b *testing.B) { + benchmarkParsePortMapping(b, []types.PortMapping{ + { + HostPort: 8080, + ContainerPort: 80, + Protocol: "tcp", + Range: 100, + }, + }) +} + +func BenchmarkParsePortMappingRange1k(b *testing.B) { + benchmarkParsePortMapping(b, []types.PortMapping{ + { + HostPort: 8080, + ContainerPort: 80, + Protocol: "tcp", + Range: 1000, + }, + }) +} + +func BenchmarkParsePortMappingRange10k(b *testing.B) { + benchmarkParsePortMapping(b, []types.PortMapping{ + { + HostPort: 8080, + ContainerPort: 80, + Protocol: "tcp", + Range: 10000, + }, + }) +} + +func BenchmarkParsePortMappingRange1m(b *testing.B) { + ports := make([]types.PortMapping, 0, 1000000) + for j := 0; j < 20; j++ { + ports = append(ports, types.PortMapping{ + HostPort: 1, + ContainerPort: 1, + Protocol: "tcp", + Range: 50000, + HostIP: fmt.Sprintf("192.168.1.%d", j), + }) + } + b.ResetTimer() + benchmarkParsePortMapping(b, ports) +} diff --git a/pkg/specgen/generate/ports_test.go b/pkg/specgen/generate/ports_test.go new file mode 100644 index 000000000..20d5d0166 --- /dev/null +++ b/pkg/specgen/generate/ports_test.go @@ -0,0 +1,989 @@ +package generate + +import ( + "testing" + + "github.com/containers/podman/v3/libpod/network/types" + "github.com/stretchr/testify/assert" +) + +func TestParsePortMappingWithHostPort(t *testing.T) { + tests := []struct { + name string + arg []types.PortMapping + arg2 map[uint16][]string + want []types.PortMapping + }{ + { + name: "no ports", + arg: nil, + want: nil, + }, + { + name: "one tcp port", + arg: []types.PortMapping{ + { + HostPort: 8080, + ContainerPort: 80, + Protocol: "tcp", + }, + }, + want: []types.PortMapping{ + { + HostPort: 8080, + ContainerPort: 80, + Protocol: "tcp", + Range: 1, + }, + }, + }, + { + name: "one tcp port no proto", + arg: []types.PortMapping{ + { + HostPort: 8080, + ContainerPort: 80, + }, + }, + want: []types.PortMapping{ + { + HostPort: 8080, + ContainerPort: 80, + Protocol: "tcp", + Range: 1, + }, + }, + }, + { + name: "one udp port", + arg: []types.PortMapping{ + { + HostPort: 8080, + ContainerPort: 80, + Protocol: "udp", + }, + }, + want: []types.PortMapping{ + { + HostPort: 8080, + ContainerPort: 80, + Protocol: "udp", + Range: 1, + }, + }, + }, + { + name: "one sctp port", + arg: []types.PortMapping{ + { + HostPort: 8080, + ContainerPort: 80, + Protocol: "sctp", + }, + }, + want: []types.PortMapping{ + { + HostPort: 8080, + ContainerPort: 80, + Protocol: "sctp", + Range: 1, + }, + }, + }, + { + name: "one port two protocols", + arg: []types.PortMapping{ + { + HostPort: 8080, + ContainerPort: 80, + Protocol: "tcp,udp", + }, + }, + want: []types.PortMapping{ + { + HostPort: 8080, + ContainerPort: 80, + Protocol: "tcp", + Range: 1, + }, + { + HostPort: 8080, + ContainerPort: 80, + Protocol: "udp", + Range: 1, + }, + }, + }, + { + name: "one port three protocols", + arg: []types.PortMapping{ + { + HostPort: 8080, + ContainerPort: 80, + Protocol: "tcp,udp,sctp", + }, + }, + want: []types.PortMapping{ + { + HostPort: 8080, + ContainerPort: 80, + Protocol: "tcp", + Range: 1, + }, + { + HostPort: 8080, + ContainerPort: 80, + Protocol: "udp", + Range: 1, + }, + { + HostPort: 8080, + ContainerPort: 80, + Protocol: "sctp", + Range: 1, + }, + }, + }, + { + name: "one port with range 1", + arg: []types.PortMapping{ + { + HostPort: 8080, + ContainerPort: 80, + Protocol: "tcp", + Range: 1, + }, + }, + want: []types.PortMapping{ + { + HostPort: 8080, + ContainerPort: 80, + Protocol: "tcp", + Range: 1, + }, + }, + }, + { + name: "one port with range 5", + arg: []types.PortMapping{ + { + HostPort: 8080, + ContainerPort: 80, + Protocol: "tcp", + Range: 5, + }, + }, + want: []types.PortMapping{ + { + HostPort: 8080, + ContainerPort: 80, + Protocol: "tcp", + Range: 5, + }, + }, + }, + { + name: "two ports joined", + arg: []types.PortMapping{ + { + HostPort: 8080, + ContainerPort: 80, + Protocol: "tcp", + }, + { + HostPort: 8081, + ContainerPort: 81, + Protocol: "tcp", + }, + }, + want: []types.PortMapping{ + { + HostPort: 8080, + ContainerPort: 80, + Protocol: "tcp", + Range: 2, + }, + }, + }, + { + name: "two ports joined with range", + arg: []types.PortMapping{ + { + HostPort: 8080, + ContainerPort: 80, + Protocol: "tcp", + Range: 2, + }, + { + HostPort: 8081, + ContainerPort: 81, + Protocol: "tcp", + }, + }, + want: []types.PortMapping{ + { + HostPort: 8080, + ContainerPort: 80, + Protocol: "tcp", + Range: 2, + }, + }, + }, + { + name: "two ports with no overlapping range", + arg: []types.PortMapping{ + { + HostPort: 8080, + ContainerPort: 80, + Protocol: "tcp", + Range: 10, + }, + { + HostPort: 9090, + ContainerPort: 9090, + Protocol: "tcp", + }, + }, + want: []types.PortMapping{ + { + HostPort: 9090, + ContainerPort: 9090, + Protocol: "tcp", + Range: 1, + }, + { + HostPort: 8080, + ContainerPort: 80, + Protocol: "tcp", + Range: 10, + }, + }, + }, + { + name: "four ports with two overlapping ranges", + arg: []types.PortMapping{ + { + HostPort: 8080, + ContainerPort: 80, + Protocol: "tcp", + Range: 10, + }, + { + HostPort: 8085, + ContainerPort: 85, + Protocol: "tcp", + Range: 10, + }, + { + HostPort: 100, + ContainerPort: 5, + Protocol: "tcp", + }, + { + HostPort: 101, + ContainerPort: 6, + Protocol: "tcp", + }, + }, + want: []types.PortMapping{ + { + HostPort: 8080, + ContainerPort: 80, + Protocol: "tcp", + Range: 15, + }, + { + HostPort: 100, + ContainerPort: 5, + Protocol: "tcp", + Range: 2, + }, + }, + }, + { + name: "two overlapping ranges", + arg: []types.PortMapping{ + { + HostPort: 8080, + ContainerPort: 80, + Protocol: "tcp", + Range: 10, + }, + { + HostPort: 8085, + ContainerPort: 85, + Protocol: "tcp", + Range: 2, + }, + }, + want: []types.PortMapping{ + { + HostPort: 8080, + ContainerPort: 80, + Protocol: "tcp", + Range: 10, + }, + }, + }, + { + name: "four overlapping ranges", + arg: []types.PortMapping{ + { + HostPort: 8080, + ContainerPort: 80, + Protocol: "tcp", + Range: 10, + }, + { + HostPort: 8085, + ContainerPort: 85, + Protocol: "tcp", + Range: 2, + }, + { + HostPort: 8090, + ContainerPort: 90, + Protocol: "tcp", + Range: 7, + }, + { + HostPort: 8095, + ContainerPort: 95, + Protocol: "tcp", + }, + }, + want: []types.PortMapping{ + { + HostPort: 8080, + ContainerPort: 80, + Protocol: "tcp", + Range: 17, + }, + }, + }, + { + name: "one port range overlaps 5 ports", + arg: []types.PortMapping{ + { + HostPort: 8080, + ContainerPort: 80, + Range: 20, + }, + { + HostPort: 8085, + ContainerPort: 85, + Range: 2, + }, + { + HostPort: 8090, + ContainerPort: 90, + }, + { + HostPort: 8095, + ContainerPort: 95, + }, + { + HostPort: 8096, + ContainerPort: 96, + }, + }, + want: []types.PortMapping{ + { + HostPort: 8080, + ContainerPort: 80, + Protocol: "tcp", + Range: 20, + }, + }, + }, + { + name: "different host ip same port", + arg: []types.PortMapping{ + { + HostPort: 8080, + ContainerPort: 80, + Protocol: "tcp", + HostIP: "192.168.1.1", + }, + { + HostPort: 8080, + ContainerPort: 80, + Protocol: "tcp", + HostIP: "192.168.2.1", + }, + }, + want: []types.PortMapping{ + { + HostPort: 8080, + ContainerPort: 80, + Protocol: "tcp", + HostIP: "192.168.1.1", + Range: 1, + }, + { + HostPort: 8080, + ContainerPort: 80, + Protocol: "tcp", + HostIP: "192.168.2.1", + Range: 1, + }, + }, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + got, err := ParsePortMapping(tt.arg, tt.arg2) + assert.NoError(t, err, "error is not nil") + // use ElementsMatch instead of Equal because the order is not consistent + assert.ElementsMatch(t, tt.want, got, "got unexpected port mapping") + }) + } +} + +func TestParsePortMappingWithoutHostPort(t *testing.T) { + tests := []struct { + name string + arg []types.PortMapping + arg2 map[uint16][]string + want []types.PortMapping + }{ + { + name: "one tcp port", + arg: []types.PortMapping{ + { + HostPort: 0, + ContainerPort: 80, + Protocol: "tcp", + }, + }, + want: []types.PortMapping{ + { + HostPort: 0, + ContainerPort: 80, + Protocol: "tcp", + Range: 1, + }, + }, + }, + { + name: "one port with two protocols", + arg: []types.PortMapping{ + { + HostPort: 0, + ContainerPort: 80, + Protocol: "tcp,udp", + }, + }, + want: []types.PortMapping{ + { + HostPort: 0, + ContainerPort: 80, + Protocol: "tcp", + Range: 1, + }, + { + HostPort: 0, + ContainerPort: 80, + Protocol: "udp", + Range: 1, + }, + }, + }, + { + name: "same port twice", + arg: []types.PortMapping{ + { + HostPort: 0, + ContainerPort: 80, + Protocol: "tcp", + }, + { + HostPort: 0, + ContainerPort: 80, + Protocol: "tcp", + }, + }, + want: []types.PortMapping{ + { + HostPort: 0, + ContainerPort: 80, + Protocol: "tcp", + Range: 1, + }, + }, + }, + { + name: "neighbor ports are not joined", + arg: []types.PortMapping{ + { + HostPort: 0, + ContainerPort: 80, + Protocol: "tcp", + }, + { + HostPort: 0, + ContainerPort: 81, + Protocol: "tcp", + }, + }, + want: []types.PortMapping{ + { + HostPort: 0, + ContainerPort: 80, + Protocol: "tcp", + Range: 1, + }, + { + HostPort: 0, + ContainerPort: 81, + Protocol: "tcp", + Range: 1, + }, + }, + }, + { + name: "overlapping range ports are joined", + arg: []types.PortMapping{ + { + HostPort: 0, + ContainerPort: 80, + Protocol: "tcp", + Range: 2, + }, + { + HostPort: 0, + ContainerPort: 81, + Protocol: "tcp", + }, + }, + want: []types.PortMapping{ + { + HostPort: 0, + ContainerPort: 80, + Protocol: "tcp", + Range: 2, + }, + }, + }, + { + name: "four overlapping range ports are joined", + arg: []types.PortMapping{ + { + HostPort: 0, + ContainerPort: 80, + Protocol: "tcp", + Range: 3, + }, + { + HostPort: 0, + ContainerPort: 81, + Protocol: "tcp", + }, + { + HostPort: 0, + ContainerPort: 82, + Protocol: "tcp", + Range: 10, + }, + { + HostPort: 0, + ContainerPort: 90, + Protocol: "tcp", + Range: 5, + }, + }, + want: []types.PortMapping{ + { + HostPort: 0, + ContainerPort: 80, + Protocol: "tcp", + Range: 15, + }, + }, + }, + { + name: "expose one tcp port", + arg2: map[uint16][]string{ + 8080: {"tcp"}, + }, + want: []types.PortMapping{ + { + HostPort: 0, + ContainerPort: 8080, + Protocol: "tcp", + Range: 1, + }, + }, + }, + { + name: "expose already defined port", + arg: []types.PortMapping{ + { + HostPort: 0, + ContainerPort: 8080, + Protocol: "tcp", + }, + }, + arg2: map[uint16][]string{ + 8080: {"tcp"}, + }, + want: []types.PortMapping{ + { + HostPort: 0, + ContainerPort: 8080, + Protocol: "tcp", + Range: 1, + }, + }, + }, + { + name: "expose different proto", + arg: []types.PortMapping{ + { + HostPort: 0, + ContainerPort: 8080, + Protocol: "tcp", + }, + }, + arg2: map[uint16][]string{ + 8080: {"udp"}, + }, + want: []types.PortMapping{ + { + HostPort: 0, + ContainerPort: 8080, + Protocol: "tcp", + Range: 1, + }, + { + HostPort: 0, + ContainerPort: 8080, + Protocol: "udp", + Range: 1, + }, + }, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + got, err := ParsePortMapping(tt.arg, tt.arg2) + assert.NoError(t, err, "error is not nil") + + // because we always get random host ports when it is set to 0 we cannot check that exactly + // check if it is not 0 and set to to 0 afterwards + for i := range got { + assert.Greater(t, got[i].HostPort, uint16(0), "host port is zero") + got[i].HostPort = 0 + } + + // use ElementsMatch instead of Equal because the order is not consistent + assert.ElementsMatch(t, tt.want, got, "got unexpected port mapping") + }) + } +} + +func TestParsePortMappingMixedHostPort(t *testing.T) { + tests := []struct { + name string + arg []types.PortMapping + want []types.PortMapping + resetHostPorts []int + }{ + { + name: "two ports one without a hostport set", + arg: []types.PortMapping{ + { + HostPort: 0, + ContainerPort: 80, + }, + { + HostPort: 8080, + ContainerPort: 8080, + }, + }, + want: []types.PortMapping{ + { + HostPort: 8080, + ContainerPort: 8080, + Protocol: "tcp", + Range: 1, + }, + { + HostPort: 0, + ContainerPort: 80, + Protocol: "tcp", + Range: 1, + }, + }, + resetHostPorts: []int{1}, + }, + { + name: "two ports one without a hostport set, inverted order", + arg: []types.PortMapping{ + { + HostPort: 8080, + ContainerPort: 8080, + }, + { + HostPort: 0, + ContainerPort: 80, + }, + }, + want: []types.PortMapping{ + { + HostPort: 8080, + ContainerPort: 8080, + Protocol: "tcp", + Range: 1, + }, + { + HostPort: 0, + ContainerPort: 80, + Protocol: "tcp", + Range: 1, + }, + }, + resetHostPorts: []int{1}, + }, + { + name: "three ports without host ports, one with a hostport set, , inverted order", + arg: []types.PortMapping{ + { + HostPort: 0, + ContainerPort: 80, + }, + { + HostPort: 0, + ContainerPort: 85, + }, + { + HostPort: 0, + ContainerPort: 90, + }, + { + HostPort: 8080, + ContainerPort: 8080, + }, + }, + want: []types.PortMapping{ + { + HostPort: 8080, + ContainerPort: 8080, + Protocol: "tcp", + Range: 1, + }, + { + HostPort: 0, + ContainerPort: 80, + Protocol: "tcp", + Range: 1, + }, + { + HostPort: 0, + ContainerPort: 85, + Protocol: "tcp", + Range: 1, + }, + { + HostPort: 0, + ContainerPort: 90, + Protocol: "tcp", + Range: 1, + }, + }, + resetHostPorts: []int{1, 2, 3}, + }, + { + name: "three ports without host ports, one with a hostport set", + arg: []types.PortMapping{ + { + HostPort: 8080, + ContainerPort: 8080, + }, + { + HostPort: 0, + ContainerPort: 90, + }, + { + HostPort: 0, + ContainerPort: 85, + }, + { + HostPort: 0, + ContainerPort: 80, + }, + }, + want: []types.PortMapping{ + { + HostPort: 8080, + ContainerPort: 8080, + Protocol: "tcp", + Range: 1, + }, + { + HostPort: 0, + ContainerPort: 80, + Protocol: "tcp", + Range: 1, + }, + { + HostPort: 0, + ContainerPort: 85, + Protocol: "tcp", + Range: 1, + }, + { + HostPort: 0, + ContainerPort: 90, + Protocol: "tcp", + Range: 1, + }, + }, + resetHostPorts: []int{1, 2, 3}, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + got, err := ParsePortMapping(tt.arg, nil) + assert.NoError(t, err, "error is not nil") + + // because we always get random host ports when it is set to 0 we cannot check that exactly + // use resetHostPorts to know which port element is 0 + for _, num := range tt.resetHostPorts { + assert.Greater(t, got[num].HostPort, uint16(0), "host port is zero") + got[num].HostPort = 0 + } + + assert.Equal(t, tt.want, got, "got unexpected port mapping") + }) + } +} + +func TestParsePortMappingError(t *testing.T) { + tests := []struct { + name string + arg []types.PortMapping + err string + }{ + { + name: "container port is 0", + arg: []types.PortMapping{ + { + HostPort: 8080, + ContainerPort: 0, + Protocol: "tcp", + }, + }, + err: "container port number must be non-0", + }, + { + name: "container port range exceeds max", + arg: []types.PortMapping{ + { + HostPort: 8080, + ContainerPort: 65000, + Protocol: "tcp", + Range: 10000, + }, + }, + err: "container port range exceeds maximum allowable port number", + }, + { + name: "host port range exceeds max", + arg: []types.PortMapping{ + { + HostPort: 60000, + ContainerPort: 1, + Protocol: "tcp", + Range: 10000, + }, + }, + err: "host port range exceeds maximum allowable port number", + }, + { + name: "invalid protocol", + arg: []types.PortMapping{ + { + HostPort: 8080, + ContainerPort: 80, + Protocol: "1", + }, + }, + err: "unrecognized protocol \"1\" in port mapping", + }, + { + name: "invalid protocol 2", + arg: []types.PortMapping{ + { + HostPort: 8080, + ContainerPort: 80, + Protocol: "udp,u", + }, + }, + err: "unrecognized protocol \"u\" in port mapping", + }, + { + name: "invalid ip address", + arg: []types.PortMapping{ + { + HostPort: 8080, + ContainerPort: 80, + HostIP: "blah", + }, + }, + err: "invalid IP address \"blah\" in port mapping", + }, + { + name: "invalid overalpping range", + arg: []types.PortMapping{ + { + HostPort: 8080, + ContainerPort: 80, + Range: 5, + }, + { + HostPort: 8081, + ContainerPort: 60, + }, + }, + err: "conflicting port mappings for host port 8081 (protocol tcp)", + }, + { + name: "big port range with host port zero does not fit", + arg: []types.PortMapping{ + { + HostPort: 0, + ContainerPort: 1, + Range: 65535, + }, + }, + err: "failed to find an open port to expose container port 1 with range 65535 on the host", + }, + { + name: "big port range with host port zero does not fit", + arg: []types.PortMapping{ + { + HostPort: 0, + ContainerPort: 80, + Range: 1, + }, + { + HostPort: 0, + ContainerPort: 1000, + Range: 64535, + }, + }, + err: "failed to find an open port to expose container port 1000 with range 64535 on the host", + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + _, err := ParsePortMapping(tt.arg, nil) + assert.EqualError(t, err, tt.err, "error does not match") + }) + } +} diff --git a/pkg/specgen/podspecgen.go b/pkg/specgen/podspecgen.go index 7713ea26c..948fb990c 100644 --- a/pkg/specgen/podspecgen.go +++ b/pkg/specgen/podspecgen.go @@ -99,7 +99,8 @@ type PodNetworkConfig struct { // Only available if NetNS is set to Bridge (the default for root). // As such, conflicts with NoInfra=true by proxy. // Optional. - StaticMAC *net.HardwareAddr `json:"static_mac,omitempty"` + // swagger:strfmt string + StaticMAC *types.HardwareAddr `json:"static_mac,omitempty"` // PortMappings is a set of ports to map into the infra container. // As, by default, containers share their network with the infra // container, this will forward the ports to the entire pod. diff --git a/pkg/specgen/specgen.go b/pkg/specgen/specgen.go index 5a07af0f9..8a4497130 100644 --- a/pkg/specgen/specgen.go +++ b/pkg/specgen/specgen.go @@ -401,7 +401,8 @@ type ContainerNetworkConfig struct { // StaticMAC is a static MAC address to set in the container. // Only available if NetNS is set to bridge. // Optional. - StaticMAC *net.HardwareAddr `json:"static_mac,omitempty"` + // swagger:strfmt string + StaticMAC *nettypes.HardwareAddr `json:"static_mac,omitempty"` // PortBindings is a set of ports to map into the container. // Only available if NetNS is set to bridge or slirp. // Optional. diff --git a/pkg/specgenutil/specgen.go b/pkg/specgenutil/specgen.go index 683cd2918..4e8f954fb 100644 --- a/pkg/specgenutil/specgen.go +++ b/pkg/specgenutil/specgen.go @@ -11,6 +11,7 @@ import ( "github.com/containers/image/v5/manifest" "github.com/containers/podman/v3/cmd/podman/parse" "github.com/containers/podman/v3/libpod/define" + "github.com/containers/podman/v3/libpod/network/types" ann "github.com/containers/podman/v3/pkg/annotations" "github.com/containers/podman/v3/pkg/domain/entities" envLib "github.com/containers/podman/v3/pkg/env" @@ -457,7 +458,8 @@ func FillOutSpecGen(s *specgen.SpecGenerator, c *entities.ContainerCreateOptions s.DNSSearch = c.Net.DNSSearch s.DNSOptions = c.Net.DNSOptions s.StaticIP = c.Net.StaticIP - s.StaticMAC = c.Net.StaticMAC + // type cast to types.HardwareAddr + s.StaticMAC = (*types.HardwareAddr)(c.Net.StaticMAC) s.NetworkOptions = c.Net.NetworkOptions s.UseImageHosts = c.Net.NoHosts } diff --git a/test/apiv2/10-images.at b/test/apiv2/10-images.at index d3fde9f9d..b7bcaf81d 100644 --- a/test/apiv2/10-images.at +++ b/test/apiv2/10-images.at @@ -218,6 +218,9 @@ if ! grep -q '400 Bad Request' "${TMPD}/headers.txt"; then BUILD_TEST_ERROR="1" fi +t POST libpod/images/prune 200 +t POST libpod/images/prune 200 length=0 [] + cleanBuildTest if [[ "${BUILD_TEST_ERROR}" ]]; then exit 1 diff --git a/test/apiv2/40-pods.at b/test/apiv2/40-pods.at index 985b26411..f45e85f61 100644 --- a/test/apiv2/40-pods.at +++ b/test/apiv2/40-pods.at @@ -110,11 +110,11 @@ t GET libpod/pods/fakename/top 404 \ .cause="no such pod" t GET libpod/pods/foo/top 200 \ - .Processes[0][-1]="/pause " \ + .Processes[0][-1]="/pause" \ .Titles[-1]="COMMAND" t GET libpod/pods/foo/top?ps_args=args,pid 200 \ - .Processes[0][0]="/pause " \ + .Processes[0][0]="/pause" \ .Processes[0][1]="1" \ .Titles[0]="COMMAND" \ .Titles[1]="PID" \ diff --git a/test/apiv2/python/rest_api/test_v2_0_0_container.py b/test/apiv2/python/rest_api/test_v2_0_0_container.py index 853e9da88..101044bbb 100644 --- a/test/apiv2/python/rest_api/test_v2_0_0_container.py +++ b/test/apiv2/python/rest_api/test_v2_0_0_container.py @@ -1,8 +1,11 @@ +import multiprocessing +import queue import random +import threading import unittest -import json import requests +import time from dateutil.parser import parse from .fixtures import APITestCase @@ -16,7 +19,10 @@ class ContainerTestCase(APITestCase): self.assertEqual(len(obj), 1) def test_list_filters(self): - r = requests.get(self.podman_url + "/v1.40/containers/json?filters%3D%7B%22status%22%3A%5B%22running%22%5D%7D") + r = requests.get( + self.podman_url + + "/v1.40/containers/json?filters%3D%7B%22status%22%3A%5B%22running%22%5D%7D" + ) self.assertEqual(r.status_code, 200, r.text) payload = r.json() containerAmnt = len(payload) @@ -33,18 +39,18 @@ class ContainerTestCase(APITestCase): self.assertId(r.content) _ = parse(r.json()["Created"]) - r = requests.post( self.podman_url + "/v1.40/containers/create?name=topcontainer", - json={"Cmd": ["top"], - "Image": "alpine:latest", - "Healthcheck": { - "Test": ["CMD", "pidof", "top"], - "Interval": 5000000000, - "Timeout": 2000000000, - "Retries": 3, - "StartPeriod": 5000000000 - } + json={ + "Cmd": ["top"], + "Image": "alpine:latest", + "Healthcheck": { + "Test": ["CMD", "pidof", "top"], + "Interval": 5000000000, + "Timeout": 2000000000, + "Retries": 3, + "StartPeriod": 5000000000, + }, }, ) self.assertEqual(r.status_code, 201, r.text) @@ -67,7 +73,7 @@ class ContainerTestCase(APITestCase): self.assertEqual(r.status_code, 200, r.text) self.assertId(r.content) out = r.json() - hc = out["Config"]["Healthcheck"]["Test"] + hc = out["Config"]["Healthcheck"]["Test"] self.assertListEqual(["CMD", "pidof", "top"], hc) r = requests.post(self.podman_url + f"/v1.40/containers/{container_id}/start") @@ -84,7 +90,9 @@ class ContainerTestCase(APITestCase): self.assertIn(r.status_code, (200, 409), r.text) if r.status_code == 200: self.assertId(r.content) - r = requests.get(self.uri(self.resolve_container("/containers/{}/stats?stream=false&one-shot=true"))) + r = requests.get( + self.uri(self.resolve_container("/containers/{}/stats?stream=false&one-shot=true")) + ) self.assertIn(r.status_code, (200, 409), r.text) if r.status_code == 200: self.assertId(r.content) @@ -136,9 +144,15 @@ class ContainerTestCase(APITestCase): payload = r.json() container_id = payload["Id"] self.assertIsNotNone(container_id) - r = requests.get(self.podman_url + f"/v1.40/containers/{payload['Id']}/logs?follow=false&stdout=true&until=0") + r = requests.get( + self.podman_url + + f"/v1.40/containers/{payload['Id']}/logs?follow=false&stdout=true&until=0" + ) self.assertEqual(r.status_code, 200, r.text) - r = requests.get(self.podman_url + f"/v1.40/containers/{payload['Id']}/logs?follow=false&stdout=true&until=1") + r = requests.get( + self.podman_url + + f"/v1.40/containers/{payload['Id']}/logs?follow=false&stdout=true&until=1" + ) self.assertEqual(r.status_code, 200, r.text) def test_commit(self): @@ -257,6 +271,63 @@ class ContainerTestCase(APITestCase): r = requests.delete(self.podman_url + f"/v1.40/containers/{container_id}") self.assertEqual(r.status_code, 204, r.text) + def test_top_no_stream(self): + uri = self.uri(self.resolve_container("/containers/{}/top")) + q = queue.Queue() + + def _impl(fifo): + fifo.put(requests.get(uri, params={"stream": False}, timeout=2)) + + top = threading.Thread(target=_impl, args=(q,)) + top.start() + time.sleep(2) + self.assertFalse(top.is_alive(), f"GET {uri} failed to return in 2s") + + qr = q.get(False) + self.assertEqual(qr.status_code, 200, qr.text) + + qr.close() + top.join() + + def test_top_stream(self): + uri = self.uri(self.resolve_container("/containers/{}/top")) + q = queue.Queue() + + stop_thread = False + + def _impl(fifo, stop): + try: + with requests.get(uri, params={"stream": True, "delay": 1}, stream=True) as r: + r.raise_for_status() + fifo.put(r) + for buf in r.iter_lines(chunk_size=None): + if stop(): + break + fifo.put(buf) + except Exception: + pass + + top = threading.Thread(target=_impl, args=(q, (lambda: stop_thread))) + top.start() + time.sleep(4) + self.assertTrue(top.is_alive(), f"GET {uri} exited too soon") + stop_thread = True + + for _ in range(10): + try: + qr = q.get_nowait() + if qr is not None: + self.assertEqual(qr.status_code, 200) + qr.close() + break + except queue.Empty: + pass + finally: + time.sleep(1) + else: + self.fail("Server failed to respond in 10s") + top.join() + if __name__ == "__main__": unittest.main() diff --git a/test/e2e/checkpoint_test.go b/test/e2e/checkpoint_test.go index a8efe1ca9..be6b782b5 100644 --- a/test/e2e/checkpoint_test.go +++ b/test/e2e/checkpoint_test.go @@ -5,9 +5,11 @@ import ( "net" "os" "os/exec" + "path/filepath" "strings" "time" + "github.com/checkpoint-restore/go-criu/v5/stats" "github.com/containers/podman/v3/pkg/checkpoint/crutils" "github.com/containers/podman/v3/pkg/criu" . "github.com/containers/podman/v3/test/utils" @@ -1191,4 +1193,55 @@ var _ = Describe("Podman checkpoint", func() { // Remove exported checkpoint os.Remove(fileName) }) + + It("podman checkpoint container with export and statistics", func() { + localRunString := getRunString([]string{ + "--rm", + ALPINE, + "top", + }) + session := podmanTest.Podman(localRunString) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1)) + cid := session.OutputToString() + fileName := "/tmp/checkpoint-" + cid + ".tar.gz" + + result := podmanTest.Podman([]string{ + "container", + "checkpoint", + "-l", "-e", + fileName, + }) + result.WaitWithDefaultTimeout() + + // As the container has been started with '--rm' it will be completely + // cleaned up after checkpointing. + Expect(result).Should(Exit(0)) + Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0)) + Expect(podmanTest.NumberOfContainers()).To(Equal(0)) + + // Extract checkpoint archive + destinationDirectory, err := CreateTempDirInTempDir() + Expect(err).ShouldNot(HaveOccurred()) + + tarsession := SystemExec( + "tar", + []string{ + "xf", + fileName, + "-C", + destinationDirectory, + }, + ) + Expect(tarsession).Should(Exit(0)) + + _, err = os.Stat(filepath.Join(destinationDirectory, stats.StatsDump)) + Expect(err).ShouldNot(HaveOccurred()) + + Expect(os.RemoveAll(destinationDirectory)).To(BeNil()) + + // Remove exported checkpoint + os.Remove(fileName) + }) }) diff --git a/test/e2e/common_test.go b/test/e2e/common_test.go index 7228682f3..e598f7ab9 100644 --- a/test/e2e/common_test.go +++ b/test/e2e/common_test.go @@ -685,7 +685,7 @@ func SkipIfContainerized(reason string) { // PodmanAsUser is the exec call to podman on the filesystem with the specified uid/gid and environment func (p *PodmanTestIntegration) PodmanAsUser(args []string, uid, gid uint32, cwd string, env []string) *PodmanSessionIntegration { - podmanSession := p.PodmanAsUserBase(args, uid, gid, cwd, env, false, false, nil) + podmanSession := p.PodmanAsUserBase(args, uid, gid, cwd, env, false, false, nil, nil) return &PodmanSessionIntegration{podmanSession} } diff --git a/test/e2e/libpod_suite_remote_test.go b/test/e2e/libpod_suite_remote_test.go index 3115c246f..ad511cc9e 100644 --- a/test/e2e/libpod_suite_remote_test.go +++ b/test/e2e/libpod_suite_remote_test.go @@ -38,11 +38,25 @@ func (p *PodmanTestIntegration) Podman(args []string) *PodmanSessionIntegration return &PodmanSessionIntegration{podmanSession} } +// PodmanSystemdScope runs the podman command in a new systemd scope +func (p *PodmanTestIntegration) PodmanSystemdScope(args []string) *PodmanSessionIntegration { + var remoteArgs = []string{"--remote", "--url", p.RemoteSocket} + remoteArgs = append(remoteArgs, args...) + + wrapper := []string{"systemd-run", "--scope"} + if rootless.IsRootless() { + wrapper = []string{"systemd-run", "--scope", "--user"} + } + + podmanSession := p.PodmanAsUserBase(remoteArgs, 0, 0, "", nil, false, false, wrapper, nil) + return &PodmanSessionIntegration{podmanSession} +} + // PodmanExtraFiles is the exec call to podman on the filesystem and passes down extra files func (p *PodmanTestIntegration) PodmanExtraFiles(args []string, extraFiles []*os.File) *PodmanSessionIntegration { var remoteArgs = []string{"--remote", "--url", p.RemoteSocket} remoteArgs = append(remoteArgs, args...) - podmanSession := p.PodmanAsUserBase(remoteArgs, 0, 0, "", nil, false, false, extraFiles) + podmanSession := p.PodmanAsUserBase(remoteArgs, 0, 0, "", nil, false, false, nil, extraFiles) return &PodmanSessionIntegration{podmanSession} } diff --git a/test/e2e/libpod_suite_test.go b/test/e2e/libpod_suite_test.go index cc03ccc96..6d2d3fee8 100644 --- a/test/e2e/libpod_suite_test.go +++ b/test/e2e/libpod_suite_test.go @@ -8,6 +8,8 @@ import ( "os" "path/filepath" "strings" + + "github.com/containers/podman/v3/pkg/rootless" ) func IsRemote() bool { @@ -23,9 +25,19 @@ func (p *PodmanTestIntegration) Podman(args []string) *PodmanSessionIntegration return &PodmanSessionIntegration{podmanSession} } +// PodmanSystemdScope runs the podman command in a new systemd scope +func (p *PodmanTestIntegration) PodmanSystemdScope(args []string) *PodmanSessionIntegration { + wrapper := []string{"systemd-run", "--scope"} + if rootless.IsRootless() { + wrapper = []string{"systemd-run", "--scope", "--user"} + } + podmanSession := p.PodmanAsUserBase(args, 0, 0, "", nil, false, false, wrapper, nil) + return &PodmanSessionIntegration{podmanSession} +} + // PodmanExtraFiles is the exec call to podman on the filesystem and passes down extra files func (p *PodmanTestIntegration) PodmanExtraFiles(args []string, extraFiles []*os.File) *PodmanSessionIntegration { - podmanSession := p.PodmanAsUserBase(args, 0, 0, "", nil, false, false, extraFiles) + podmanSession := p.PodmanAsUserBase(args, 0, 0, "", nil, false, false, nil, extraFiles) return &PodmanSessionIntegration{podmanSession} } diff --git a/test/e2e/run_networking_test.go b/test/e2e/run_networking_test.go index bdf3ce5d6..c64cfd2d5 100644 --- a/test/e2e/run_networking_test.go +++ b/test/e2e/run_networking_test.go @@ -494,6 +494,23 @@ var _ = Describe("Podman run networking", func() { Expect(containerConfig[0].NetworkSettings.Ports["80/tcp"][0].HostPort).ToNot(Equal(80)) }) + It("podman run forward sctp protocol", func() { + SkipIfRootless("sctp protocol only works as root") + session := podmanTest.Podman([]string{"--log-level=info", "run", "--name=test", "-p", "80/sctp", "-p", "81/sctp", ALPINE}) + session.Wait(90) + Expect(session).Should(Exit(0)) + // we can only check logrus on local podman + if !IsRemote() { + // check that the info message for sctp protocol is only displayed once + Expect(strings.Count(session.ErrorToString(), "Port reservation for SCTP is not supported")).To(Equal(1), "`Port reservation for SCTP is not supported` is not displayed exactly one time in the logrus logs") + } + results := podmanTest.Podman([]string{"inspect", "test"}) + results.Wait(30) + Expect(results).Should(Exit(0)) + Expect(results.OutputToString()).To(ContainSubstring(`"80/sctp":`)) + Expect(results.OutputToString()).To(ContainSubstring(`"81/sctp":`)) + }) + It("podman run hostname test", func() { session := podmanTest.Podman([]string{"run", "--rm", ALPINE, "printenv", "HOSTNAME"}) session.WaitWithDefaultTimeout() diff --git a/test/e2e/run_test.go b/test/e2e/run_test.go index 95660bfc9..ed2d8938d 100644 --- a/test/e2e/run_test.go +++ b/test/e2e/run_test.go @@ -1381,13 +1381,13 @@ USER mail`, BB) } } - container := podmanTest.Podman([]string{"run", "--rm", "--cgroups=split", ALPINE, "cat", "/proc/self/cgroup"}) + container := podmanTest.PodmanSystemdScope([]string{"run", "--rm", "--cgroups=split", ALPINE, "cat", "/proc/self/cgroup"}) container.WaitWithDefaultTimeout() Expect(container).Should(Exit(0)) checkLines(container.OutputToStringArray()) // check that --cgroups=split is honored also when a container runs in a pod - container = podmanTest.Podman([]string{"run", "--rm", "--pod", "new:split-test-pod", "--cgroups=split", ALPINE, "cat", "/proc/self/cgroup"}) + container = podmanTest.PodmanSystemdScope([]string{"run", "--rm", "--pod", "new:split-test-pod", "--cgroups=split", ALPINE, "cat", "/proc/self/cgroup"}) container.WaitWithDefaultTimeout() Expect(container).Should(Exit(0)) checkLines(container.OutputToStringArray()) diff --git a/test/e2e/trust_test.go b/test/e2e/trust_test.go index 7f97f280a..b591e1c02 100644 --- a/test/e2e/trust_test.go +++ b/test/e2e/trust_test.go @@ -14,7 +14,8 @@ import ( var _ = Describe("Podman trust", func() { var ( - tempdir string + tempdir string + err error podmanTest *PodmanTestIntegration ) @@ -38,21 +39,17 @@ var _ = Describe("Podman trust", func() { }) It("podman image trust show", func() { - path, err := os.Getwd() - if err != nil { - os.Exit(1) - } - session := podmanTest.Podman([]string{"image", "trust", "show", "--registrypath", filepath.Dir(path), "--policypath", filepath.Join(filepath.Dir(path), "policy.json")}) + session := podmanTest.Podman([]string{"image", "trust", "show", "--registrypath", filepath.Join(INTEGRATION_ROOT, "test"), "--policypath", filepath.Join(INTEGRATION_ROOT, "test/policy.json")}) session.WaitWithDefaultTimeout() Expect(session).Should(Exit(0)) outArray := session.OutputToStringArray() Expect(len(outArray)).To(Equal(3)) - // image order is not guaranteed. All we can do is check that - // these strings appear in output, we can't cross-check them. - Expect(session.OutputToString()).To(ContainSubstring("accept")) - Expect(session.OutputToString()).To(ContainSubstring("reject")) - Expect(session.OutputToString()).To(ContainSubstring("signed")) + // Repository order is not guaranteed. So, check that + // all expected lines appear in output; we also check total number of lines, so that handles all of them. + Expect(string(session.Out.Contents())).To(MatchRegexp(`(?m)^default\s+accept\s*$`)) + Expect(string(session.Out.Contents())).To(MatchRegexp(`(?m)^docker.io/library/hello-world\s+reject\s*$`)) + Expect(string(session.Out.Contents())).To(MatchRegexp(`(?m)^registry.access.redhat.com\s+signedBy\s+security@redhat.com, security@redhat.com\s+https://access.redhat.com/webassets/docker/content/sigstore\s*$`)) }) It("podman image trust set", func() { @@ -76,24 +73,52 @@ var _ = Describe("Podman trust", func() { }) It("podman image trust show --json", func() { - session := podmanTest.Podman([]string{"image", "trust", "show", "--json"}) + session := podmanTest.Podman([]string{"image", "trust", "show", "--registrypath", filepath.Join(INTEGRATION_ROOT, "test"), "--policypath", filepath.Join(INTEGRATION_ROOT, "test/policy.json"), "--json"}) session.WaitWithDefaultTimeout() Expect(session).Should(Exit(0)) Expect(session.IsJSONOutputValid()).To(BeTrue()) var teststruct []map[string]string json.Unmarshal(session.Out.Contents(), &teststruct) - Expect(teststruct[0]["name"]).To(Equal("* (default)")) - Expect(teststruct[0]["repo_name"]).To(Equal("default")) - Expect(teststruct[0]["type"]).To(Equal("accept")) - Expect(teststruct[1]["type"]).To(Equal("insecureAcceptAnything")) + Expect(len(teststruct)).To(Equal(3)) + // To ease comparison, group the unordered array of repos by repo (and we expect only one entry by repo, so order within groups doesn’t matter) + repoMap := map[string][]map[string]string{} + for _, e := range teststruct { + key := e["name"] + repoMap[key] = append(repoMap[key], e) + } + Expect(repoMap).To(Equal(map[string][]map[string]string{ + "* (default)": {{ + "name": "* (default)", + "repo_name": "default", + "sigstore": "", + "transport": "", + "type": "accept", + }}, + "docker.io/library/hello-world": {{ + "name": "docker.io/library/hello-world", + "repo_name": "docker.io/library/hello-world", + "sigstore": "", + "transport": "", + "type": "reject", + }}, + "registry.access.redhat.com": {{ + "name": "registry.access.redhat.com", + "repo_name": "registry.access.redhat.com", + "sigstore": "https://access.redhat.com/webassets/docker/content/sigstore", + "transport": "", + "type": "signedBy", + "gpg_id": "security@redhat.com, security@redhat.com", + }}, + })) }) It("podman image trust show --raw", func() { - session := podmanTest.Podman([]string{"image", "trust", "show", "--raw"}) + session := podmanTest.Podman([]string{"image", "trust", "show", "--policypath", filepath.Join(INTEGRATION_ROOT, "test/policy.json"), "--raw"}) session.WaitWithDefaultTimeout() Expect(session).Should(Exit(0)) + contents, err := ioutil.ReadFile(filepath.Join(INTEGRATION_ROOT, "test/policy.json")) + Expect(err).ShouldNot(HaveOccurred()) Expect(session.IsJSONOutputValid()).To(BeTrue()) - Expect(session.OutputToString()).To(ContainSubstring("default")) - Expect(session.OutputToString()).To(ContainSubstring("insecureAcceptAnything")) + Expect(string(session.Out.Contents())).To(Equal(string(contents) + "\n")) }) }) diff --git a/test/e2e/unshare_test.go b/test/e2e/unshare_test.go index 79ce68e89..cf1b8db53 100644 --- a/test/e2e/unshare_test.go +++ b/test/e2e/unshare_test.go @@ -51,7 +51,7 @@ var _ = Describe("Podman unshare", func() { }) It("podman unshare --rootles-cni", func() { - session := podmanTest.Podman([]string{"unshare", "--rootless-cni", "ip", "addr"}) + session := podmanTest.Podman([]string{"unshare", "--rootless-netns", "ip", "addr"}) session.WaitWithDefaultTimeout() Expect(session).Should(Exit(0)) Expect(session.OutputToString()).To(ContainSubstring("tap0")) diff --git a/test/system/035-logs.bats b/test/system/035-logs.bats index 44b66676e..7fb3e62e4 100644 --- a/test/system/035-logs.bats +++ b/test/system/035-logs.bats @@ -30,6 +30,17 @@ load helpers run_podman rm $cid } +function _additional_events_backend() { + local driver=$1 + # Since PR#10431, 'logs -f' with journald driver is only supported with journald events backend. + if [[ $driver = "journald" ]]; then + run_podman info --format '{{.Host.EventLogger}}' >/dev/null + if [[ $output != "journald" ]]; then + echo "--events-backend journald" + fi + fi +} + function _log_test_multi() { local driver=$1 @@ -42,10 +53,12 @@ function _log_test_multi() { etc='.*' fi + local events_backend=$(_additional_events_backend $driver) + # Simple helper to make the container starts, below, easier to read local -a cid doit() { - run_podman run --log-driver=$driver --rm -d --name "$1" $IMAGE sh -c "$2"; + run_podman ${events_backend} run --log-driver=$driver --rm -d --name "$1" $IMAGE sh -c "$2"; cid+=($(echo "${output:0:12}")) } @@ -57,7 +70,7 @@ function _log_test_multi() { doit c1 "echo a;sleep 10;echo d;sleep 3" doit c2 "sleep 1;echo b;sleep 2;echo c;sleep 3" - run_podman logs -f c1 c2 + run_podman ${events_backend} logs -f c1 c2 is "$output" \ "${cid[0]} a$etc ${cid[1]} b$etc @@ -187,15 +200,20 @@ function _log_test_follow() { contentA=$(random_string) contentB=$(random_string) contentC=$(random_string) + local events_backend=$(_additional_events_backend $driver) + + if [[ -n "${events_backend}" ]]; then + skip_if_remote "remote does not support --events-backend" + fi # Note: it seems we need at least three log lines to hit #11461. - run_podman run --log-driver=$driver --name $cname $IMAGE sh -c "echo $contentA; echo $contentB; echo $contentC" - run_podman logs -f $cname + run_podman ${events_backend} run --log-driver=$driver --name $cname $IMAGE sh -c "echo $contentA; echo $contentB; echo $contentC" + run_podman ${events_backend} logs -f $cname is "$output" "$contentA $contentB $contentC" "logs -f on exitted container works" - run_podman rm -t 0 -f $cname + run_podman ${events_backend} rm -t 0 -f $cname } @test "podman logs - --follow k8s-file" { diff --git a/test/system/070-build.bats b/test/system/070-build.bats index 1d24eef15..3c47b1f5b 100644 --- a/test/system/070-build.bats +++ b/test/system/070-build.bats @@ -39,6 +39,7 @@ EOF cat >$dockerfile <<EOF FROM $IMAGE RUN echo $rand_content > /$rand_filename +VOLUME /a/b/c VOLUME ['/etc/foo', '/etc/bar'] EOF @@ -56,6 +57,25 @@ EOF /\[/etc/foo, /etc/bar]" "weird VOLUME gets converted to directories with brackets and comma" + # Now confirm that each volume got a unique device ID + run_podman run --rm build_test stat -c '%D' / /a /a/b /a/b/c /\[ /\[/etc /\[/etc/foo, /etc /etc/bar\] + # First, the non-volumes should all be the same... + is "${lines[0]}" "${lines[1]}" "devnum( / ) = devnum( /a )" + is "${lines[0]}" "${lines[2]}" "devnum( / ) = devnum( /a/b )" + is "${lines[0]}" "${lines[4]}" "devnum( / ) = devnum( /[ )" + is "${lines[0]}" "${lines[5]}" "devnum( / ) = devnum( /[etc )" + is "${lines[0]}" "${lines[7]}" "devnum( / ) = devnum( /etc )" + is "${lines[6]}" "${lines[8]}" "devnum( /[etc/foo, ) = devnum( /etc/bar] )" + # ...then, each volume should be different + if [[ "${lines[0]}" = "${lines[3]}" ]]; then + die "devnum( / ) (${lines[0]}) = devnum( volume0 ) (${lines[3]}) -- they should differ" + fi + if [[ "${lines[0]}" = "${lines[6]}" ]]; then + die "devnum( / ) (${lines[0]}) = devnum( volume1 ) (${lines[6]}) -- they should differ" + fi + # FIXME: is this expected? I thought /a/b/c and /[etc/foo, would differ + is "${lines[3]}" "${lines[6]}" "devnum( volume0 ) = devnum( volume1 )" + run_podman rmi -f build_test } diff --git a/test/system/130-kill.bats b/test/system/130-kill.bats index d85f0a6a9..1ff3a7b61 100644 --- a/test/system/130-kill.bats +++ b/test/system/130-kill.bats @@ -6,9 +6,22 @@ load helpers @test "podman kill - test signal handling in containers" { + + # Prepare for 'logs -f' + run_podman info --format '{{.Host.LogDriver}}' + log_driver=$output + run_podman info --format '{{.Host.EventLogger}}' + event_logger=$output + opt_log_driver= + if [ $log_driver = "journald" ] && [ $event_logger != "journald" ]; then + # Since PR#10431, 'logs -f' with journald driver is only supported with journald events backend. + # Set '--log driver' temporally because remote doesn't support '--events-backend'. + opt_log_driver="--log-driver k8s-file" + fi + # Start a container that will handle all signals by emitting 'got: N' local -a signals=(1 2 3 4 5 6 8 10 12 13 14 15 16 20 21 22 23 24 25 26 64) - run_podman run -d $IMAGE sh -c \ + run_podman run -d ${opt_log_driver} $IMAGE sh -c \ "for i in ${signals[*]}; do trap \"echo got: \$i\" \$i; done; echo READY; while ! test -e /stop; do sleep 0.05; done; diff --git a/test/system/500-networking.bats b/test/system/500-networking.bats index cb73cf24d..b3471b425 100644 --- a/test/system/500-networking.bats +++ b/test/system/500-networking.bats @@ -444,6 +444,14 @@ load helpers die "MAC address did not change after podman network disconnect/connect" fi + # FIXME FIXME FIXME: #11825: bodhi tests are failing, remote+rootless only, + # with "dnsmasq: failed to create inotify". This error has never occurred + # in CI, and Ed has been unable to reproduce it on 1minutetip. This next + # line is a suggestion from Paul Holzinger for trying to shed light on + # the system context before the failure. This output will be invisible + # if the test passes. + for foo in /proc/\*/fd/*; do readlink -f $foo; done |grep '^/proc/.*inotify' |cut -d/ -f3 | xargs -I '{}' -- ps --no-headers -o '%p %U %a' -p '{}' |uniq -c |sort -n + # connect a second network run_podman network connect $netname2 $cid diff --git a/test/upgrade/test-upgrade.bats b/test/upgrade/test-upgrade.bats index 5cb302a85..f6f32242b 100644 --- a/test/upgrade/test-upgrade.bats +++ b/test/upgrade/test-upgrade.bats @@ -97,8 +97,10 @@ podman \$opts run --name myfailedcontainer --label mylabel=$LABEL_FAILED \ podman \$opts run -d --name myrunningcontainer --label mylabel=$LABEL_RUNNING \ --network bridge \ -p $HOST_PORT:80 \ + -p 127.0.0.1:8080-8082:8080-8082 \ -v $pmroot/var/www:/var/www \ -w /var/www \ + --mac-address aa:bb:cc:dd:ee:ff \ $IMAGE /bin/busybox-extras httpd -f -p 80 podman \$opts pod create --name mypod @@ -185,7 +187,7 @@ EOF is "${lines[1]}" "mycreatedcontainer--Created----$LABEL_CREATED" "created" is "${lines[2]}" "mydonecontainer--Exited (0).*----<no value>" "done" is "${lines[3]}" "myfailedcontainer--Exited (17) .*----$LABEL_FAILED" "fail" - is "${lines[4]}" "myrunningcontainer--Up .*--0.0.0.0:$HOST_PORT->80/tcp--$LABEL_RUNNING" "running" + is "${lines[4]}" "myrunningcontainer--Up .*--0\.0\.0\.0:$HOST_PORT->80\/tcp, 127\.0\.0\.1\:8080-8082->8080-8082\/tcp--$LABEL_RUNNING" "running" # For debugging: dump containers and IDs if [[ -n "$PODMAN_UPGRADE_TEST_DEBUG" ]]; then diff --git a/test/utils/podmantest_test.go b/test/utils/podmantest_test.go index 9bd1c4a66..1bb9ecb6b 100644 --- a/test/utils/podmantest_test.go +++ b/test/utils/podmantest_test.go @@ -23,7 +23,7 @@ var _ = Describe("PodmanTest test", func() { FakeOutputs["check"] = []string{"check"} os.Setenv("HOOK_OPTION", "hook_option") env := os.Environ() - session := podmanTest.PodmanAsUserBase([]string{"check"}, 1000, 1000, "", env, true, false, nil) + session := podmanTest.PodmanAsUserBase([]string{"check"}, 1000, 1000, "", env, true, false, nil, nil) os.Unsetenv("HOOK_OPTION") session.WaitWithDefaultTimeout() Expect(session.Command.Process).ShouldNot(BeNil()) diff --git a/test/utils/utils.go b/test/utils/utils.go index bfefc58ec..d4d5e6e2f 100644 --- a/test/utils/utils.go +++ b/test/utils/utils.go @@ -66,27 +66,31 @@ func (p *PodmanTest) MakeOptions(args []string, noEvents, noCache bool) []string // PodmanAsUserBase exec podman as user. uid and gid is set for credentials usage. env is used // to record the env for debugging -func (p *PodmanTest) PodmanAsUserBase(args []string, uid, gid uint32, cwd string, env []string, noEvents, noCache bool, extraFiles []*os.File) *PodmanSession { +func (p *PodmanTest) PodmanAsUserBase(args []string, uid, gid uint32, cwd string, env []string, noEvents, noCache bool, wrapper []string, extraFiles []*os.File) *PodmanSession { var command *exec.Cmd podmanOptions := p.MakeOptions(args, noEvents, noCache) podmanBinary := p.PodmanBinary if p.RemoteTest { podmanBinary = p.RemotePodmanBinary } + + runCmd := append(wrapper, podmanBinary) if p.RemoteTest { podmanOptions = append([]string{"--remote", "--url", p.RemoteSocket}, podmanOptions...) } if env == nil { - fmt.Printf("Running: %s %s\n", podmanBinary, strings.Join(podmanOptions, " ")) + fmt.Printf("Running: %s %s\n", strings.Join(runCmd, " "), strings.Join(podmanOptions, " ")) } else { - fmt.Printf("Running: (env: %v) %s %s\n", env, podmanBinary, strings.Join(podmanOptions, " ")) + fmt.Printf("Running: (env: %v) %s %s\n", env, strings.Join(runCmd, " "), strings.Join(podmanOptions, " ")) } if uid != 0 || gid != 0 { pythonCmd := fmt.Sprintf("import os; import sys; uid = %d; gid = %d; cwd = '%s'; os.setgid(gid); os.setuid(uid); os.chdir(cwd) if len(cwd)>0 else True; os.execv(sys.argv[1], sys.argv[1:])", gid, uid, cwd) - nsEnterOpts := append([]string{"-c", pythonCmd, podmanBinary}, podmanOptions...) + runCmd = append(runCmd, podmanOptions...) + nsEnterOpts := append([]string{"-c", pythonCmd}, runCmd...) command = exec.Command("python", nsEnterOpts...) } else { - command = exec.Command(podmanBinary, podmanOptions...) + runCmd = append(runCmd, podmanOptions...) + command = exec.Command(runCmd[0], runCmd[1:]...) } if env != nil { command.Env = env @@ -106,7 +110,7 @@ func (p *PodmanTest) PodmanAsUserBase(args []string, uid, gid uint32, cwd string // PodmanBase exec podman with default env. func (p *PodmanTest) PodmanBase(args []string, noEvents, noCache bool) *PodmanSession { - return p.PodmanAsUserBase(args, 0, 0, "", nil, noEvents, noCache, nil) + return p.PodmanAsUserBase(args, 0, 0, "", nil, noEvents, noCache, nil, nil) } // WaitForContainer waits on a started container diff --git a/vendor/github.com/checkpoint-restore/go-criu/v5/.gitignore b/vendor/github.com/checkpoint-restore/go-criu/v5/.gitignore index 6c7385fa2..1b87ff10e 100644 --- a/vendor/github.com/checkpoint-restore/go-criu/v5/.gitignore +++ b/vendor/github.com/checkpoint-restore/go-criu/v5/.gitignore @@ -1,5 +1,6 @@ test/test +test/test.coverage test/piggie/piggie -test/phaul +test/phaul/phaul +test/phaul/phaul.coverage image -stats/stats.proto diff --git a/vendor/github.com/checkpoint-restore/go-criu/v5/Makefile b/vendor/github.com/checkpoint-restore/go-criu/v5/Makefile index a5c5f5542..558e61453 100644 --- a/vendor/github.com/checkpoint-restore/go-criu/v5/Makefile +++ b/vendor/github.com/checkpoint-restore/go-criu/v5/Makefile @@ -1,5 +1,7 @@ +SHELL = /bin/bash GO ?= go CC ?= gcc +COVERAGE_PATH ?= $(shell pwd)/.coverage all: build test phaul-test @@ -9,13 +11,15 @@ lint: build: $(GO) build -v ./... -TEST_BINARIES := test/test test/piggie/piggie test/phaul/phaul +TEST_PAYLOAD := test/piggie/piggie +TEST_BINARIES := test/test $(TEST_PAYLOAD) test/phaul/phaul +COVERAGE_BINARIES := test/test.coverage test/phaul/phaul.coverage test-bin: $(TEST_BINARIES) test/piggie/piggie: test/piggie/piggie.c $(CC) $^ -o $@ -test/test: test/*.go +test/test: test/main.go $(GO) build -v -o $@ $^ test: $(TEST_BINARIES) @@ -27,7 +31,7 @@ test: $(TEST_BINARIES) } rm -rf image -test/phaul/phaul: test/phaul/*.go +test/phaul/phaul: test/phaul/main.go $(GO) build -v -o $@ $^ phaul-test: $(TEST_BINARIES) @@ -37,10 +41,39 @@ phaul-test: $(TEST_BINARIES) pkill -9 piggie; \ } +test/test.coverage: test/*.go + $(GO) test \ + -covermode=count \ + -coverpkg=./... \ + -mod=vendor \ + -tags coverage \ + -buildmode=pie -c -o $@ $^ + +test/phaul/phaul.coverage: test/phaul/*.go + $(GO) test \ + -covermode=count \ + -coverpkg=./... \ + -mod=vendor \ + -tags coverage \ + -buildmode=pie -c -o $@ $^ + +coverage: $(COVERAGE_BINARIES) $(TEST_PAYLOAD) + mkdir -p $(COVERAGE_PATH) + mkdir -p image + PID=$$(test/piggie/piggie) && { \ + test/test.coverage -test.coverprofile=coverprofile.integration.$$RANDOM -test.outputdir=${COVERAGE_PATH} COVERAGE dump $$PID image && \ + test/test.coverage -test.coverprofile=coverprofile.integration.$$RANDOM -test.outputdir=${COVERAGE_PATH} COVERAGE restore image; \ + pkill -9 piggie; \ + } + rm -rf image + PID=$$(test/piggie/piggie) && { \ + test/phaul/phaul.coverage -test.coverprofile=coverprofile.integration.$$RANDOM -test.outputdir=${COVERAGE_PATH} COVERAGE $$PID; \ + pkill -9 piggie; \ + } + clean: - @rm -f $(TEST_BINARIES) - @rm -rf image - @rm -f rpc/rpc.proto stats/stats.proto + @rm -f $(TEST_BINARIES) $(COVERAGE_BINARIES) codecov + @rm -rf image $(COVERAGE_PATH) rpc/rpc.proto: curl -sSL https://raw.githubusercontent.com/checkpoint-restore/criu/master/images/rpc.proto -o $@ @@ -49,14 +82,19 @@ stats/stats.proto: curl -sSL https://raw.githubusercontent.com/checkpoint-restore/criu/master/images/stats.proto -o $@ rpc/rpc.pb.go: rpc/rpc.proto - protoc --go_out=. --go_opt=Mrpc/rpc.proto=rpc/ $^ + protoc --go_out=. --go_opt=M$^=rpc/ $^ stats/stats.pb.go: stats/stats.proto - protoc --go_out=. $^ + protoc --go_out=. --go_opt=M$^=stats/ $^ vendor: GO111MODULE=on $(GO) mod tidy GO111MODULE=on $(GO) mod vendor GO111MODULE=on $(GO) mod verify -.PHONY: build test phaul-test test-bin clean lint vendor +codecov: + curl -Os https://uploader.codecov.io/latest/linux/codecov + chmod +x codecov + ./codecov -f '.coverage/*' + +.PHONY: build test phaul-test test-bin clean lint vendor coverage codecov diff --git a/vendor/github.com/checkpoint-restore/go-criu/v5/README.md b/vendor/github.com/checkpoint-restore/go-criu/v5/README.md index 390da3e98..a7483321b 100644 --- a/vendor/github.com/checkpoint-restore/go-criu/v5/README.md +++ b/vendor/github.com/checkpoint-restore/go-criu/v5/README.md @@ -16,7 +16,7 @@ The following example would print the version of CRIU: import ( "log" - "github.com/checkpoint/restore/go-criu/v5" + "github.com/checkpoint-restore/go-criu/v5" ) func main() { @@ -50,6 +50,7 @@ The following table shows the relation between go-criu and criu versions: | Major version | Latest release | CRIU version | | -------------- | -------------- | ------------ | +| v5 | 5.2.0 | 3.16 | | v5 | 5.0.0 | 3.15 | | v4 | 4.1.0 | 3.14 | diff --git a/vendor/github.com/checkpoint-restore/go-criu/v5/go.mod b/vendor/github.com/checkpoint-restore/go-criu/v5/go.mod index 69595701f..cf4fea9f0 100644 --- a/vendor/github.com/checkpoint-restore/go-criu/v5/go.mod +++ b/vendor/github.com/checkpoint-restore/go-criu/v5/go.mod @@ -3,7 +3,6 @@ module github.com/checkpoint-restore/go-criu/v5 go 1.13 require ( - github.com/golang/protobuf v1.5.2 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c - google.golang.org/protobuf v1.26.0 + google.golang.org/protobuf v1.27.1 ) diff --git a/vendor/github.com/checkpoint-restore/go-criu/v5/go.sum b/vendor/github.com/checkpoint-restore/go-criu/v5/go.sum index 7e17df214..789fdcb11 100644 --- a/vendor/github.com/checkpoint-restore/go-criu/v5/go.sum +++ b/vendor/github.com/checkpoint-restore/go-criu/v5/go.sum @@ -1,6 +1,4 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c h1:VwygUrnw9jn88c4u8GD3rZQbqrP/tgas88tPUbBxQrk= @@ -8,5 +6,5 @@ golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= diff --git a/vendor/github.com/checkpoint-restore/go-criu/v5/magic/types.go b/vendor/github.com/checkpoint-restore/go-criu/v5/magic/types.go new file mode 100644 index 000000000..24cc3989a --- /dev/null +++ b/vendor/github.com/checkpoint-restore/go-criu/v5/magic/types.go @@ -0,0 +1,12 @@ +package magic + +const ( + ImgCommonMagic = 0x54564319 /* Sarov (a.k.a. Arzamas-16) */ + ImgServiceMagic = 0x55105940 /* Zlatoust */ + StatsMagic = 0x57093306 /* Ostashkov */ + + PrimaryMagicOffset = 0x0 + SecondaryMagicOffset = 0x4 + SizeOffset = 0x8 + PayloadOffset = 0xC +) diff --git a/vendor/github.com/checkpoint-restore/go-criu/v5/rpc/rpc.pb.go b/vendor/github.com/checkpoint-restore/go-criu/v5/rpc/rpc.pb.go index 9f22f1539..15e33fea5 100644 --- a/vendor/github.com/checkpoint-restore/go-criu/v5/rpc/rpc.pb.go +++ b/vendor/github.com/checkpoint-restore/go-criu/v5/rpc/rpc.pb.go @@ -2,8 +2,8 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.26.0 -// protoc v3.14.0 +// protoc-gen-go v1.27.1 +// protoc v3.12.4 // source: rpc/rpc.proto package rpc diff --git a/vendor/github.com/checkpoint-restore/go-criu/v5/stats/stats.pb.go b/vendor/github.com/checkpoint-restore/go-criu/v5/stats/stats.pb.go new file mode 100644 index 000000000..ff011fc2c --- /dev/null +++ b/vendor/github.com/checkpoint-restore/go-criu/v5/stats/stats.pb.go @@ -0,0 +1,462 @@ +// SPDX-License-Identifier: MIT + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.27.1 +// protoc v3.12.4 +// source: stats/stats.proto + +package stats + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// This one contains statistics about dump/restore process +type DumpStatsEntry struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + FreezingTime *uint32 `protobuf:"varint,1,req,name=freezing_time,json=freezingTime" json:"freezing_time,omitempty"` + FrozenTime *uint32 `protobuf:"varint,2,req,name=frozen_time,json=frozenTime" json:"frozen_time,omitempty"` + MemdumpTime *uint32 `protobuf:"varint,3,req,name=memdump_time,json=memdumpTime" json:"memdump_time,omitempty"` + MemwriteTime *uint32 `protobuf:"varint,4,req,name=memwrite_time,json=memwriteTime" json:"memwrite_time,omitempty"` + PagesScanned *uint64 `protobuf:"varint,5,req,name=pages_scanned,json=pagesScanned" json:"pages_scanned,omitempty"` + PagesSkippedParent *uint64 `protobuf:"varint,6,req,name=pages_skipped_parent,json=pagesSkippedParent" json:"pages_skipped_parent,omitempty"` + PagesWritten *uint64 `protobuf:"varint,7,req,name=pages_written,json=pagesWritten" json:"pages_written,omitempty"` + IrmapResolve *uint32 `protobuf:"varint,8,opt,name=irmap_resolve,json=irmapResolve" json:"irmap_resolve,omitempty"` + PagesLazy *uint64 `protobuf:"varint,9,req,name=pages_lazy,json=pagesLazy" json:"pages_lazy,omitempty"` + PagePipes *uint64 `protobuf:"varint,10,opt,name=page_pipes,json=pagePipes" json:"page_pipes,omitempty"` + PagePipeBufs *uint64 `protobuf:"varint,11,opt,name=page_pipe_bufs,json=pagePipeBufs" json:"page_pipe_bufs,omitempty"` + ShpagesScanned *uint64 `protobuf:"varint,12,opt,name=shpages_scanned,json=shpagesScanned" json:"shpages_scanned,omitempty"` + ShpagesSkippedParent *uint64 `protobuf:"varint,13,opt,name=shpages_skipped_parent,json=shpagesSkippedParent" json:"shpages_skipped_parent,omitempty"` + ShpagesWritten *uint64 `protobuf:"varint,14,opt,name=shpages_written,json=shpagesWritten" json:"shpages_written,omitempty"` +} + +func (x *DumpStatsEntry) Reset() { + *x = DumpStatsEntry{} + if protoimpl.UnsafeEnabled { + mi := &file_stats_stats_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *DumpStatsEntry) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DumpStatsEntry) ProtoMessage() {} + +func (x *DumpStatsEntry) ProtoReflect() protoreflect.Message { + mi := &file_stats_stats_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DumpStatsEntry.ProtoReflect.Descriptor instead. +func (*DumpStatsEntry) Descriptor() ([]byte, []int) { + return file_stats_stats_proto_rawDescGZIP(), []int{0} +} + +func (x *DumpStatsEntry) GetFreezingTime() uint32 { + if x != nil && x.FreezingTime != nil { + return *x.FreezingTime + } + return 0 +} + +func (x *DumpStatsEntry) GetFrozenTime() uint32 { + if x != nil && x.FrozenTime != nil { + return *x.FrozenTime + } + return 0 +} + +func (x *DumpStatsEntry) GetMemdumpTime() uint32 { + if x != nil && x.MemdumpTime != nil { + return *x.MemdumpTime + } + return 0 +} + +func (x *DumpStatsEntry) GetMemwriteTime() uint32 { + if x != nil && x.MemwriteTime != nil { + return *x.MemwriteTime + } + return 0 +} + +func (x *DumpStatsEntry) GetPagesScanned() uint64 { + if x != nil && x.PagesScanned != nil { + return *x.PagesScanned + } + return 0 +} + +func (x *DumpStatsEntry) GetPagesSkippedParent() uint64 { + if x != nil && x.PagesSkippedParent != nil { + return *x.PagesSkippedParent + } + return 0 +} + +func (x *DumpStatsEntry) GetPagesWritten() uint64 { + if x != nil && x.PagesWritten != nil { + return *x.PagesWritten + } + return 0 +} + +func (x *DumpStatsEntry) GetIrmapResolve() uint32 { + if x != nil && x.IrmapResolve != nil { + return *x.IrmapResolve + } + return 0 +} + +func (x *DumpStatsEntry) GetPagesLazy() uint64 { + if x != nil && x.PagesLazy != nil { + return *x.PagesLazy + } + return 0 +} + +func (x *DumpStatsEntry) GetPagePipes() uint64 { + if x != nil && x.PagePipes != nil { + return *x.PagePipes + } + return 0 +} + +func (x *DumpStatsEntry) GetPagePipeBufs() uint64 { + if x != nil && x.PagePipeBufs != nil { + return *x.PagePipeBufs + } + return 0 +} + +func (x *DumpStatsEntry) GetShpagesScanned() uint64 { + if x != nil && x.ShpagesScanned != nil { + return *x.ShpagesScanned + } + return 0 +} + +func (x *DumpStatsEntry) GetShpagesSkippedParent() uint64 { + if x != nil && x.ShpagesSkippedParent != nil { + return *x.ShpagesSkippedParent + } + return 0 +} + +func (x *DumpStatsEntry) GetShpagesWritten() uint64 { + if x != nil && x.ShpagesWritten != nil { + return *x.ShpagesWritten + } + return 0 +} + +type RestoreStatsEntry struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + PagesCompared *uint64 `protobuf:"varint,1,req,name=pages_compared,json=pagesCompared" json:"pages_compared,omitempty"` + PagesSkippedCow *uint64 `protobuf:"varint,2,req,name=pages_skipped_cow,json=pagesSkippedCow" json:"pages_skipped_cow,omitempty"` + ForkingTime *uint32 `protobuf:"varint,3,req,name=forking_time,json=forkingTime" json:"forking_time,omitempty"` + RestoreTime *uint32 `protobuf:"varint,4,req,name=restore_time,json=restoreTime" json:"restore_time,omitempty"` + PagesRestored *uint64 `protobuf:"varint,5,opt,name=pages_restored,json=pagesRestored" json:"pages_restored,omitempty"` +} + +func (x *RestoreStatsEntry) Reset() { + *x = RestoreStatsEntry{} + if protoimpl.UnsafeEnabled { + mi := &file_stats_stats_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RestoreStatsEntry) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RestoreStatsEntry) ProtoMessage() {} + +func (x *RestoreStatsEntry) ProtoReflect() protoreflect.Message { + mi := &file_stats_stats_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RestoreStatsEntry.ProtoReflect.Descriptor instead. +func (*RestoreStatsEntry) Descriptor() ([]byte, []int) { + return file_stats_stats_proto_rawDescGZIP(), []int{1} +} + +func (x *RestoreStatsEntry) GetPagesCompared() uint64 { + if x != nil && x.PagesCompared != nil { + return *x.PagesCompared + } + return 0 +} + +func (x *RestoreStatsEntry) GetPagesSkippedCow() uint64 { + if x != nil && x.PagesSkippedCow != nil { + return *x.PagesSkippedCow + } + return 0 +} + +func (x *RestoreStatsEntry) GetForkingTime() uint32 { + if x != nil && x.ForkingTime != nil { + return *x.ForkingTime + } + return 0 +} + +func (x *RestoreStatsEntry) GetRestoreTime() uint32 { + if x != nil && x.RestoreTime != nil { + return *x.RestoreTime + } + return 0 +} + +func (x *RestoreStatsEntry) GetPagesRestored() uint64 { + if x != nil && x.PagesRestored != nil { + return *x.PagesRestored + } + return 0 +} + +type StatsEntry struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Dump *DumpStatsEntry `protobuf:"bytes,1,opt,name=dump" json:"dump,omitempty"` + Restore *RestoreStatsEntry `protobuf:"bytes,2,opt,name=restore" json:"restore,omitempty"` +} + +func (x *StatsEntry) Reset() { + *x = StatsEntry{} + if protoimpl.UnsafeEnabled { + mi := &file_stats_stats_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *StatsEntry) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StatsEntry) ProtoMessage() {} + +func (x *StatsEntry) ProtoReflect() protoreflect.Message { + mi := &file_stats_stats_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StatsEntry.ProtoReflect.Descriptor instead. +func (*StatsEntry) Descriptor() ([]byte, []int) { + return file_stats_stats_proto_rawDescGZIP(), []int{2} +} + +func (x *StatsEntry) GetDump() *DumpStatsEntry { + if x != nil { + return x.Dump + } + return nil +} + +func (x *StatsEntry) GetRestore() *RestoreStatsEntry { + if x != nil { + return x.Restore + } + return nil +} + +var File_stats_stats_proto protoreflect.FileDescriptor + +var file_stats_stats_proto_rawDesc = []byte{ + 0x0a, 0x11, 0x73, 0x74, 0x61, 0x74, 0x73, 0x2f, 0x73, 0x74, 0x61, 0x74, 0x73, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x22, 0xad, 0x04, 0x0a, 0x10, 0x64, 0x75, 0x6d, 0x70, 0x5f, 0x73, 0x74, 0x61, + 0x74, 0x73, 0x5f, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x23, 0x0a, 0x0d, 0x66, 0x72, 0x65, 0x65, + 0x7a, 0x69, 0x6e, 0x67, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x02, 0x28, 0x0d, 0x52, + 0x0c, 0x66, 0x72, 0x65, 0x65, 0x7a, 0x69, 0x6e, 0x67, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x1f, 0x0a, + 0x0b, 0x66, 0x72, 0x6f, 0x7a, 0x65, 0x6e, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x02, + 0x28, 0x0d, 0x52, 0x0a, 0x66, 0x72, 0x6f, 0x7a, 0x65, 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x21, + 0x0a, 0x0c, 0x6d, 0x65, 0x6d, 0x64, 0x75, 0x6d, 0x70, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x03, + 0x20, 0x02, 0x28, 0x0d, 0x52, 0x0b, 0x6d, 0x65, 0x6d, 0x64, 0x75, 0x6d, 0x70, 0x54, 0x69, 0x6d, + 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x6d, 0x65, 0x6d, 0x77, 0x72, 0x69, 0x74, 0x65, 0x5f, 0x74, 0x69, + 0x6d, 0x65, 0x18, 0x04, 0x20, 0x02, 0x28, 0x0d, 0x52, 0x0c, 0x6d, 0x65, 0x6d, 0x77, 0x72, 0x69, + 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x70, 0x61, 0x67, 0x65, 0x73, 0x5f, + 0x73, 0x63, 0x61, 0x6e, 0x6e, 0x65, 0x64, 0x18, 0x05, 0x20, 0x02, 0x28, 0x04, 0x52, 0x0c, 0x70, + 0x61, 0x67, 0x65, 0x73, 0x53, 0x63, 0x61, 0x6e, 0x6e, 0x65, 0x64, 0x12, 0x30, 0x0a, 0x14, 0x70, + 0x61, 0x67, 0x65, 0x73, 0x5f, 0x73, 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64, 0x5f, 0x70, 0x61, 0x72, + 0x65, 0x6e, 0x74, 0x18, 0x06, 0x20, 0x02, 0x28, 0x04, 0x52, 0x12, 0x70, 0x61, 0x67, 0x65, 0x73, + 0x53, 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x12, 0x23, 0x0a, + 0x0d, 0x70, 0x61, 0x67, 0x65, 0x73, 0x5f, 0x77, 0x72, 0x69, 0x74, 0x74, 0x65, 0x6e, 0x18, 0x07, + 0x20, 0x02, 0x28, 0x04, 0x52, 0x0c, 0x70, 0x61, 0x67, 0x65, 0x73, 0x57, 0x72, 0x69, 0x74, 0x74, + 0x65, 0x6e, 0x12, 0x23, 0x0a, 0x0d, 0x69, 0x72, 0x6d, 0x61, 0x70, 0x5f, 0x72, 0x65, 0x73, 0x6f, + 0x6c, 0x76, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0c, 0x69, 0x72, 0x6d, 0x61, 0x70, + 0x52, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x70, 0x61, 0x67, 0x65, 0x73, + 0x5f, 0x6c, 0x61, 0x7a, 0x79, 0x18, 0x09, 0x20, 0x02, 0x28, 0x04, 0x52, 0x09, 0x70, 0x61, 0x67, + 0x65, 0x73, 0x4c, 0x61, 0x7a, 0x79, 0x12, 0x1d, 0x0a, 0x0a, 0x70, 0x61, 0x67, 0x65, 0x5f, 0x70, + 0x69, 0x70, 0x65, 0x73, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x04, 0x52, 0x09, 0x70, 0x61, 0x67, 0x65, + 0x50, 0x69, 0x70, 0x65, 0x73, 0x12, 0x24, 0x0a, 0x0e, 0x70, 0x61, 0x67, 0x65, 0x5f, 0x70, 0x69, + 0x70, 0x65, 0x5f, 0x62, 0x75, 0x66, 0x73, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0c, 0x70, + 0x61, 0x67, 0x65, 0x50, 0x69, 0x70, 0x65, 0x42, 0x75, 0x66, 0x73, 0x12, 0x27, 0x0a, 0x0f, 0x73, + 0x68, 0x70, 0x61, 0x67, 0x65, 0x73, 0x5f, 0x73, 0x63, 0x61, 0x6e, 0x6e, 0x65, 0x64, 0x18, 0x0c, + 0x20, 0x01, 0x28, 0x04, 0x52, 0x0e, 0x73, 0x68, 0x70, 0x61, 0x67, 0x65, 0x73, 0x53, 0x63, 0x61, + 0x6e, 0x6e, 0x65, 0x64, 0x12, 0x34, 0x0a, 0x16, 0x73, 0x68, 0x70, 0x61, 0x67, 0x65, 0x73, 0x5f, + 0x73, 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64, 0x5f, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x18, 0x0d, + 0x20, 0x01, 0x28, 0x04, 0x52, 0x14, 0x73, 0x68, 0x70, 0x61, 0x67, 0x65, 0x73, 0x53, 0x6b, 0x69, + 0x70, 0x70, 0x65, 0x64, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x12, 0x27, 0x0a, 0x0f, 0x73, 0x68, + 0x70, 0x61, 0x67, 0x65, 0x73, 0x5f, 0x77, 0x72, 0x69, 0x74, 0x74, 0x65, 0x6e, 0x18, 0x0e, 0x20, + 0x01, 0x28, 0x04, 0x52, 0x0e, 0x73, 0x68, 0x70, 0x61, 0x67, 0x65, 0x73, 0x57, 0x72, 0x69, 0x74, + 0x74, 0x65, 0x6e, 0x22, 0xd5, 0x01, 0x0a, 0x13, 0x72, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x5f, + 0x73, 0x74, 0x61, 0x74, 0x73, 0x5f, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x25, 0x0a, 0x0e, 0x70, + 0x61, 0x67, 0x65, 0x73, 0x5f, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x72, 0x65, 0x64, 0x18, 0x01, 0x20, + 0x02, 0x28, 0x04, 0x52, 0x0d, 0x70, 0x61, 0x67, 0x65, 0x73, 0x43, 0x6f, 0x6d, 0x70, 0x61, 0x72, + 0x65, 0x64, 0x12, 0x2a, 0x0a, 0x11, 0x70, 0x61, 0x67, 0x65, 0x73, 0x5f, 0x73, 0x6b, 0x69, 0x70, + 0x70, 0x65, 0x64, 0x5f, 0x63, 0x6f, 0x77, 0x18, 0x02, 0x20, 0x02, 0x28, 0x04, 0x52, 0x0f, 0x70, + 0x61, 0x67, 0x65, 0x73, 0x53, 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64, 0x43, 0x6f, 0x77, 0x12, 0x21, + 0x0a, 0x0c, 0x66, 0x6f, 0x72, 0x6b, 0x69, 0x6e, 0x67, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x03, + 0x20, 0x02, 0x28, 0x0d, 0x52, 0x0b, 0x66, 0x6f, 0x72, 0x6b, 0x69, 0x6e, 0x67, 0x54, 0x69, 0x6d, + 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x72, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x5f, 0x74, 0x69, 0x6d, + 0x65, 0x18, 0x04, 0x20, 0x02, 0x28, 0x0d, 0x52, 0x0b, 0x72, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, + 0x54, 0x69, 0x6d, 0x65, 0x12, 0x25, 0x0a, 0x0e, 0x70, 0x61, 0x67, 0x65, 0x73, 0x5f, 0x72, 0x65, + 0x73, 0x74, 0x6f, 0x72, 0x65, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0d, 0x70, 0x61, + 0x67, 0x65, 0x73, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x64, 0x22, 0x64, 0x0a, 0x0b, 0x73, + 0x74, 0x61, 0x74, 0x73, 0x5f, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x25, 0x0a, 0x04, 0x64, 0x75, + 0x6d, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x64, 0x75, 0x6d, 0x70, 0x5f, + 0x73, 0x74, 0x61, 0x74, 0x73, 0x5f, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x64, 0x75, 0x6d, + 0x70, 0x12, 0x2e, 0x0a, 0x07, 0x72, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x72, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x5f, 0x73, 0x74, 0x61, + 0x74, 0x73, 0x5f, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x07, 0x72, 0x65, 0x73, 0x74, 0x6f, 0x72, + 0x65, +} + +var ( + file_stats_stats_proto_rawDescOnce sync.Once + file_stats_stats_proto_rawDescData = file_stats_stats_proto_rawDesc +) + +func file_stats_stats_proto_rawDescGZIP() []byte { + file_stats_stats_proto_rawDescOnce.Do(func() { + file_stats_stats_proto_rawDescData = protoimpl.X.CompressGZIP(file_stats_stats_proto_rawDescData) + }) + return file_stats_stats_proto_rawDescData +} + +var file_stats_stats_proto_msgTypes = make([]protoimpl.MessageInfo, 3) +var file_stats_stats_proto_goTypes = []interface{}{ + (*DumpStatsEntry)(nil), // 0: dump_stats_entry + (*RestoreStatsEntry)(nil), // 1: restore_stats_entry + (*StatsEntry)(nil), // 2: stats_entry +} +var file_stats_stats_proto_depIdxs = []int32{ + 0, // 0: stats_entry.dump:type_name -> dump_stats_entry + 1, // 1: stats_entry.restore:type_name -> restore_stats_entry + 2, // [2:2] is the sub-list for method output_type + 2, // [2:2] is the sub-list for method input_type + 2, // [2:2] is the sub-list for extension type_name + 2, // [2:2] is the sub-list for extension extendee + 0, // [0:2] is the sub-list for field type_name +} + +func init() { file_stats_stats_proto_init() } +func file_stats_stats_proto_init() { + if File_stats_stats_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_stats_stats_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*DumpStatsEntry); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_stats_stats_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*RestoreStatsEntry); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_stats_stats_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*StatsEntry); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_stats_stats_proto_rawDesc, + NumEnums: 0, + NumMessages: 3, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_stats_stats_proto_goTypes, + DependencyIndexes: file_stats_stats_proto_depIdxs, + MessageInfos: file_stats_stats_proto_msgTypes, + }.Build() + File_stats_stats_proto = out.File + file_stats_stats_proto_rawDesc = nil + file_stats_stats_proto_goTypes = nil + file_stats_stats_proto_depIdxs = nil +} diff --git a/vendor/github.com/checkpoint-restore/go-criu/v5/stats/stats.proto b/vendor/github.com/checkpoint-restore/go-criu/v5/stats/stats.proto new file mode 100644 index 000000000..64e46181d --- /dev/null +++ b/vendor/github.com/checkpoint-restore/go-criu/v5/stats/stats.proto @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: MIT + +syntax = "proto2"; + +// This one contains statistics about dump/restore process +message dump_stats_entry { + required uint32 freezing_time = 1; + required uint32 frozen_time = 2; + required uint32 memdump_time = 3; + required uint32 memwrite_time = 4; + + required uint64 pages_scanned = 5; + required uint64 pages_skipped_parent = 6; + required uint64 pages_written = 7; + + optional uint32 irmap_resolve = 8; + + required uint64 pages_lazy = 9; + optional uint64 page_pipes = 10; + optional uint64 page_pipe_bufs = 11; + + optional uint64 shpages_scanned = 12; + optional uint64 shpages_skipped_parent = 13; + optional uint64 shpages_written = 14; +} + +message restore_stats_entry { + required uint64 pages_compared = 1; + required uint64 pages_skipped_cow = 2; + + required uint32 forking_time = 3; + required uint32 restore_time = 4; + + optional uint64 pages_restored = 5; +} + +message stats_entry { + optional dump_stats_entry dump = 1; + optional restore_stats_entry restore = 2; +} diff --git a/vendor/github.com/checkpoint-restore/go-criu/v5/stats/types.go b/vendor/github.com/checkpoint-restore/go-criu/v5/stats/types.go new file mode 100644 index 000000000..9044ad976 --- /dev/null +++ b/vendor/github.com/checkpoint-restore/go-criu/v5/stats/types.go @@ -0,0 +1,6 @@ +package stats + +const ( + StatsDump = "stats-dump" + StatsRestore = "stats-restore" +) diff --git a/vendor/github.com/checkpoint-restore/go-criu/v5/stats/utils.go b/vendor/github.com/checkpoint-restore/go-criu/v5/stats/utils.go new file mode 100644 index 000000000..368b2039e --- /dev/null +++ b/vendor/github.com/checkpoint-restore/go-criu/v5/stats/utils.go @@ -0,0 +1,54 @@ +package stats + +import ( + "encoding/binary" + "errors" + "io/ioutil" + "os" + "path/filepath" + + "github.com/checkpoint-restore/go-criu/v5/magic" + "google.golang.org/protobuf/proto" +) + +func readStatisticsFile(imgDir *os.File, fileName string) (*StatsEntry, error) { + buf, err := ioutil.ReadFile(filepath.Join(imgDir.Name(), fileName)) + if err != nil { + return nil, err + } + + if binary.LittleEndian.Uint32(buf[magic.PrimaryMagicOffset:magic.SecondaryMagicOffset]) != magic.ImgServiceMagic { + return nil, errors.New("Primary magic not found") + } + + if binary.LittleEndian.Uint32(buf[magic.SecondaryMagicOffset:magic.SizeOffset]) != magic.StatsMagic { + return nil, errors.New("Secondary magic not found") + } + + payloadSize := binary.LittleEndian.Uint32(buf[magic.SizeOffset:magic.PayloadOffset]) + + st := &StatsEntry{} + if err := proto.Unmarshal(buf[magic.PayloadOffset:magic.PayloadOffset+payloadSize], st); err != nil { + return nil, err + } + + return st, nil +} + +func CriuGetDumpStats(imgDir *os.File) (*DumpStatsEntry, error) { + st, err := readStatisticsFile(imgDir, StatsDump) + if err != nil { + return nil, err + } + + return st.GetDump(), nil +} + +func CriuGetRestoreStats(imgDir *os.File) (*RestoreStatsEntry, error) { + st, err := readStatisticsFile(imgDir, StatsRestore) + if err != nil { + return nil, err + } + + return st.GetRestore(), nil +} diff --git a/vendor/google.golang.org/protobuf/encoding/prototext/decode.go b/vendor/google.golang.org/protobuf/encoding/prototext/decode.go index 8fb1d9e08..179d6e8fc 100644 --- a/vendor/google.golang.org/protobuf/encoding/prototext/decode.go +++ b/vendor/google.golang.org/protobuf/encoding/prototext/decode.go @@ -744,9 +744,6 @@ func (d decoder) skipValue() error { // Skip items. This will not validate whether skipped values are // of the same type or not, same behavior as C++ // TextFormat::Parser::AllowUnknownField(true) version 3.8.0. - if err := d.skipValue(); err != nil { - return err - } } } } diff --git a/vendor/google.golang.org/protobuf/internal/encoding/text/encode.go b/vendor/google.golang.org/protobuf/internal/encoding/text/encode.go index aa66bdd06..da289ccce 100644 --- a/vendor/google.golang.org/protobuf/internal/encoding/text/encode.go +++ b/vendor/google.golang.org/protobuf/internal/encoding/text/encode.go @@ -263,3 +263,8 @@ func (e *Encoder) Snapshot() encoderState { func (e *Encoder) Reset(es encoderState) { e.encoderState = es } + +// AppendString appends the escaped form of the input string to b. +func AppendString(b []byte, s string) []byte { + return appendString(b, s, false) +} diff --git a/vendor/google.golang.org/protobuf/internal/impl/legacy_message.go b/vendor/google.golang.org/protobuf/internal/impl/legacy_message.go index 3759b010c..029feeefd 100644 --- a/vendor/google.golang.org/protobuf/internal/impl/legacy_message.go +++ b/vendor/google.golang.org/protobuf/internal/impl/legacy_message.go @@ -440,6 +440,13 @@ func legacyMerge(in piface.MergeInput) piface.MergeOutput { if !ok { return piface.MergeOutput{} } + if !in.Source.IsValid() { + // Legacy Marshal methods may not function on nil messages. + // Check for a typed nil source only after we confirm that + // legacy Marshal/Unmarshal methods are present, for + // consistency. + return piface.MergeOutput{Flags: piface.MergeComplete} + } b, err := marshaler.Marshal() if err != nil { return piface.MergeOutput{} diff --git a/vendor/google.golang.org/protobuf/internal/version/version.go b/vendor/google.golang.org/protobuf/internal/version/version.go index 5879131da..14e774fb2 100644 --- a/vendor/google.golang.org/protobuf/internal/version/version.go +++ b/vendor/google.golang.org/protobuf/internal/version/version.go @@ -52,8 +52,8 @@ import ( // 10. Send out the CL for review and submit it. const ( Major = 1 - Minor = 26 - Patch = 0 + Minor = 27 + Patch = 1 PreRelease = "" ) diff --git a/vendor/google.golang.org/protobuf/reflect/protoregistry/registry.go b/vendor/google.golang.org/protobuf/reflect/protoregistry/registry.go index 66dcbcd0d..59f024c44 100644 --- a/vendor/google.golang.org/protobuf/reflect/protoregistry/registry.go +++ b/vendor/google.golang.org/protobuf/reflect/protoregistry/registry.go @@ -94,7 +94,8 @@ type Files struct { // Note that enum values are in the top-level since that are in the same // scope as the parent enum. descsByName map[protoreflect.FullName]interface{} - filesByPath map[string]protoreflect.FileDescriptor + filesByPath map[string][]protoreflect.FileDescriptor + numFiles int } type packageDescriptor struct { @@ -117,17 +118,16 @@ func (r *Files) RegisterFile(file protoreflect.FileDescriptor) error { r.descsByName = map[protoreflect.FullName]interface{}{ "": &packageDescriptor{}, } - r.filesByPath = make(map[string]protoreflect.FileDescriptor) + r.filesByPath = make(map[string][]protoreflect.FileDescriptor) } path := file.Path() - if prev := r.filesByPath[path]; prev != nil { + if prev := r.filesByPath[path]; len(prev) > 0 { r.checkGenProtoConflict(path) err := errors.New("file %q is already registered", file.Path()) - err = amendErrorWithCaller(err, prev, file) - if r == GlobalFiles && ignoreConflict(file, err) { - err = nil + err = amendErrorWithCaller(err, prev[0], file) + if !(r == GlobalFiles && ignoreConflict(file, err)) { + return err } - return err } for name := file.Package(); name != ""; name = name.Parent() { @@ -168,7 +168,8 @@ func (r *Files) RegisterFile(file protoreflect.FileDescriptor) error { rangeTopLevelDescriptors(file, func(d protoreflect.Descriptor) { r.descsByName[d.FullName()] = d }) - r.filesByPath[path] = file + r.filesByPath[path] = append(r.filesByPath[path], file) + r.numFiles++ return nil } @@ -308,6 +309,7 @@ func (s *nameSuffix) Pop() (name protoreflect.Name) { // FindFileByPath looks up a file by the path. // // This returns (nil, NotFound) if not found. +// This returns an error if multiple files have the same path. func (r *Files) FindFileByPath(path string) (protoreflect.FileDescriptor, error) { if r == nil { return nil, NotFound @@ -316,13 +318,19 @@ func (r *Files) FindFileByPath(path string) (protoreflect.FileDescriptor, error) globalMutex.RLock() defer globalMutex.RUnlock() } - if fd, ok := r.filesByPath[path]; ok { - return fd, nil + fds := r.filesByPath[path] + switch len(fds) { + case 0: + return nil, NotFound + case 1: + return fds[0], nil + default: + return nil, errors.New("multiple files named %q", path) } - return nil, NotFound } -// NumFiles reports the number of registered files. +// NumFiles reports the number of registered files, +// including duplicate files with the same name. func (r *Files) NumFiles() int { if r == nil { return 0 @@ -331,10 +339,11 @@ func (r *Files) NumFiles() int { globalMutex.RLock() defer globalMutex.RUnlock() } - return len(r.filesByPath) + return r.numFiles } // RangeFiles iterates over all registered files while f returns true. +// If multiple files have the same name, RangeFiles iterates over all of them. // The iteration order is undefined. func (r *Files) RangeFiles(f func(protoreflect.FileDescriptor) bool) { if r == nil { @@ -344,9 +353,11 @@ func (r *Files) RangeFiles(f func(protoreflect.FileDescriptor) bool) { globalMutex.RLock() defer globalMutex.RUnlock() } - for _, file := range r.filesByPath { - if !f(file) { - return + for _, files := range r.filesByPath { + for _, file := range files { + if !f(file) { + return + } } } } diff --git a/vendor/google.golang.org/protobuf/types/descriptorpb/descriptor.pb.go b/vendor/google.golang.org/protobuf/types/descriptorpb/descriptor.pb.go index f77239fc3..abe4ab511 100644 --- a/vendor/google.golang.org/protobuf/types/descriptorpb/descriptor.pb.go +++ b/vendor/google.golang.org/protobuf/types/descriptorpb/descriptor.pb.go @@ -43,7 +43,6 @@ package descriptorpb import ( protoreflect "google.golang.org/protobuf/reflect/protoreflect" - protoiface "google.golang.org/protobuf/runtime/protoiface" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" sync "sync" @@ -829,15 +828,6 @@ func (*ExtensionRangeOptions) Descriptor() ([]byte, []int) { return file_google_protobuf_descriptor_proto_rawDescGZIP(), []int{3} } -var extRange_ExtensionRangeOptions = []protoiface.ExtensionRangeV1{ - {Start: 1000, End: 536870911}, -} - -// Deprecated: Use ExtensionRangeOptions.ProtoReflect.Descriptor.ExtensionRanges instead. -func (*ExtensionRangeOptions) ExtensionRangeArray() []protoiface.ExtensionRangeV1 { - return extRange_ExtensionRangeOptions -} - func (x *ExtensionRangeOptions) GetUninterpretedOption() []*UninterpretedOption { if x != nil { return x.UninterpretedOption @@ -1520,15 +1510,6 @@ func (*FileOptions) Descriptor() ([]byte, []int) { return file_google_protobuf_descriptor_proto_rawDescGZIP(), []int{10} } -var extRange_FileOptions = []protoiface.ExtensionRangeV1{ - {Start: 1000, End: 536870911}, -} - -// Deprecated: Use FileOptions.ProtoReflect.Descriptor.ExtensionRanges instead. -func (*FileOptions) ExtensionRangeArray() []protoiface.ExtensionRangeV1 { - return extRange_FileOptions -} - func (x *FileOptions) GetJavaPackage() string { if x != nil && x.JavaPackage != nil { return *x.JavaPackage @@ -1776,15 +1757,6 @@ func (*MessageOptions) Descriptor() ([]byte, []int) { return file_google_protobuf_descriptor_proto_rawDescGZIP(), []int{11} } -var extRange_MessageOptions = []protoiface.ExtensionRangeV1{ - {Start: 1000, End: 536870911}, -} - -// Deprecated: Use MessageOptions.ProtoReflect.Descriptor.ExtensionRanges instead. -func (*MessageOptions) ExtensionRangeArray() []protoiface.ExtensionRangeV1 { - return extRange_MessageOptions -} - func (x *MessageOptions) GetMessageSetWireFormat() bool { if x != nil && x.MessageSetWireFormat != nil { return *x.MessageSetWireFormat @@ -1930,15 +1902,6 @@ func (*FieldOptions) Descriptor() ([]byte, []int) { return file_google_protobuf_descriptor_proto_rawDescGZIP(), []int{12} } -var extRange_FieldOptions = []protoiface.ExtensionRangeV1{ - {Start: 1000, End: 536870911}, -} - -// Deprecated: Use FieldOptions.ProtoReflect.Descriptor.ExtensionRanges instead. -func (*FieldOptions) ExtensionRangeArray() []protoiface.ExtensionRangeV1 { - return extRange_FieldOptions -} - func (x *FieldOptions) GetCtype() FieldOptions_CType { if x != nil && x.Ctype != nil { return *x.Ctype @@ -2030,15 +1993,6 @@ func (*OneofOptions) Descriptor() ([]byte, []int) { return file_google_protobuf_descriptor_proto_rawDescGZIP(), []int{13} } -var extRange_OneofOptions = []protoiface.ExtensionRangeV1{ - {Start: 1000, End: 536870911}, -} - -// Deprecated: Use OneofOptions.ProtoReflect.Descriptor.ExtensionRanges instead. -func (*OneofOptions) ExtensionRangeArray() []protoiface.ExtensionRangeV1 { - return extRange_OneofOptions -} - func (x *OneofOptions) GetUninterpretedOption() []*UninterpretedOption { if x != nil { return x.UninterpretedOption @@ -2101,15 +2055,6 @@ func (*EnumOptions) Descriptor() ([]byte, []int) { return file_google_protobuf_descriptor_proto_rawDescGZIP(), []int{14} } -var extRange_EnumOptions = []protoiface.ExtensionRangeV1{ - {Start: 1000, End: 536870911}, -} - -// Deprecated: Use EnumOptions.ProtoReflect.Descriptor.ExtensionRanges instead. -func (*EnumOptions) ExtensionRangeArray() []protoiface.ExtensionRangeV1 { - return extRange_EnumOptions -} - func (x *EnumOptions) GetAllowAlias() bool { if x != nil && x.AllowAlias != nil { return *x.AllowAlias @@ -2183,15 +2128,6 @@ func (*EnumValueOptions) Descriptor() ([]byte, []int) { return file_google_protobuf_descriptor_proto_rawDescGZIP(), []int{15} } -var extRange_EnumValueOptions = []protoiface.ExtensionRangeV1{ - {Start: 1000, End: 536870911}, -} - -// Deprecated: Use EnumValueOptions.ProtoReflect.Descriptor.ExtensionRanges instead. -func (*EnumValueOptions) ExtensionRangeArray() []protoiface.ExtensionRangeV1 { - return extRange_EnumValueOptions -} - func (x *EnumValueOptions) GetDeprecated() bool { if x != nil && x.Deprecated != nil { return *x.Deprecated @@ -2258,15 +2194,6 @@ func (*ServiceOptions) Descriptor() ([]byte, []int) { return file_google_protobuf_descriptor_proto_rawDescGZIP(), []int{16} } -var extRange_ServiceOptions = []protoiface.ExtensionRangeV1{ - {Start: 1000, End: 536870911}, -} - -// Deprecated: Use ServiceOptions.ProtoReflect.Descriptor.ExtensionRanges instead. -func (*ServiceOptions) ExtensionRangeArray() []protoiface.ExtensionRangeV1 { - return extRange_ServiceOptions -} - func (x *ServiceOptions) GetDeprecated() bool { if x != nil && x.Deprecated != nil { return *x.Deprecated @@ -2335,15 +2262,6 @@ func (*MethodOptions) Descriptor() ([]byte, []int) { return file_google_protobuf_descriptor_proto_rawDescGZIP(), []int{17} } -var extRange_MethodOptions = []protoiface.ExtensionRangeV1{ - {Start: 1000, End: 536870911}, -} - -// Deprecated: Use MethodOptions.ProtoReflect.Descriptor.ExtensionRanges instead. -func (*MethodOptions) ExtensionRangeArray() []protoiface.ExtensionRangeV1 { - return extRange_MethodOptions -} - func (x *MethodOptions) GetDeprecated() bool { if x != nil && x.Deprecated != nil { return *x.Deprecated diff --git a/vendor/modules.txt b/vendor/modules.txt index 3fd9cbcc8..10fb28eef 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -45,9 +45,11 @@ github.com/buger/goterm github.com/cespare/xxhash/v2 # github.com/checkpoint-restore/checkpointctl v0.0.0-20210922093614-c31748bec9f2 github.com/checkpoint-restore/checkpointctl/lib -# github.com/checkpoint-restore/go-criu/v5 v5.1.0 +# github.com/checkpoint-restore/go-criu/v5 v5.2.0 github.com/checkpoint-restore/go-criu/v5 +github.com/checkpoint-restore/go-criu/v5/magic github.com/checkpoint-restore/go-criu/v5/rpc +github.com/checkpoint-restore/go-criu/v5/stats # github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e github.com/chzyer/readline # github.com/container-orchestrated-devices/container-device-interface v0.0.0-20210325223243-f99e8b6c10b9 @@ -754,7 +756,7 @@ google.golang.org/grpc/serviceconfig google.golang.org/grpc/stats google.golang.org/grpc/status google.golang.org/grpc/tap -# google.golang.org/protobuf v1.26.0 +# google.golang.org/protobuf v1.27.1 google.golang.org/protobuf/encoding/prototext google.golang.org/protobuf/encoding/protowire google.golang.org/protobuf/internal/descfmt |