implement pod stats

Implement pod stats for the local and remote client. Both code paths end
up in infra/abi to allow for code share.

Signed-off-by: Valentin Rothberg <rothberg@redhat.com>

12 files changed, 432 insertions, 8 deletions

diff --git a/cmd/podman/pods/stats.go b/cmd/podman/pods/stats.go
new file mode 100644
index 000000000..7c3597d9a
--- /dev/null
+++ b/cmd/podman/pods/stats.go
@@ -0,0 +1,189 @@
+package pods
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"reflect"
+	"strings"
+	"text/tabwriter"
+	"text/template"
+	"time"
+
+	"github.com/buger/goterm"
+	"github.com/containers/buildah/pkg/formats"
+	"github.com/containers/libpod/cmd/podman/registry"
+	"github.com/containers/libpod/pkg/domain/entities"
+	"github.com/containers/libpod/pkg/util/camelcase"
+	"github.com/spf13/cobra"
+)
+
+type podStatsOptionsWrapper struct {
+	entities.PodStatsOptions
+
+	// Format - pretty-print to JSON or a go template.
+	Format string
+	// NoReset - do not reset the screen when streaming.
+	NoReset bool
+	// NoStream - do not stream stats but write them once.
+	NoStream bool
+}
+
+var (
+	statsOptions     = podStatsOptionsWrapper{}
+	statsDescription = `Display the containers' resource-usage statistics of one or more running pod`
+	// Command: podman pod _pod_
+	statsCmd = &cobra.Command{
+		Use:   "stats [flags] [POD...]",
+		Short: "Display resource-usage statistics of pods",
+		Long:  statsDescription,
+		RunE:  stats,
+		Example: `podman pod stats
+  podman pod stats a69b23034235 named-pod
+  podman pod stats --latest
+  podman pod stats --all`,
+	}
+)
+
+func init() {
+	registry.Commands = append(registry.Commands, registry.CliCommand{
+		Mode:    []entities.EngineMode{entities.ABIMode, entities.TunnelMode},
+		Command: statsCmd,
+		Parent:  podCmd,
+	})
+
+	flags := statsCmd.Flags()
+	flags.BoolVarP(&statsOptions.All, "all", "a", false, "Provide stats for all pods")
+	flags.StringVar(&statsOptions.Format, "format", "", "Pretty-print container statistics to JSON or using a Go template")
+	flags.BoolVarP(&statsOptions.Latest, "latest", "l", false, "Provide stats on the latest pod Podman is aware of")
+	flags.BoolVar(&statsOptions.NoReset, "no-reset", false, "Disable resetting the screen when streaming")
+	flags.BoolVar(&statsOptions.NoStream, "no-stream", false, "Disable streaming stats and only pull the first result")
+
+	if registry.IsRemote() {
+		_ = flags.MarkHidden("latest")
+	}
+}
+
+func stats(cmd *cobra.Command, args []string) error {
+	// Validate input.
+	if err := entities.ValidatePodStatsOptions(args, &statsOptions.PodStatsOptions); err != nil {
+		return err
+	}
+
+	format := statsOptions.Format
+	doJson := strings.ToLower(format) == formats.JSONString
+	header := getPodStatsHeader(format)
+
+	for {
+		reports, err := registry.ContainerEngine().PodStats(context.Background(), args, statsOptions.PodStatsOptions)
+		if err != nil {
+			return err
+		}
+		// Print the stats in the requested format and configuration.
+		if doJson {
+			if err := printJSONPodStats(reports); err != nil {
+				return err
+			}
+		} else {
+			if !statsOptions.NoReset {
+				goterm.Clear()
+				goterm.MoveCursor(1, 1)
+				goterm.Flush()
+			}
+			if len(format) == 0 {
+				printPodStatsLines(reports)
+			} else if err := printFormattedPodStatsLines(format, reports, header); err != nil {
+				return err
+			}
+		}
+		if statsOptions.NoStream {
+			break
+		}
+		time.Sleep(time.Second)
+	}
+
+	return nil
+}
+
+func printJSONPodStats(stats []*entities.PodStatsReport) error {
+	b, err := json.MarshalIndent(&stats, "", "     ")
+	if err != nil {
+		return err
+	}
+	fmt.Fprintf(os.Stdout, "%s\n", string(b))
+	return nil
+}
+
+func printPodStatsLines(stats []*entities.PodStatsReport) {
+	w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
+	outFormat := "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n"
+	fmt.Fprintf(w, outFormat, "POD", "CID", "NAME", "CPU %", "MEM USAGE/ LIMIT", "MEM %", "NET IO", "BLOCK IO", "PIDS")
+	for _, i := range stats {
+		if len(stats) == 0 {
+			fmt.Fprintf(w, outFormat, i.Pod, "--", "--", "--", "--", "--", "--", "--", "--")
+		} else {
+			fmt.Fprintf(w, outFormat, i.Pod, i.CID, i.Name, i.CPU, i.MemUsage, i.Mem, i.NetIO, i.BlockIO, i.PIDS)
+		}
+	}
+	w.Flush()
+}
+
+func printFormattedPodStatsLines(format string, stats []*entities.PodStatsReport, headerNames map[string]string) error {
+	if len(stats) == 0 {
+		return nil
+	}
+
+	// Use a tabwriter to align column format
+	w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0)
+	// Spit out the header if "table" is present in the format
+	if strings.HasPrefix(format, "table") {
+		hformat := strings.Replace(strings.TrimSpace(format[5:]), " ", "\t", -1)
+		format = hformat
+		headerTmpl, err := template.New("header").Parse(hformat)
+		if err != nil {
+			return err
+		}
+		if err := headerTmpl.Execute(w, headerNames); err != nil {
+			return err
+		}
+		fmt.Fprintln(w, "")
+	}
+
+	// Spit out the data rows now
+	dataTmpl, err := template.New("data").Parse(format)
+	if err != nil {
+		return err
+	}
+	for _, s := range stats {
+		if err := dataTmpl.Execute(w, s); err != nil {
+			return err
+		}
+		fmt.Fprintln(w, "")
+	}
+	// Flush the writer
+	return w.Flush()
+
+}
+
+// getPodStatsHeader returns the stats header for the specified options.
+func getPodStatsHeader(format string) map[string]string {
+	headerNames := make(map[string]string)
+	if format == "" {
+		return headerNames
+	}
+	// Make a map of the field names for the headers
+	v := reflect.ValueOf(entities.PodStatsReport{})
+	t := v.Type()
+	for i := 0; i < t.NumField(); i++ {
+		split := camelcase.Split(t.Field(i).Name)
+		value := strings.ToUpper(strings.Join(split, " "))
+		switch value {
+		case "CPU", "MEM":
+			value += " %"
+		case "MEM USAGE":
+			value = "MEM USAGE / LIMIT"
+		}
+		headerNames[t.Field(i).Name] = value
+	}
+	return headerNames
+}
diff --git a/docs/source/markdown/podman-pod-stats.1.md b/docs/source/markdown/podman-pod-stats.1.md
index 962edbda0..f70a5a919 100644
--- a/docs/source/markdown/podman-pod-stats.1.md
+++ b/docs/source/markdown/podman-pod-stats.1.md
@@ -7,7 +7,7 @@ podman\-pod\-stats - Display a live stream of resource usage stats for container
 **podman pod stats** [*options*] [*pod*]
 
 ## DESCRIPTION
-Display a live stream of containers in one or more pods resource usage statistics
+Display a live stream of containers in one or more pods resource usage statistics.  Running rootless is only supported on cgroups v2.
 
 ## OPTIONS
 
diff --git a/pkg/api/handlers/libpod/pods.go b/pkg/api/handlers/libpod/pods.go
index 618d48ac0..0b15ab0d6 100644
--- a/pkg/api/handlers/libpod/pods.go
+++ b/pkg/api/handlers/libpod/pods.go
@@ -11,6 +11,7 @@ import (
 	"github.com/containers/libpod/pkg/api/handlers"
 	"github.com/containers/libpod/pkg/api/handlers/utils"
 	"github.com/containers/libpod/pkg/domain/entities"
+	"github.com/containers/libpod/pkg/domain/infra/abi"
 	"github.com/containers/libpod/pkg/specgen"
 	"github.com/containers/libpod/pkg/specgen/generate"
 	"github.com/containers/libpod/pkg/util"
@@ -419,3 +420,44 @@ func PodExists(w http.ResponseWriter, r *http.Request) {
 	}
 	utils.WriteResponse(w, http.StatusNoContent, "")
 }
+
+func PodStats(w http.ResponseWriter, r *http.Request) {
+	runtime := r.Context().Value("runtime").(*libpod.Runtime)
+	decoder := r.Context().Value("decoder").(*schema.Decoder)
+
+	query := struct {
+		NamesOrIDs []string `schema:"namesOrIDs"`
+		All        bool     `schema:"all"`
+	}{
+		// default would go here
+	}
+	if err := decoder.Decode(&query, r.URL.Query()); err != nil {
+		utils.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest,
+			errors.Wrapf(err, "Failed to parse parameters for %s", r.URL.String()))
+		return
+	}
+
+	// Validate input.
+	options := entities.PodStatsOptions{All: query.All}
+	if err := entities.ValidatePodStatsOptions(query.NamesOrIDs, &options); err != nil {
+		utils.InternalServerError(w, err)
+	}
+
+	// Collect the stats and send them over the wire.
+	containerEngine := abi.ContainerEngine{Libpod: runtime}
+	reports, err := containerEngine.PodStats(r.Context(), query.NamesOrIDs, options)
+
+	// Error checks as documented in swagger.
+	switch errors.Cause(err) {
+	case define.ErrNoSuchPod:
+		utils.Error(w, "one or more pods not found", http.StatusNotFound, err)
+		return
+	case nil:
+		// Nothing to do.
+	default:
+		utils.InternalServerError(w, err)
+		return
+	}
+
+	utils.WriteResponse(w, http.StatusOK, reports)
+}
diff --git a/pkg/api/handlers/swagger/swagger.go b/pkg/api/handlers/swagger/swagger.go
index 87891d4a8..d9ea55b3e 100644
--- a/pkg/api/handlers/swagger/swagger.go
+++ b/pkg/api/handlers/swagger/swagger.go
@@ -122,6 +122,13 @@ type swagPodTopResponse struct {
 	}
 }
 
+// List processes in pod
+// swagger:response DocsPodStatsResponse
+type swagPodStatsResponse struct {
+	// in:body
+	Body []*entities.PodStatsReport
+}
+
 // Inspect container
 // swagger:response LibpodInspectContainerResponse
 type swagLibpodInspectContainerResponse struct {
diff --git a/pkg/api/handlers/utils/errors.go b/pkg/api/handlers/utils/errors.go
index aafc64353..3253a9be3 100644
--- a/pkg/api/handlers/utils/errors.go
+++ b/pkg/api/handlers/utils/errors.go
@@ -14,6 +14,9 @@ var (
 	ErrLinkNotSupport = errors.New("Link is not supported")
 )
 
+// TODO: document the exported functions in this file and make them more
+// generic (e.g., not tied to one ctr/pod).
+
 // Error formats an API response to an error
 //
 // apiMessage and code must match the container API, and are sent to client
diff --git a/pkg/api/server/register_pods.go b/pkg/api/server/register_pods.go
index 63060af41..4156dd86b 100644
--- a/pkg/api/server/register_pods.go
+++ b/pkg/api/server/register_pods.go
@@ -286,9 +286,36 @@ func (s *APIServer) registerPodsHandlers(r *mux.Router) error {
 	//   200:
 	//     $ref: "#/responses/DocsPodTopResponse"
 	//   404:
-	//     $ref: "#/responses/NoSuchContainer"
+	//     $ref: "#/responses/NoSuchPod"
 	//   500:
 	//     $ref: "#/responses/InternalError"
 	r.Handle(VersionedPath("/libpod/pods/{name}/top"), s.APIHandler(libpod.PodTop)).Methods(http.MethodGet)
+	// swagger:operation GET /libpod/pods/stats pods statsPod
+	// ---
+	// tags:
+	//  - pods
+	// summary: Get stats for one or more pods
+	// description: Display a live stream of resource usage statistics for the containers in one or more pods
+	// parameters:
+	//  - in: query
+	//    name: all
+	//    description: Provide statistics for all running pods.
+	//    type: boolean
+	//  - in: query
+	//    name: namesOrIDs
+	//    description: Names or IDs of pods.
+	//    type: array
+	//    items:
+	//      type: string
+	// produces:
+	// - application/json
+	// responses:
+	//   200:
+	//     $ref: "#/responses/DocsPodTopResponse"
+	//   404:
+	//     $ref: "#/responses/NoSuchPod"
+	//   500:
+	//     $ref: "#/responses/InternalError"
+	r.Handle(VersionedPath("/libpod/pods/stats"), s.APIHandler(libpod.PodStats)).Methods(http.MethodGet)
 	return nil
 }
diff --git a/pkg/bindings/pods/pods.go b/pkg/bindings/pods/pods.go
index 3c60fa2a0..b213c8c73 100644
--- a/pkg/bindings/pods/pods.go
+++ b/pkg/bindings/pods/pods.go
@@ -2,6 +2,7 @@ package pods
 
 import (
 	"context"
+	"errors"
 	"net/http"
 	"net/url"
 	"strconv"
@@ -189,11 +190,6 @@ func Start(ctx context.Context, nameOrID string) (*entities.PodStartReport, erro
 	return &report, response.Process(&report)
 }
 
-func Stats() error {
-	// TODO
-	return bindings.ErrNotImplemented
-}
-
 // Stop stops all containers in a Pod. The optional timeout parameter can be
 // used to override the timeout before the container is killed.
 func Stop(ctx context.Context, nameOrID string, timeout *int) (*entities.PodStopReport, error) {
@@ -264,3 +260,26 @@ func Unpause(ctx context.Context, nameOrID string) (*entities.PodUnpauseReport,
 	}
 	return &report, response.Process(&report)
 }
+
+// Stats display resource-usage statistics of one or more pods.
+func Stats(ctx context.Context, namesOrIDs []string, options entities.PodStatsOptions) ([]*entities.PodStatsReport, error) {
+	if options.Latest {
+		return nil, errors.New("latest is not supported")
+	}
+	conn, err := bindings.GetClient(ctx)
+	if err != nil {
+		return nil, err
+	}
+	params := url.Values{}
+	for _, i := range namesOrIDs {
+		params.Add("namesOrIDs", i)
+	}
+	params.Set("all", strconv.FormatBool(options.All))
+
+	var reports []*entities.PodStatsReport
+	response, err := conn.DoRequest(nil, http.MethodGet, "/pods/stats", params)
+	if err != nil {
+		return nil, err
+	}
+	return reports, response.Process(&reports)
+}
diff --git a/pkg/domain/entities/engine_container.go b/pkg/domain/entities/engine_container.go
index 8c5bc3058..502279bcf 100644
--- a/pkg/domain/entities/engine_container.go
+++ b/pkg/domain/entities/engine_container.go
@@ -53,6 +53,7 @@ type ContainerEngine interface {
 	PodRestart(ctx context.Context, namesOrIds []string, options PodRestartOptions) ([]*PodRestartReport, error)
 	PodRm(ctx context.Context, namesOrIds []string, options PodRmOptions) ([]*PodRmReport, error)
 	PodStart(ctx context.Context, namesOrIds []string, options PodStartOptions) ([]*PodStartReport, error)
+	PodStats(ctx context.Context, namesOrIds []string, options PodStatsOptions) ([]*PodStatsReport, error)
 	PodStop(ctx context.Context, namesOrIds []string, options PodStopOptions) ([]*PodStopReport, error)
 	PodTop(ctx context.Context, options PodTopOptions) (*StringSliceReport, error)
 	PodUnpause(ctx context.Context, namesOrIds []string, options PodunpauseOptions) ([]*PodUnpauseReport, error)
diff --git a/pkg/domain/entities/pods.go b/pkg/domain/entities/pods.go
index aa1445a6a..a4896ce4d 100644
--- a/pkg/domain/entities/pods.go
+++ b/pkg/domain/entities/pods.go
@@ -1,6 +1,7 @@
 package entities
 
 import (
+	"errors"
 	"strings"
 	"time"
 
@@ -188,3 +189,50 @@ type PodInspectOptions struct {
 type PodInspectReport struct {
 	*define.InspectPodData
 }
+
+// PodStatsOptions are options for the pod stats command.
+type PodStatsOptions struct {
+	// All - provide stats for all running pods.
+	All bool
+	// Latest - provide stats for the latest pod.
+	Latest bool
+}
+
+// PodStatsReport includes pod-resource statistics data.
+type PodStatsReport struct {
+	CPU      string
+	MemUsage string
+	Mem      string
+	NetIO    string
+	BlockIO  string
+	PIDS     string
+	Pod      string
+	CID      string
+	Name     string
+}
+
+// ValidatePodStatsOptions validates the specified slice and options. Allows
+// for sharing code in the front- and the back-end.
+func ValidatePodStatsOptions(args []string, options *PodStatsOptions) error {
+	num := 0
+	if len(args) > 0 {
+		num++
+	}
+	if options.All {
+		num++
+	}
+	if options.Latest {
+		num++
+	}
+	switch num {
+	case 0:
+		// Podman v1 compat: if nothing's specified get all running
+		// pods.
+		options.All = true
+		return nil
+	case 1:
+		return nil
+	default:
+		return errors.New("--all, --latest and arguments cannot be used together")
+	}
+}
diff --git a/pkg/domain/infra/abi/pods_stats.go b/pkg/domain/infra/abi/pods_stats.go
new file mode 100644
index 000000000..a41c01da0
--- /dev/null
+++ b/pkg/domain/infra/abi/pods_stats.go
@@ -0,0 +1,85 @@
+package abi
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/containers/libpod/libpod"
+	"github.com/containers/libpod/pkg/cgroups"
+	"github.com/containers/libpod/pkg/domain/entities"
+	"github.com/containers/libpod/pkg/rootless"
+	"github.com/docker/go-units"
+	"github.com/pkg/errors"
+)
+
+// PodStats implements printing stats about pods.
+func (ic *ContainerEngine) PodStats(ctx context.Context, namesOrIds []string, options entities.PodStatsOptions) ([]*entities.PodStatsReport, error) {
+	// Cgroups v2 check for rootless.
+	if rootless.IsRootless() {
+		unified, err := cgroups.IsCgroup2UnifiedMode()
+		if err != nil {
+			return nil, err
+		}
+		if !unified {
+			return nil, errors.New("pod stats is not supported in rootless mode without cgroups v2")
+		}
+	}
+	// Get the (running) pods and convert them to the entities format.
+	pods, err := getPodsByContext(options.All, options.Latest, namesOrIds, ic.Libpod)
+	if err != nil {
+		return nil, errors.Wrap(err, "unable to get list of pods")
+	}
+	return ic.podsToStatsReport(pods)
+}
+
+// podsToStatsReport converts a slice of pods into a corresponding slice of stats reports.
+func (ic *ContainerEngine) podsToStatsReport(pods []*libpod.Pod) ([]*entities.PodStatsReport, error) {
+	reports := []*entities.PodStatsReport{}
+	for i := range pods { // Access by index to prevent potential loop-variable leaks.
+		podStats, err := pods[i].GetPodStats(nil)
+		if err != nil {
+			return nil, err
+		}
+		podID := pods[i].ID()[:12]
+		for j := range podStats {
+			r := entities.PodStatsReport{
+				CPU:      floatToPercentString(podStats[j].CPU),
+				MemUsage: combineHumanValues(podStats[j].MemUsage, podStats[j].MemLimit),
+				Mem:      floatToPercentString(podStats[j].MemPerc),
+				NetIO:    combineHumanValues(podStats[j].NetInput, podStats[j].NetOutput),
+				BlockIO:  combineHumanValues(podStats[j].BlockInput, podStats[j].BlockOutput),
+				PIDS:     pidsToString(podStats[j].PIDs),
+				CID:      podStats[j].ContainerID[:12],
+				Name:     podStats[j].Name,
+				Pod:      podID,
+			}
+			reports = append(reports, &r)
+		}
+	}
+
+	return reports, nil
+}
+
+func combineHumanValues(a, b uint64) string {
+	if a == 0 && b == 0 {
+		return "-- / --"
+	}
+	return fmt.Sprintf("%s / %s", units.HumanSize(float64(a)), units.HumanSize(float64(b)))
+}
+
+func floatToPercentString(f float64) string {
+	strippedFloat, err := libpod.RemoveScientificNotationFromFloat(f)
+	if err != nil || strippedFloat == 0 {
+		// If things go bazinga, return a safe value
+		return "--"
+	}
+	return fmt.Sprintf("%.2f", strippedFloat) + "%"
+}
+
+func pidsToString(pid uint64) string {
+	if pid == 0 {
+		// If things go bazinga, return a safe value
+		return "--"
+	}
+	return fmt.Sprintf("%d", pid)
+}
diff --git a/pkg/domain/infra/tunnel/pods.go b/pkg/domain/infra/tunnel/pods.go
index e7641c077..c193c6752 100644
--- a/pkg/domain/infra/tunnel/pods.go
+++ b/pkg/domain/infra/tunnel/pods.go
@@ -211,3 +211,7 @@ func (ic *ContainerEngine) PodInspect(ctx context.Context, options entities.PodI
 	}
 	return pods.Inspect(ic.ClientCxt, options.NameOrID)
 }
+
+func (ic *ContainerEngine) PodStats(ctx context.Context, namesOrIds []string, options entities.PodStatsOptions) ([]*entities.PodStatsReport, error) {
+	return pods.Stats(ic.ClientCxt, namesOrIds, options)
+}
diff --git a/test/e2e/pod_stats_test.go b/test/e2e/pod_stats_test.go
index bb3610a27..347f33e62 100644
--- a/test/e2e/pod_stats_test.go
+++ b/test/e2e/pod_stats_test.go
@@ -18,7 +18,6 @@ var _ = Describe("Podman pod stats", func() {
 	)
 
 	BeforeEach(func() {
-		Skip(v2fail)
 		cgroupsv2, err := cgroups.IsCgroup2UnifiedMode()
 		Expect(err).To(BeNil())