kpod stats

Move kpod stats to the libpod backend.

Signed-off-by: baude <bbaude@redhat.com>

Closes: #113
Approved by: baude

9 files changed, 364 insertions, 208 deletions

diff --git a/cmd/kpod/stats.go b/cmd/kpod/stats.go
index 37cd9090b..d98c2ee27 100644
--- a/cmd/kpod/stats.go
+++ b/cmd/kpod/stats.go
@@ -1,34 +1,28 @@
 package main
 
 import (
-	"encoding/json"
 	"fmt"
-	"os"
+	"reflect"
 	"strings"
-	"text/template"
 	"time"
 
-	"github.com/docker/go-units"
-
 	tm "github.com/buger/goterm"
+	"github.com/docker/go-units"
 	"github.com/pkg/errors"
-	"github.com/projectatomic/libpod/libkpod"
-	"github.com/projectatomic/libpod/oci"
+	"github.com/projectatomic/libpod/cmd/kpod/formats"
+	"github.com/projectatomic/libpod/libpod"
 	"github.com/urfave/cli"
 )
 
-var printf func(format string, a ...interface{}) (n int, err error)
-var println func(a ...interface{}) (n int, err error)
-
 type statsOutputParams struct {
-	Container string
-	ID        string
-	CPUPerc   string
-	MemUsage  string
-	MemPerc   string
-	NetIO     string
-	BlockIO   string
-	PIDs      uint64
+	Container string `json:"name"`
+	ID        string `json:"id"`
+	CPUPerc   string `json:"cpu_percent"`
+	MemUsage  string `json:"mem_usage"`
+	MemPerc   string `json:"mem_percent"`
+	NetIO     string `json:"netio"`
+	BlockIO   string `json:"blocki"`
+	PIDS      uint64 `json:"pids"`
 }
 
 var (
@@ -46,8 +40,8 @@ var (
 			Usage: "pretty-print container statistics using a Go template",
 		},
 		cli.BoolFlag{
-			Name:  "json",
-			Usage: "output container statistics in json format",
+			Name:  "no-reset",
+			Usage: "disable resetting the screen between intervals",
 		},
 	}
 
@@ -66,49 +60,45 @@ func statsCmd(c *cli.Context) error {
 	if err := validateFlags(c, statsFlags); err != nil {
 		return err
 	}
-	config, err := getConfig(c)
-	if err != nil {
-		return errors.Wrapf(err, "could not read config")
-	}
-	containerServer, err := libkpod.New(config)
-	if err != nil {
-		return errors.Wrapf(err, "could not create container server")
-	}
-	defer containerServer.Shutdown()
-	err = containerServer.Update()
+
+	runtime, err := getRuntime(c)
 	if err != nil {
-		return errors.Wrapf(err, "could not update list of containers")
+		return errors.Wrapf(err, "could not get runtime")
 	}
+	defer runtime.Shutdown(false)
+
 	times := -1
 	if c.Bool("no-stream") {
 		times = 1
 	}
-	statsChan := make(chan []*libkpod.ContainerStats)
-	// iterate over the channel until it is closed
-	go func() {
-		// print using goterm
-		printf = tm.Printf
-		println = tm.Println
-		for stats := range statsChan {
-			// Continually refresh statistics
-			tm.Clear()
-			tm.MoveCursor(1, 1)
-			outputStats(stats, c.String("format"), c.Bool("json"))
-			tm.Flush()
-			time.Sleep(time.Second)
-		}
-	}()
-	return getStats(containerServer, c.Args(), c.Bool("all"), statsChan, times)
-}
 
-func getStats(server *libkpod.ContainerServer, args []string, all bool, statsChan chan []*libkpod.ContainerStats, times int) error {
-	ctrs, err := server.ListContainers(isRunning, ctrInList(args))
+	var format string
+	var ctrs []*libpod.Container
+	var containerFunc func() ([]*libpod.Container, error)
+	all := c.Bool("all")
+
+	if c.IsSet("format") {
+		format = c.String("format")
+	} else {
+		format = genStatsFormat()
+	}
+
+	if len(c.Args()) > 0 {
+		containerFunc = func() ([]*libpod.Container, error) { return runtime.GetContainersByList(c.Args()) }
+	} else if all {
+		containerFunc = runtime.GetAllContainers
+	} else {
+		containerFunc = runtime.GetRunningContainers
+	}
+
+	ctrs, err = containerFunc()
 	if err != nil {
-		return err
+		return errors.Wrapf(err, "unable to get list of containers")
 	}
-	containerStats := map[string]*libkpod.ContainerStats{}
+
+	containerStats := map[string]*libpod.ContainerStats{}
 	for _, ctr := range ctrs {
-		initialStats, err := server.GetContainerStats(ctr, &libkpod.ContainerStats{})
+		initialStats, err := ctr.GetContainerStats(&libpod.ContainerStats{})
 		if err != nil {
 			return err
 		}
@@ -120,17 +110,17 @@ func getStats(server *libkpod.ContainerServer, args []string, all bool, statsCha
 		step = 0
 	}
 	for i := 0; i < times; i += step {
-		reportStats := []*libkpod.ContainerStats{}
+		reportStats := []*libpod.ContainerStats{}
 		for _, ctr := range ctrs {
 			id := ctr.ID()
 			if _, ok := containerStats[ctr.ID()]; !ok {
-				initialStats, err := server.GetContainerStats(ctr, &libkpod.ContainerStats{})
+				initialStats, err := ctr.GetContainerStats(&libpod.ContainerStats{})
 				if err != nil {
 					return err
 				}
 				containerStats[id] = initialStats
 			}
-			stats, err := server.GetContainerStats(ctr, containerStats[id])
+			stats, err := ctr.GetContainerStats(containerStats[id])
 			if err != nil {
 				return err
 			}
@@ -138,48 +128,75 @@ func getStats(server *libkpod.ContainerServer, args []string, all bool, statsCha
 			containerStats[id] = stats
 			reportStats = append(reportStats, stats)
 		}
-		statsChan <- reportStats
-
-		err := server.Update()
+		ctrs, err = containerFunc()
 		if err != nil {
 			return err
 		}
-		ctrs, err = server.ListContainers(isRunning, ctrInList(args))
-		if err != nil {
-			return err
+		if strings.ToLower(format) != formats.JSONString && !c.Bool("no-reset") {
+			tm.Clear()
+			tm.MoveCursor(1, 1)
+			tm.Flush()
 		}
+		outputStats(reportStats, format)
+		time.Sleep(time.Second)
 	}
 	return nil
 }
 
-func outputStats(stats []*libkpod.ContainerStats, format string, json bool) error {
-	if format == "" {
-		outputStatsHeader()
+func outputStats(stats []*libpod.ContainerStats, format string) error {
+	var out formats.Writer
+	var outputStats []statsOutputParams
+	for _, s := range stats {
+		outputStats = append(outputStats, getStatsOutputParams(s))
 	}
-	if json {
-		return outputStatsAsJSON(stats)
+	if len(outputStats) == 0 {
+		return nil
 	}
-	var err error
-	for _, s := range stats {
-		if format == "" {
-			outputStatsUsingFormatString(s)
-		} else {
-			params := getStatsOutputParams(s)
-			err2 := outputStatsUsingTemplate(format, params)
-			if err2 != nil {
-				err = errors.Wrapf(err, err2.Error())
-			}
-		}
+	if strings.ToLower(format) == formats.JSONString {
+		out = formats.JSONStructArray{Output: statsToGeneric(outputStats, []statsOutputParams{})}
+	} else {
+		out = formats.StdoutTemplateArray{Output: statsToGeneric(outputStats, []statsOutputParams{}), Template: format, Fields: outputStats[0].headerMap()}
 	}
-	return err
+	return formats.Writer(out).Out()
 }
 
-func outputStatsHeader() {
-	printf("%-64s %-16s %-32s %-16s %-24s %-24s %s\n", "CONTAINER", "CPU %", "MEM USAGE / MEM LIMIT", "MEM %", "NET I/O", "BLOCK I/O", "PIDS")
+func genStatsFormat() (format string) {
+	return "table {{.Container}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.MemPerc}}\t{{.NetIO}}\t{{.BlockIO}}\t{{.PIDS}}"
 }
 
-func outputStatsUsingFormatString(stats *libkpod.ContainerStats) {
-	printf("%-64s %-16s %-32s %-16s %-24s %-24s %d\n", stats.Container, floatToPercentString(stats.CPU), combineHumanValues(stats.MemUsage, stats.MemLimit), floatToPercentString(stats.MemPerc), combineHumanValues(stats.NetInput, stats.NetOutput), combineHumanValues(stats.BlockInput, stats.BlockOutput), stats.PIDs)
+// imagesToGeneric creates an empty array of interfaces for output
+func statsToGeneric(templParams []statsOutputParams, JSONParams []statsOutputParams) (genericParams []interface{}) {
+	if len(templParams) > 0 {
+		for _, v := range templParams {
+			genericParams = append(genericParams, interface{}(v))
+		}
+		return
+	}
+	for _, v := range JSONParams {
+		genericParams = append(genericParams, interface{}(v))
+	}
+	return
+}
+
+// generate the header based on the template provided
+func (i *statsOutputParams) headerMap() map[string]string {
+	v := reflect.Indirect(reflect.ValueOf(i))
+	values := make(map[string]string)
+
+	for i := 0; i < v.NumField(); i++ {
+		key := v.Type().Field(i).Name
+		value := key
+		switch value {
+		case "CPUPerc":
+			value = "CPU%"
+		case "MemUsage":
+			value = "MemUsage/Limit"
+		case "MemPerc":
+			value = "Mem%"
+		}
+		values[key] = strings.ToUpper(splitCamelCase(value))
+	}
+	return values
 }
 
 func combineHumanValues(a, b uint64) string {
@@ -187,59 +204,23 @@ func combineHumanValues(a, b uint64) string {
 }
 
 func floatToPercentString(f float64) string {
-	return fmt.Sprintf("%.2f %s", f, "%")
+	strippedFloat, err := libpod.RemoveScientificNotationFromFloat(f)
+	if err != nil {
+		// If things go bazinga, return a safe value
+		return "0.00 %"
+	}
+	return fmt.Sprintf("%.2f", strippedFloat) + "%"
 }
 
-func getStatsOutputParams(stats *libkpod.ContainerStats) statsOutputParams {
+func getStatsOutputParams(stats *libpod.ContainerStats) statsOutputParams {
 	return statsOutputParams{
-		Container: stats.Container,
-		ID:        stats.Container,
+		Container: stats.ContainerID[:12],
+		ID:        stats.ContainerID,
 		CPUPerc:   floatToPercentString(stats.CPU),
 		MemUsage:  combineHumanValues(stats.MemUsage, stats.MemLimit),
 		MemPerc:   floatToPercentString(stats.MemPerc),
 		NetIO:     combineHumanValues(stats.NetInput, stats.NetOutput),
 		BlockIO:   combineHumanValues(stats.BlockInput, stats.BlockOutput),
-		PIDs:      stats.PIDs,
-	}
-}
-
-func outputStatsUsingTemplate(format string, params statsOutputParams) error {
-	tmpl, err := template.New("stats").Parse(format)
-	if err != nil {
-		return errors.Wrapf(err, "template parsing error")
-	}
-
-	err = tmpl.Execute(os.Stdout, params)
-	if err != nil {
-		return err
-	}
-	println()
-	return nil
-}
-
-func outputStatsAsJSON(stats []*libkpod.ContainerStats) error {
-	s, err := json.Marshal(stats)
-	if err != nil {
-		return err
-	}
-	println(s)
-	return nil
-}
-
-func isRunning(ctr *oci.Container) bool {
-	return ctr.State().Status == "running"
-}
-
-func ctrInList(idsOrNames []string) func(ctr *oci.Container) bool {
-	if len(idsOrNames) == 0 {
-		return func(*oci.Container) bool { return true }
-	}
-	return func(ctr *oci.Container) bool {
-		for _, idOrName := range idsOrNames {
-			if strings.HasPrefix(ctr.ID(), idOrName) || strings.HasSuffix(ctr.Name(), idOrName) {
-				return true
-			}
-		}
-		return false
+		PIDS:      stats.PIDs,
 	}
 }
diff --git a/completions/bash/kpod b/completions/bash/kpod
index 6c66c63c0..d403fed3e 100644
--- a/completions/bash/kpod
+++ b/completions/bash/kpod
@@ -1267,6 +1267,7 @@ _kpod_stats() {
      -a
      --no-stream
      --format
+     --no-reset
     "
 
     case "$cur" in
diff --git a/docs/kpod-stats.1.md b/docs/kpod-stats.1.md
index 1c1c0b35d..2b73616a0 100644
--- a/docs/kpod-stats.1.md
+++ b/docs/kpod-stats.1.md
@@ -17,6 +17,10 @@ Display a live stream of one or more containers' resource usage statistics
 
 Show all containers.  Only running containers are shown by default
 
+**--no-reset**
+
+Do not clear the terminal/screen in between reporting intervals
+
 **--no-stream**
 
 Disable streaming stats and only pull the first result, default setting is false
@@ -28,7 +32,43 @@ Pretty-print images using a Go template
 
 ## EXAMPLE
 
-TODO
+```
+# kpod stats -a --no-stream
+
+CONTAINER      CPU %   MEM USAGE / LIMIT   MEM %   NET IO    BLOCK IO       PIDS
+132ade621b5d   0.00%   1.618MB / 33.08GB   0.00%   0B / 0B   0B / 0B        0
+940e00a40a77   0.00%   1.544MB / 33.08GB   0.00%   0B / 0B   0B / 0B        0
+72a1dfb44ca7   0.00%   1.528MB / 33.08GB   0.00%   0B / 0B   0B / 0B        0
+f5a62a71b07b   0.00%   5.669MB / 33.08GB   0.02%   0B / 0B   0B / 0B        3
+31eab2cf93f4   0.00%   16.42MB / 33.08GB   0.05%   0B / 0B   22.43MB / 0B   0
+
+#
+```
+
+```
+# kpod stats --no-stream 31eab2cf93f4
+CONTAINER      CPU %   MEM USAGE / LIMIT   MEM %   NET IO    BLOCK IO       PIDS
+31eab2cf93f4   0.00%   16.42MB / 33.08GB   0.05%   0B / 0B   22.43MB / 0B   0
+
+#
+```
+```
+# kpod stats --no-stream --format=json 31eab2cf93f4
+[
+    {
+        "name": "31eab2cf93f4",
+        "id": "31eab2cf93f413af64a3f13d8d78393238658465d75e527333a8577f251162ec",
+        "cpu_percent": "0.00%",
+        "mem_usage": "16.42MB / 33.08GB",
+        "mem_percent": "0.05%",
+        "netio": "0B / 0B",
+        "blocki": "22.43MB / 0B",
+        "pids": 0
+    }
+]
+#
+```
+
 
 ## SEE ALSO
 kpod(1)
diff --git a/libpod/container.go b/libpod/container.go
index d53a863c0..604b5fe10 100644
--- a/libpod/container.go
+++ b/libpod/container.go
@@ -11,6 +11,7 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/containerd/cgroups"
 	"github.com/containers/storage"
 	"github.com/containers/storage/pkg/archive"
 	"github.com/docker/docker/daemon/caps"
@@ -55,6 +56,9 @@ const (
 	artifactsDir = "artifacts"
 )
 
+// CGroupParent is the prefix to a cgroup path in libpod
+var CGroupParent = "/libpod_parent"
+
 // Container is a single OCI container
 type Container struct {
 	config *ContainerConfig
@@ -577,7 +581,7 @@ func (c *Container) Init() (err error) {
 
 	// With the spec complete, do an OCI create
 	// TODO set cgroup parent in a sane fashion
-	if err := c.runtime.ociRuntime.createContainer(c, "/libpod_parent"); err != nil {
+	if err := c.runtime.ociRuntime.createContainer(c, CGroupParent); err != nil {
 		return err
 	}
 
@@ -1044,3 +1048,8 @@ func (c *Container) cleanupStorage() error {
 
 	return c.save()
 }
+
+// CGroupPath returns a cgroups "path" for a given container.
+func (c *Container) CGroupPath() cgroups.Path {
+	return cgroups.StaticPath(filepath.Join(CGroupParent, fmt.Sprintf("libpod-conmon-%s", c.ID())))
+}
diff --git a/libpod/runtime_ctr.go b/libpod/runtime_ctr.go
index f314b302f..320821b38 100644
--- a/libpod/runtime_ctr.go
+++ b/libpod/runtime_ctr.go
@@ -232,3 +232,31 @@ func (r *Runtime) GetContainers(filters ...ContainerFilter) ([]*Container, error
 
 	return ctrsFiltered, nil
 }
+
+// GetAllContainers is a helper function for GetContainers
+func (r *Runtime) GetAllContainers() ([]*Container, error) {
+	return r.state.AllContainers()
+}
+
+// GetRunningContainers is a helper function for GetContainers
+func (r *Runtime) GetRunningContainers() ([]*Container, error) {
+	running := func(c *Container) bool {
+		state, _ := c.State()
+		return state == ContainerStateRunning
+	}
+	return r.GetContainers(running)
+}
+
+// GetContainersByList is a helper function for GetContainers
+// which takes a []string of container IDs or names
+func (r *Runtime) GetContainersByList(containers []string) ([]*Container, error) {
+	var ctrs []*Container
+	for _, inputContainer := range containers {
+		ctr, err := r.LookupContainer(inputContainer)
+		if err != nil {
+			return ctrs, errors.Wrapf(err, "unable to lookup container %s", inputContainer)
+		}
+		ctrs = append(ctrs, ctr)
+	}
+	return ctrs, nil
+}
diff --git a/libpod/stats.go b/libpod/stats.go
new file mode 100644
index 000000000..86da0679e
--- /dev/null
+++ b/libpod/stats.go
@@ -0,0 +1,120 @@
+package libpod
+
+import (
+	"strings"
+	"syscall"
+	"time"
+
+	"github.com/containerd/cgroups"
+	"github.com/opencontainers/runc/libcontainer"
+	"github.com/pkg/errors"
+)
+
+// ContainerStats contains the statistics information for a running container
+type ContainerStats struct {
+	ContainerID string
+	CPU         float64
+	CPUNano     uint64
+	SystemNano  uint64
+	MemUsage    uint64
+	MemLimit    uint64
+	MemPerc     float64
+	NetInput    uint64
+	NetOutput   uint64
+	BlockInput  uint64
+	BlockOutput uint64
+	PIDs        uint64
+}
+
+// GetContainerStats gets the running stats for a given container
+func (c *Container) GetContainerStats(previousStats *ContainerStats) (*ContainerStats, error) {
+	stats := new(ContainerStats)
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	if err := c.syncContainer(); err != nil {
+		return stats, errors.Wrapf(err, "error updating container %s state", c.ID())
+	}
+	cgroup, err := cgroups.Load(cgroups.V1, c.CGroupPath())
+	if err != nil {
+		return stats, errors.Wrapf(err, "unable to load cgroup at %+v", c.CGroupPath())
+	}
+
+	cgroupStats, err := cgroup.Stat()
+	if err != nil {
+		return stats, errors.Wrapf(err, "unable to obtain cgroup stats")
+	}
+	conState := c.state.State
+	if err != nil {
+		return stats, errors.Wrapf(err, "unable to determine container state")
+	}
+
+	previousCPU := previousStats.CPUNano
+	previousSystem := previousStats.SystemNano
+	stats.ContainerID = c.ID()
+	stats.CPU = calculateCPUPercent(cgroupStats, previousCPU, previousSystem)
+	stats.MemUsage = cgroupStats.Memory.Usage.Usage
+	stats.MemLimit = getMemLimit(cgroupStats.Memory.Usage.Limit)
+	stats.MemPerc = (float64(stats.MemUsage) / float64(stats.MemLimit)) * 100
+	stats.PIDs = 0
+	if conState == ContainerStateRunning {
+		stats.PIDs = cgroupStats.Pids.Current - 1
+	}
+	stats.BlockInput, stats.BlockOutput = calculateBlockIO(cgroupStats)
+	stats.CPUNano = cgroupStats.Cpu.Usage.Total
+	stats.SystemNano = cgroupStats.Cpu.Usage.Kernel
+	// TODO Figure out where to get the Netout stuff.
+	//stats.NetInput, stats.NetOutput = getContainerNetIO(cgroupStats)
+	return stats, nil
+}
+
+// getMemory limit returns the memory limit for a given cgroup
+// If the configured memory limit is larger than the total memory on the sys, the
+// physical system memory size is returned
+func getMemLimit(cgroupLimit uint64) uint64 {
+	si := &syscall.Sysinfo_t{}
+	err := syscall.Sysinfo(si)
+	if err != nil {
+		return cgroupLimit
+	}
+
+	physicalLimit := uint64(si.Totalram)
+	if cgroupLimit > physicalLimit {
+		return physicalLimit
+	}
+	return cgroupLimit
+}
+
+// Returns the total number of bytes transmitted and received for the given container stats
+func getContainerNetIO(stats *libcontainer.Stats) (received uint64, transmitted uint64) { //nolint
+	for _, iface := range stats.Interfaces {
+		received += iface.RxBytes
+		transmitted += iface.TxBytes
+	}
+	return
+}
+
+func calculateCPUPercent(stats *cgroups.Stats, previousCPU, previousSystem uint64) float64 {
+	var (
+		cpuPercent  = 0.0
+		cpuDelta    = float64(stats.Cpu.Usage.Total - previousCPU)
+		systemDelta = float64(uint64(time.Now().UnixNano()) - previousSystem)
+	)
+	if systemDelta > 0.0 && cpuDelta > 0.0 {
+		// gets a ratio of container cpu usage total, multiplies it by the number of cores (4 cores running
+		// at 100% utilization should be 400% utilization), and multiplies that by 100 to get a percentage
+		cpuPercent = (cpuDelta / systemDelta) * float64(len(stats.Cpu.Usage.PerCpu)) * 100
+	}
+	return cpuPercent
+}
+
+func calculateBlockIO(stats *cgroups.Stats) (read uint64, write uint64) {
+	for _, blkIOEntry := range stats.Blkio.IoServiceBytesRecursive {
+		switch strings.ToLower(blkIOEntry.Op) {
+		case "read":
+			read += blkIOEntry.Value
+		case "write":
+			write += blkIOEntry.Value
+		}
+	}
+	return
+}
diff --git a/libpod/util.go b/libpod/util.go
index 61089b525..de5c3ff31 100644
--- a/libpod/util.go
+++ b/libpod/util.go
@@ -9,6 +9,8 @@ import (
 
 	"github.com/containers/image/signature"
 	"github.com/containers/image/types"
+	"github.com/pkg/errors"
+	"strconv"
 )
 
 // Runtime API constants
@@ -76,3 +78,21 @@ func GetPolicyContext(path string) (*signature.PolicyContext, error) {
 	}
 	return signature.NewPolicyContext(policy)
 }
+
+// RemoveScientificNotationFromFloat returns a float without any
+// scientific notation if the number has any.
+// golang does not handle conversion of float64s that have scientific
+// notation in them and otherwise stinks.  please replace this if you have
+// a better implementation.
+func RemoveScientificNotationFromFloat(x float64) (float64, error) {
+	bigNum := strconv.FormatFloat(x, 'g', -1, 64)
+	breakPoint := strings.IndexAny(bigNum, "Ee")
+	if breakPoint > 0 {
+		bigNum = bigNum[:breakPoint]
+	}
+	result, err := strconv.ParseFloat(bigNum, 64)
+	if err != nil {
+		return x, errors.Wrapf(err, "unable to remove scientific number from calculations")
+	}
+	return result, nil
+}
diff --git a/libpod/util_test.go b/libpod/util_test.go
index 24e5fdfac..7b9d19a43 100644
--- a/libpod/util_test.go
+++ b/libpod/util_test.go
@@ -17,3 +17,13 @@ func TestStringInSlice(t *testing.T) {
 	// string is not in empty slice
 	assert.False(t, StringInSlice("one", []string{}))
 }
+
+func TestRemoveScientificNotationFromFloat(t *testing.T) {
+	numbers := []float64{0.0, .5, 1.99999932, 1.04e+10}
+	results := []float64{0.0, .5, 1.99999932, 1.04}
+	for i, x := range numbers {
+		result, err := RemoveScientificNotationFromFloat(x)
+		assert.NoError(t, err)
+		assert.Equal(t, result, results[i])
+	}
+}
diff --git a/test/kpod_stats.bats b/test/kpod_stats.bats
index b98c83c90..8eb112496 100644
--- a/test/kpod_stats.bats
+++ b/test/kpod_stats.bats
@@ -10,104 +10,51 @@ function setup() {
     copy_images
 }
 
-@test "stats single output" {
-    skip "Test needs to be converted to kpod run"
-    start_crio
-    run crioctl pod run --config "$TESTDATA"/sandbox_config.json
+@test "stats should run with no containers" {
+    run ${KPOD_BINARY} ${KPOD_OPTIONS} stats --no-stream
     echo "$output"
     [ "$status" -eq 0 ]
-    pod_id="$output"
-    run crioctl ctr create --config "$TESTDATA"/container_redis.json --pod "$pod_id"
+}
+
+@test "stats with bogus container id" {
+    run ${KPOD_BINARY} ${KPOD_OPTIONS} stats --no-stream  123
     echo "$output"
-    [ "$status" -eq 0 ]
+    [ "$status" -eq 1 ]
+}
+
+@test "stats on a running container" {
+    run ${KPOD_BINARY} ${KPOD_OPTIONS} run -d -t ${ALPINE} sleep 99
     ctr_id="$output"
-    run crioctl ctr start --id "$ctr_id"
     echo "$output"
-    [ "$status" -eq 0 ]
-    run bash -c ${KPOD_BINARY} ${KPOD_OPTIONS} stats --no-stream "$ctr_id"
+    run bash -c "${KPOD_BINARY} ${KPOD_OPTIONS} stats --no-stream  $ctr_id"
     echo "$output"
     [ "$status" -eq 0 ]
-    cleanup_ctrs
-    cleanup_pods
-    stop_crio
 }
 
-@test "stats does not output stopped container" {
-    skip "Test needs to be converted to kpod run"
-    start_crio
-    run crioctl pod run --config "$TESTDATA"/sandbox_config.json
-    echo "$output"
-    [ "$status" -eq 0 ]
-    pod_id="$output"
-    run crioctl ctr create --config "$TESTDATA"/container_redis.json --pod "$pod_id"
-    echo "$output"
-    [ "$status" -eq 0 ]
-    ctr_id="$output"
-    run bash -c ${KPOD_BINARY} ${KPOD_OPTIONS} stats --no-stream
+@test "stats on a running container no id" {
+    ${KPOD_BINARY} ${KPOD_OPTIONS} run -d -t ${ALPINE} sleep 99
+    run ${KPOD_BINARY} ${KPOD_OPTIONS} stats --no-stream
     echo "$output"
     [ "$status" -eq 0 ]
-    cleanup_ctrs
-    cleanup_pods
-    stop_crio
 }
 
-@test "stats outputs stopped container with all flag" {
-    skip "Test needs to be converted to kpod run"
-    start_crio
-    run crioctl pod run --config "$TESTDATA"/sandbox_config.json
+@test "stats on all containers" {
+    ${KPOD_BINARY} ${KPOD_OPTIONS} run -d -t ${ALPINE} ls
+    run ${KPOD_BINARY} ${KPOD_OPTIONS} stats --no-stream -a
     echo "$output"
     [ "$status" -eq 0 ]
-    pod_id="$output"
-    run crioctl ctr create --config "$TESTDATA"/container_redis.json --pod "$pod_id"
-    echo "$output"
-    [ "$status" -eq 0 ]
-    ctr_id="$output"
-    run bash -c ${KPOD_BINARY} ${KPOD_OPTIONS} stats --no-stream --all
-    echo "$output"
-    [ "$status" -eq 0 ]
-    cleanup_ctrs
-    cleanup_pods
-    stop_crio
 }
 
-@test "stats output only id" {
-    skip "Test needs to be converted to kpod run"
-    start_crio
-    run crioctl pod run --config "$TESTDATA"/sandbox_config.json
-    echo "$output"
-    [ "$status" -eq 0 ]
-    pod_id="$output"
-    run crioctl ctr create --config "$TESTDATA"/container_redis.json --pod "$pod_id"
-    echo "$output"
-    [ "$status" -eq 0 ]
-    ctr_id="$output"
-    run crioctl ctr start --id "$ctr_id"
+@test "stats only output IDs" {
+    ${KPOD_BINARY} ${KPOD_OPTIONS} run -d -t ${ALPINE} sleep 99
+    run ${KPOD_BINARY} ${KPOD_OPTIONS} stats --no-stream --format "{{.Container}}"
     echo "$output"
     [ "$status" -eq 0 ]
-    run bash -c ${KPOD_BINARY} ${KPOD_OPTIONS} stats --no-stream --format {{.ID}} "$ctr_id"
-    echo "$output"
-    [ "$status" -eq 0 ]
-    # once ps is implemented, run ps -q and see if that equals the output from above
-    cleanup_ctrs
-    cleanup_pods
-    stop_crio
 }
 
-@test "stats streaming output" {
-    skip "Test needs to be converted to kpod run"
-    start_crio
-    run crioctl pod run --config "$TESTDATA"/sandbox_config.json
-    echo "$output"
-    [ "$status" -eq 0 ]
-    pod_id="$output"
-    run crioctl ctr create --config "$TESTDATA"/container_redis.json --pod "$pod_id"
+@test "stats json output" {
+    ${KPOD_BINARY} ${KPOD_OPTIONS} run -d -t ${ALPINE} ls
+    run bash -c "${KPOD_BINARY} ${KPOD_OPTIONS} stats --no-stream -a --format json | python -m json.tool"
     echo "$output"
     [ "$status" -eq 0 ]
-    ctr_id="$output"
-    run timeout 5s bash -c "${KPOD_BINARY} ${KPOD_OPTIONS} stats --all"
-    echo "$output"
-    [ "$status" -eq 124 ] #124 is the status set by timeout when it has to kill the command at the end of the given time
-    cleanup_ctrs
-    cleanup_pods
-    stop_crio
 }