EXPERIMENTAL: Do not call out to runc for sync

When syncing container state, we normally call out to runc to see
the container's status. This does have significant performance
implications, though, and we've seen issues with large amounts of
runc processes being spawned.

This patch attempts to use stat calls on the container exit file
created by Conmon instead to sync state. This massively decreases
the cost of calling updateContainer (it has gone from an
almost-unconditional fork/exec of runc to a single stat call that
can be avoided in most states).

Signed-off-by: Matthew Heon <matthew.heon@gmail.com>

1 files changed, 35 insertions, 23 deletions

diff --git a/libpod/oci.go b/libpod/oci.go
index ca8f967c4..b390b1967 100644
--- a/libpod/oci.go
+++ b/libpod/oci.go
@@ -11,12 +11,10 @@ import (
 	"os/exec"
 	"path/filepath"
 	"runtime"
-	"strconv"
 	"strings"
 	"syscall"
 	"time"
 
-	"github.com/containers/libpod/pkg/ctime"
 	"github.com/containers/libpod/pkg/rootless"
 	"github.com/containers/libpod/pkg/util"
 	"github.com/coreos/go-systemd/activation"
@@ -451,17 +449,47 @@ func (r *OCIRuntime) createOCIContainer(ctr *Container, cgroupParent string, res
 
 // updateContainerStatus retrieves the current status of the container from the
 // runtime. It updates the container's state but does not save it.
-func (r *OCIRuntime) updateContainerStatus(ctr *Container) error {
-	state := new(spec.State)
+// If useRunc is false, we will not directly hit runc to see the container's
+// status, but will instead only check for the existance of the conmon exit file
+// and update state to stopped if it exists.
+func (r *OCIRuntime) updateContainerStatus(ctr *Container, useRunc bool) error {
+	exitFile := ctr.exitFilePath()
 
 	runtimeDir, err := util.GetRootlessRuntimeDir()
 	if err != nil {
 		return err
 	}
 
+	// If not using runc, we don't need to do most of this.
+	if !useRunc {
+		// If the container's not running, nothing to do.
+		if ctr.state.State != ContainerStateRunning {
+			return nil
+		}
+
+		// Check for the exit file conmon makes
+		info, err := os.Stat(exitFile)
+		if err != nil {
+			if os.IsNotExist(err) {
+				// Container is still running, no error
+				return nil
+			}
+
+			return errors.Wrapf(err, "error running stat on container %s exit file", ctr.ID())
+		}
+
+		// Alright, it exists. Transition to Stopped state.
+		ctr.state.State = ContainerStateStopped
+
+		// Read the exit file to get our stopped time and exit code.
+		return ctr.handleExitFile(exitFile, info)
+	}
+
 	// Store old state so we know if we were already stopped
 	oldState := ctr.state.State
 
+	state := new(spec.State)
+
 	cmd := exec.Command(r.path, "state", ctr.ID())
 	cmd.Env = append(cmd.Env, fmt.Sprintf("XDG_RUNTIME_DIR=%s", runtimeDir))
 	outPipe, err := cmd.StdoutPipe()
@@ -514,7 +542,6 @@ func (r *OCIRuntime) updateContainerStatus(ctr *Container) error {
 	// Only grab exit status if we were not already stopped
 	// If we were, it should already be in the database
 	if ctr.state.State == ContainerStateStopped && oldState != ContainerStateStopped {
-		exitFile := filepath.Join(r.exitsDir, ctr.ID())
 		var fi os.FileInfo
 		err = kwait.ExponentialBackoff(
 			kwait.Backoff{
@@ -538,24 +565,7 @@ func (r *OCIRuntime) updateContainerStatus(ctr *Container) error {
 			return nil
 		}
 
-		ctr.state.FinishedTime = ctime.Created(fi)
-		statusCodeStr, err := ioutil.ReadFile(exitFile)
-		if err != nil {
-			return errors.Wrapf(err, "failed to read exit file for container %s", ctr.ID())
-		}
-		statusCode, err := strconv.Atoi(string(statusCodeStr))
-		if err != nil {
-			return errors.Wrapf(err, "error converting exit status code for container %s to int",
-				ctr.ID())
-		}
-		ctr.state.ExitCode = int32(statusCode)
-
-		oomFilePath := filepath.Join(ctr.bundlePath(), "oom")
-		if _, err = os.Stat(oomFilePath); err == nil {
-			ctr.state.OOMKilled = true
-		}
-
-		ctr.state.Exited = true
+		return ctr.handleExitFile(exitFile, fi)
 	}
 
 	return nil
@@ -601,6 +611,8 @@ func (r *OCIRuntime) killContainer(ctr *Container, signal uint) error {
 // Does not set finished time for container, assumes you will run updateStatus
 // after to pull the exit code
 func (r *OCIRuntime) stopContainer(ctr *Container, timeout uint) error {
+	logrus.Debugf("Stopping container %s (PID %d)", ctr.ID(), ctr.state.PID)
+
 	// Ping the container to see if it's alive
 	// If it's not, it's already stopped, return
 	err := unix.Kill(ctr.state.PID, 0)