Add ability to update container status from runc

Wire this in to all state-bound container operations to ensure
syncronization of container state.

Also exposes PID of running containers via API.

Signed-off-by: Matthew Heon <matthew.heon@gmail.com>

Closes: #56
Approved by: rhatdan

6 files changed, 197 insertions, 60 deletions

diff --git a/libpod/container.go b/libpod/container.go
index 728a29dec..710ae866b 100644
--- a/libpod/container.go
+++ b/libpod/container.go
@@ -76,6 +76,11 @@ type containerRuntimeInfo struct {
 	FinishedTime time.Time `json:"finishedTime,omitempty"`
 	// ExitCode is the exit code returned when the container stopped
 	ExitCode int32 `json:"exitCode,omitempty"`
+	// OOMKilled indicates that the container was killed as it ran out of
+	// memory
+	OOMKilled bool `json:"oomKilled,omitempty"`
+	// PID is the PID of a running container
+	PID int `json:"pid,omitempty"`
 
 	// TODO: Save information about image used in container if one is used
 }
@@ -129,10 +134,10 @@ func (c *Container) Name() string {
 // The spec returned is the one used to create the container. The running
 // spec may differ slightly as mounts are added based on the image
 func (c *Container) Spec() *spec.Spec {
-	spec := new(spec.Spec)
-	deepcopier.Copy(c.config.Spec).To(spec)
+	returnSpec := new(spec.Spec)
+	deepcopier.Copy(c.config.Spec).To(returnSpec)
 
-	return spec
+	return returnSpec
 }
 
 // Labels returns the container's labels
@@ -150,13 +155,26 @@ func (c *Container) State() (ContainerState, error) {
 	c.lock.Lock()
 	defer c.lock.Unlock()
 
-	if err := c.runtime.state.UpdateContainer(c); err != nil {
-		return ContainerStateUnknown, errors.Wrapf(err, "error updating container %s state", c.ID())
+	if err := c.syncContainer(); err != nil {
+		return ContainerStateUnknown, err
 	}
 
 	return c.state.State, nil
 }
 
+// PID returns the PID of the container
+// An error is returned if the container is not running
+func (c *Container) PID() (int, error) {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+
+	if err := c.syncContainer(); err != nil {
+		return -1, err
+	}
+
+	return c.state.PID, nil
+}
+
 // The path to the container's root filesystem - where the OCI spec will be
 // placed, amongst other things
 func (c *Container) bundlePath() string {
@@ -168,6 +186,32 @@ func (c *Container) attachSocketPath() string {
 	return filepath.Join(c.runtime.ociRuntime.socketsDir, c.ID(), "attach")
 }
 
+// Sync this container with on-disk state and runc status
+// Should only be called with container lock held
+func (c *Container) syncContainer() error {
+	if err := c.runtime.state.UpdateContainer(c); err != nil {
+		return err
+	}
+
+	// If runc knows about the container, update its status in runc
+	// And then save back to disk
+	if (c.state.State != ContainerStateUnknown) &&
+		(c.state.State != ContainerStateConfigured) {
+		if err := c.runtime.ociRuntime.updateContainerStatus(c); err != nil {
+			return err
+		}
+		if err := c.runtime.state.SaveContainer(c); err != nil {
+			return err
+		}
+	}
+
+	if !c.valid {
+		return errors.Wrapf(ErrCtrRemoved, "container %s is not valid", c.ID())
+	}
+
+	return nil
+}
+
 // Make a new container
 func newContainer(rspec *spec.Spec) (*Container, error) {
 	if rspec == nil {
@@ -191,9 +235,6 @@ func newContainer(rspec *spec.Spec) (*Container, error) {
 
 // Create container root filesystem for use
 func (c *Container) setupStorage() error {
-	c.lock.Lock()
-	defer c.lock.Unlock()
-
 	if !c.valid {
 		return errors.Wrapf(ErrCtrRemoved, "container %s is not valid", c.ID())
 	}
@@ -220,9 +261,6 @@ func (c *Container) setupStorage() error {
 
 // Tear down a container's storage prior to removal
 func (c *Container) teardownStorage() error {
-	c.lock.Lock()
-	defer c.lock.Unlock()
-
 	if !c.valid {
 		return errors.Wrapf(ErrCtrRemoved, "container %s is not valid", c.ID())
 	}
@@ -251,12 +289,8 @@ func (c *Container) Init() (err error) {
 	c.lock.Lock()
 	defer c.lock.Unlock()
 
-	if err := c.runtime.state.UpdateContainer(c); err != nil {
-		return errors.Wrapf(err, "error updating container %s state", c.ID())
-	}
-
-	if !c.valid {
-		return errors.Wrapf(ErrCtrRemoved, "container %s is not valid", c.ID())
+	if err := c.syncContainer(); err != nil {
+		return err
 	}
 
 	if c.state.State != ContainerStateConfigured {
@@ -325,12 +359,8 @@ func (c *Container) Start() error {
 	c.lock.Lock()
 	defer c.lock.Unlock()
 
-	if err := c.runtime.state.UpdateContainer(c); err != nil {
-		return errors.Wrapf(err, "error updating container %s state", c.ID())
-	}
-
-	if !c.valid {
-		return ErrCtrRemoved
+	if err := c.syncContainer(); err != nil {
+		return err
 	}
 
 	// Container must be created or stopped to be started
@@ -344,8 +374,10 @@ func (c *Container) Start() error {
 
 	logrus.Debugf("Started container %s", c.ID())
 
-	c.state.StartedTime = time.Now()
-	c.state.State = ContainerStateRunning
+	// Update container's state as it should be ContainerStateRunning now
+	if err := c.runtime.ociRuntime.updateContainerStatus(c); err != nil {
+		return err
+	}
 
 	if err := c.runtime.state.SaveContainer(c); err != nil {
 		return errors.Wrapf(err, "error saving container %s state", c.ID())
@@ -373,12 +405,8 @@ func (c *Container) Exec(cmd []string, tty bool, stdin bool) (string, error) {
 // Attach attaches to a container
 // Returns fully qualified URL of streaming server for the container
 func (c *Container) Attach(noStdin bool, keys string, attached chan<- bool) error {
-	if err := c.runtime.state.UpdateContainer(c); err != nil {
-		return errors.Wrapf(err, "error updating container %s state", c.ID())
-	}
-
-	if !c.valid {
-		return errors.Wrapf(ErrCtrRemoved, "container %s is not valid", c.ID())
+	if err := c.syncContainer(); err != nil {
+		return err
 	}
 
 	if c.state.State == ContainerStateRunning || c.state.State == ContainerStatePaused {
diff --git a/libpod/finished_amd64.go b/libpod/finished_amd64.go
new file mode 100644
index 000000000..11dbc9181
--- /dev/null
+++ b/libpod/finished_amd64.go
@@ -0,0 +1,16 @@
+// +build !arm,!386
+
+package libpod
+
+import (
+	"os"
+	"syscall"
+	"time"
+)
+
+// Get the created time of a file
+// Only works on 64-bit OSes
+func getFinishedTime(fi os.FileInfo) time.Time {
+	st := fi.Sys().(*syscall.Stat_t)
+	return time.Unix(st.Ctim.Sec, st.Ctim.Nsec)
+}
diff --git a/libpod/oci.go b/libpod/oci.go
index 9fa2d3788..3ad3fdd8e 100644
--- a/libpod/oci.go
+++ b/libpod/oci.go
@@ -4,9 +4,11 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strconv"
 	"syscall"
 	"time"
 
@@ -15,6 +17,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/sys/unix"
+	kwait "k8s.io/apimachinery/pkg/util/wait"
 
 	// TODO import these functions into libpod and remove the import
 	// Trying to keep libpod from depending on CRI-O code
@@ -255,19 +258,90 @@ func (r *OCIRuntime) createContainer(ctr *Container, cgroupParent string) error
 }
 
 // updateContainerStatus retrieves the current status of the container from the
-// runtime
-// remove nolint when implemented
-func (r *OCIRuntime) updateContainerStatus(ctr *Container) error { //nolint
-	return ErrNotImplemented
+// runtime. It updates the container's state but does not save it.
+func (r *OCIRuntime) updateContainerStatus(ctr *Container) error {
+	state := new(spec.State)
+
+	out, err := exec.Command(r.path, "state", ctr.ID()).CombinedOutput()
+	if err != nil {
+		return errors.Wrapf(err, "error getting container %s state. stderr/out: %s", ctr.ID(), out)
+	}
+
+	if err := json.NewDecoder(bytes.NewBuffer(out)).Decode(state); err != nil {
+		return errors.Wrapf(err, "error decoding container status for container %s", ctr.ID())
+	}
+
+	ctr.state.PID = state.Pid
+
+	switch state.Status {
+	case "created":
+		ctr.state.State = ContainerStateCreated
+	case "paused":
+		ctr.state.State = ContainerStatePaused
+	case "running":
+		ctr.state.State = ContainerStateRunning
+	case "stopped":
+		ctr.state.State = ContainerStateStopped
+	default:
+		return errors.Wrapf(ErrInternal, "unrecognized status returned by runc for container %s: %s",
+			ctr.ID(), state.Status)
+	}
+
+	if ctr.state.State == ContainerStateStopped {
+		exitFile := filepath.Join(r.exitsDir, ctr.ID())
+		var fi os.FileInfo
+		err = kwait.ExponentialBackoff(
+			kwait.Backoff{
+				Duration: 500 * time.Millisecond,
+				Factor:   1.2,
+				Steps:    6,
+			},
+			func() (bool, error) {
+				var err error
+				fi, err = os.Stat(exitFile)
+				if err != nil {
+					// wait longer
+					return false, nil
+				}
+				return true, nil
+			})
+		if err != nil {
+			ctr.state.ExitCode = -1
+			ctr.state.FinishedTime = time.Now()
+			return errors.Wrapf(err, "no exit file for container %s found", ctr.ID())
+		}
+
+		ctr.state.FinishedTime = getFinishedTime(fi)
+		statusCodeStr, err := ioutil.ReadFile(exitFile)
+		if err != nil {
+			return errors.Wrapf(err, "failed to read exit file for container %s", ctr.ID())
+		}
+		statusCode, err := strconv.Atoi(string(statusCodeStr))
+		if err != nil {
+			return errors.Wrapf(err, "error convertaing exit status code for container %s to int",
+				ctr.ID())
+		}
+		ctr.state.ExitCode = int32(statusCode)
+
+		oomFilePath := filepath.Join(ctr.bundlePath(), "oom")
+		if _, err = os.Stat(oomFilePath); err == nil {
+			ctr.state.OOMKilled = true
+		}
+
+	}
+
+	return nil
 }
 
 // startContainer starts the given container
-// remove nolint when function is complete
-func (r *OCIRuntime) startContainer(ctr *Container) error { //nolint
+// Sets time the container was started, but does not save it.
+func (r *OCIRuntime) startContainer(ctr *Container) error {
 	// TODO: streams should probably *not* be our STDIN/OUT/ERR - redirect to buffers?
-	err := utils.ExecCmdWithStdStreams(os.Stdin, os.Stdout, os.Stderr, r.path, "start", ctr.ID())
+	if err := utils.ExecCmdWithStdStreams(os.Stdin, os.Stdout, os.Stderr, r.path, "start", ctr.ID()); err != nil {
+		return err
+	}
 
-	// TODO record start time in container struct
+	ctr.state.StartedTime = time.Now()
 
-	return err
+	return nil
 }
diff --git a/libpod/runtime_ctr.go b/libpod/runtime_ctr.go
index aa8ff7d88..fd92a3411 100644
--- a/libpod/runtime_ctr.go
+++ b/libpod/runtime_ctr.go
@@ -91,12 +91,8 @@ func (r *Runtime) RemoveContainer(c *Container, force bool) error {
 		return ErrRuntimeStopped
 	}
 
-	if !c.valid {
-		return ErrCtrRemoved
-	}
-
 	// Update the container to get current state
-	if err := r.state.UpdateContainer(c); err != nil {
+	if err := c.syncContainer(); err != nil {
 		return err
 	}
 
@@ -225,9 +221,3 @@ func (r *Runtime) removeMultipleContainers(containers []storage.Container) error
 	}
 	return nil
 }
-
-// ContainerConfigToDisk saves a container's nonvolatile configuration to disk
-// remove nolint when implemented
-func (r *Runtime) containerConfigToDisk(ctr *Container) error { //nolint
-	return ErrNotImplemented
-}
diff --git a/libpod/sql_state.go b/libpod/sql_state.go
index 466ad66a5..893223914 100644
--- a/libpod/sql_state.go
+++ b/libpod/sql_state.go
@@ -103,7 +103,9 @@ func (s *SQLState) Container(id string) (*Container, error) {
                               containerState.MountPoint,
                               containerState.StartedTime,
                               containerState.FinishedTime,
-                              containerState.ExitCode
+                              containerState.ExitCode,
+                              containerState.OomKilled,
+                              containerState.Pid
                        FROM containers
                        INNER JOIN
                            containerState ON containers.Id = containerState.Id
@@ -136,7 +138,9 @@ func (s *SQLState) LookupContainer(idOrName string) (*Container, error) {
                               containerState.MountPoint,
                               containerState.StartedTime,
                               containerState.FinishedTime,
-                              containerState.ExitCode
+                              containerState.ExitCode,
+                              containerState.OomKilled,
+                              containerState.Pid
                        FROM containers
                        INNER JOIN
                            containerState ON containers.Id = containerState.Id
@@ -220,7 +224,7 @@ func (s *SQLState) AddContainer(ctr *Container) (err error) {
                     ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
                 );`
 		addCtrState = `INSERT INTO containerState VALUES (
-                    ?, ?, ?, ?, ?, ?, ?, ?
+                    ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
                 );`
 	)
 
@@ -278,7 +282,9 @@ func (s *SQLState) AddContainer(ctr *Container) (err error) {
 		ctr.state.Mountpoint,
 		timeToSQL(ctr.state.StartedTime),
 		timeToSQL(ctr.state.FinishedTime),
-		ctr.state.ExitCode)
+		ctr.state.ExitCode,
+		boolToSQL(ctr.state.OOMKilled),
+		ctr.state.PID)
 	if err != nil {
 		return errors.Wrapf(err, "error adding container %s state to database", ctr.ID())
 	}
@@ -315,7 +321,9 @@ func (s *SQLState) UpdateContainer(ctr *Container) error {
                               Mountpoint,
                               StartedTime,
                               FinishedTime,
-                              ExitCode
+                              ExitCode,
+                              OomKilled,
+                              Pid
                        FROM containerState WHERE ID=?;`
 
 	var (
@@ -326,6 +334,8 @@ func (s *SQLState) UpdateContainer(ctr *Container) error {
 		startedTimeString  string
 		finishedTimeString string
 		exitCode           int32
+		oomKilled          int
+		pid                int
 	)
 
 	if !s.valid {
@@ -344,7 +354,9 @@ func (s *SQLState) UpdateContainer(ctr *Container) error {
 		&mountpoint,
 		&startedTimeString,
 		&finishedTimeString,
-		&exitCode)
+		&exitCode,
+		&oomKilled,
+		&pid)
 	if err != nil {
 		// The container may not exist in the database
 		if err == sql.ErrNoRows {
@@ -364,6 +376,8 @@ func (s *SQLState) UpdateContainer(ctr *Container) error {
 	newState.RunDir = runDir
 	newState.Mountpoint = mountpoint
 	newState.ExitCode = exitCode
+	newState.OOMKilled = boolFromSQL(oomKilled)
+	newState.PID = pid
 
 	if newState.Mountpoint != "" {
 		newState.Mounted = true
@@ -396,7 +410,9 @@ func (s *SQLState) SaveContainer(ctr *Container) error {
                           Mountpoint=?,
                           StartedTime=?,
                           FinishedTime=?,
-                          ExitCode=?
+                          ExitCode=?,
+                          OomKilled=?,
+                          Pid=?
                        WHERE Id=?;`
 
 	s.lock.Lock()
@@ -431,6 +447,8 @@ func (s *SQLState) SaveContainer(ctr *Container) error {
 		timeToSQL(ctr.state.StartedTime),
 		timeToSQL(ctr.state.FinishedTime),
 		ctr.state.ExitCode,
+		boolToSQL(ctr.state.OOMKilled),
+		ctr.state.PID,
 		ctr.ID())
 	if err != nil {
 		return errors.Wrapf(err, "error updating container %s state in database", ctr.ID())
@@ -521,7 +539,9 @@ func (s *SQLState) AllContainers() ([]*Container, error) {
                               containerState.MountPoint,
                               containerState.StartedTime,
                               containerState.FinishedTime,
-                              containerState.ExitCode
+                              containerState.ExitCode,
+                              containerState.OomKilled,
+                              containerState.Pid
                       FROM containers
                       INNER JOIN
                           containerState ON containers.Id = containerState.Id;`
diff --git a/libpod/sql_state_internal.go b/libpod/sql_state_internal.go
index 698b0433c..b6816e008 100644
--- a/libpod/sql_state_internal.go
+++ b/libpod/sql_state_internal.go
@@ -61,7 +61,10 @@ func prepareDB(db *sql.DB) (err error) {
             StartedTime TEXT NUT NULL,
             FinishedTime TEXT NOT NULL,
             ExitCode INTEGER NOT NULL,
+            OomKilled INTEGER NOT NULL,
+            Pid INTEGER NOT NULL,
             CHECK (State>0),
+            CHECK (OomKilled IN (0, 1)),
             FOREIGN KEY (Id) REFERENCES containers(Id) DEFERRABLE INITIALLY DEFERRED
         );
         `
@@ -149,6 +152,8 @@ func ctrFromScannable(row scannable, runtime *Runtime, specsDir string) (*Contai
 		startedTimeString  string
 		finishedTimeString string
 		exitCode           int32
+		oomKilled          int
+		pid                int
 	)
 
 	err := row.Scan(
@@ -169,7 +174,9 @@ func ctrFromScannable(row scannable, runtime *Runtime, specsDir string) (*Contai
 		&mountpoint,
 		&startedTimeString,
 		&finishedTimeString,
-		&exitCode)
+		&exitCode,
+		&oomKilled,
+		&pid)
 	if err != nil {
 		if err == sql.ErrNoRows {
 			return nil, ErrNoSuchCtr
@@ -197,6 +204,8 @@ func ctrFromScannable(row scannable, runtime *Runtime, specsDir string) (*Contai
 	ctr.state.RunDir = runDir
 	ctr.state.Mountpoint = mountpoint
 	ctr.state.ExitCode = exitCode
+	ctr.state.OOMKilled = boolFromSQL(oomKilled)
+	ctr.state.PID = pid
 
 	// TODO should we store this in the database separately instead?
 	if ctr.state.Mountpoint != "" {