diff options
-rw-r--r-- | libpod/container.go | 90 | ||||
-rw-r--r-- | libpod/finished_amd64.go | 16 | ||||
-rw-r--r-- | libpod/oci.go | 92 | ||||
-rw-r--r-- | libpod/runtime_ctr.go | 12 | ||||
-rw-r--r-- | libpod/sql_state.go | 36 | ||||
-rw-r--r-- | libpod/sql_state_internal.go | 11 |
6 files changed, 197 insertions, 60 deletions
diff --git a/libpod/container.go b/libpod/container.go index 728a29dec..710ae866b 100644 --- a/libpod/container.go +++ b/libpod/container.go @@ -76,6 +76,11 @@ type containerRuntimeInfo struct { FinishedTime time.Time `json:"finishedTime,omitempty"` // ExitCode is the exit code returned when the container stopped ExitCode int32 `json:"exitCode,omitempty"` + // OOMKilled indicates that the container was killed as it ran out of + // memory + OOMKilled bool `json:"oomKilled,omitempty"` + // PID is the PID of a running container + PID int `json:"pid,omitempty"` // TODO: Save information about image used in container if one is used } @@ -129,10 +134,10 @@ func (c *Container) Name() string { // The spec returned is the one used to create the container. The running // spec may differ slightly as mounts are added based on the image func (c *Container) Spec() *spec.Spec { - spec := new(spec.Spec) - deepcopier.Copy(c.config.Spec).To(spec) + returnSpec := new(spec.Spec) + deepcopier.Copy(c.config.Spec).To(returnSpec) - return spec + return returnSpec } // Labels returns the container's labels @@ -150,13 +155,26 @@ func (c *Container) State() (ContainerState, error) { c.lock.Lock() defer c.lock.Unlock() - if err := c.runtime.state.UpdateContainer(c); err != nil { - return ContainerStateUnknown, errors.Wrapf(err, "error updating container %s state", c.ID()) + if err := c.syncContainer(); err != nil { + return ContainerStateUnknown, err } return c.state.State, nil } +// PID returns the PID of the container +// An error is returned if the container is not running +func (c *Container) PID() (int, error) { + c.lock.Lock() + defer c.lock.Unlock() + + if err := c.syncContainer(); err != nil { + return -1, err + } + + return c.state.PID, nil +} + // The path to the container's root filesystem - where the OCI spec will be // placed, amongst other things func (c *Container) bundlePath() string { @@ -168,6 +186,32 @@ func (c *Container) attachSocketPath() string { return filepath.Join(c.runtime.ociRuntime.socketsDir, c.ID(), "attach") } +// Sync this container with on-disk state and runc status +// Should only be called with container lock held +func (c *Container) syncContainer() error { + if err := c.runtime.state.UpdateContainer(c); err != nil { + return err + } + + // If runc knows about the container, update its status in runc + // And then save back to disk + if (c.state.State != ContainerStateUnknown) && + (c.state.State != ContainerStateConfigured) { + if err := c.runtime.ociRuntime.updateContainerStatus(c); err != nil { + return err + } + if err := c.runtime.state.SaveContainer(c); err != nil { + return err + } + } + + if !c.valid { + return errors.Wrapf(ErrCtrRemoved, "container %s is not valid", c.ID()) + } + + return nil +} + // Make a new container func newContainer(rspec *spec.Spec) (*Container, error) { if rspec == nil { @@ -191,9 +235,6 @@ func newContainer(rspec *spec.Spec) (*Container, error) { // Create container root filesystem for use func (c *Container) setupStorage() error { - c.lock.Lock() - defer c.lock.Unlock() - if !c.valid { return errors.Wrapf(ErrCtrRemoved, "container %s is not valid", c.ID()) } @@ -220,9 +261,6 @@ func (c *Container) setupStorage() error { // Tear down a container's storage prior to removal func (c *Container) teardownStorage() error { - c.lock.Lock() - defer c.lock.Unlock() - if !c.valid { return errors.Wrapf(ErrCtrRemoved, "container %s is not valid", c.ID()) } @@ -251,12 +289,8 @@ func (c *Container) Init() (err error) { c.lock.Lock() defer c.lock.Unlock() - if err := c.runtime.state.UpdateContainer(c); err != nil { - return errors.Wrapf(err, "error updating container %s state", c.ID()) - } - - if !c.valid { - return errors.Wrapf(ErrCtrRemoved, "container %s is not valid", c.ID()) + if err := c.syncContainer(); err != nil { + return err } if c.state.State != ContainerStateConfigured { @@ -325,12 +359,8 @@ func (c *Container) Start() error { c.lock.Lock() defer c.lock.Unlock() - if err := c.runtime.state.UpdateContainer(c); err != nil { - return errors.Wrapf(err, "error updating container %s state", c.ID()) - } - - if !c.valid { - return ErrCtrRemoved + if err := c.syncContainer(); err != nil { + return err } // Container must be created or stopped to be started @@ -344,8 +374,10 @@ func (c *Container) Start() error { logrus.Debugf("Started container %s", c.ID()) - c.state.StartedTime = time.Now() - c.state.State = ContainerStateRunning + // Update container's state as it should be ContainerStateRunning now + if err := c.runtime.ociRuntime.updateContainerStatus(c); err != nil { + return err + } if err := c.runtime.state.SaveContainer(c); err != nil { return errors.Wrapf(err, "error saving container %s state", c.ID()) @@ -373,12 +405,8 @@ func (c *Container) Exec(cmd []string, tty bool, stdin bool) (string, error) { // Attach attaches to a container // Returns fully qualified URL of streaming server for the container func (c *Container) Attach(noStdin bool, keys string, attached chan<- bool) error { - if err := c.runtime.state.UpdateContainer(c); err != nil { - return errors.Wrapf(err, "error updating container %s state", c.ID()) - } - - if !c.valid { - return errors.Wrapf(ErrCtrRemoved, "container %s is not valid", c.ID()) + if err := c.syncContainer(); err != nil { + return err } if c.state.State == ContainerStateRunning || c.state.State == ContainerStatePaused { diff --git a/libpod/finished_amd64.go b/libpod/finished_amd64.go new file mode 100644 index 000000000..11dbc9181 --- /dev/null +++ b/libpod/finished_amd64.go @@ -0,0 +1,16 @@ +// +build !arm,!386 + +package libpod + +import ( + "os" + "syscall" + "time" +) + +// Get the created time of a file +// Only works on 64-bit OSes +func getFinishedTime(fi os.FileInfo) time.Time { + st := fi.Sys().(*syscall.Stat_t) + return time.Unix(st.Ctim.Sec, st.Ctim.Nsec) +} diff --git a/libpod/oci.go b/libpod/oci.go index 9fa2d3788..3ad3fdd8e 100644 --- a/libpod/oci.go +++ b/libpod/oci.go @@ -4,9 +4,11 @@ import ( "bytes" "encoding/json" "fmt" + "io/ioutil" "os" "os/exec" "path/filepath" + "strconv" "syscall" "time" @@ -15,6 +17,7 @@ import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" "golang.org/x/sys/unix" + kwait "k8s.io/apimachinery/pkg/util/wait" // TODO import these functions into libpod and remove the import // Trying to keep libpod from depending on CRI-O code @@ -255,19 +258,90 @@ func (r *OCIRuntime) createContainer(ctr *Container, cgroupParent string) error } // updateContainerStatus retrieves the current status of the container from the -// runtime -// remove nolint when implemented -func (r *OCIRuntime) updateContainerStatus(ctr *Container) error { //nolint - return ErrNotImplemented +// runtime. It updates the container's state but does not save it. +func (r *OCIRuntime) updateContainerStatus(ctr *Container) error { + state := new(spec.State) + + out, err := exec.Command(r.path, "state", ctr.ID()).CombinedOutput() + if err != nil { + return errors.Wrapf(err, "error getting container %s state. stderr/out: %s", ctr.ID(), out) + } + + if err := json.NewDecoder(bytes.NewBuffer(out)).Decode(state); err != nil { + return errors.Wrapf(err, "error decoding container status for container %s", ctr.ID()) + } + + ctr.state.PID = state.Pid + + switch state.Status { + case "created": + ctr.state.State = ContainerStateCreated + case "paused": + ctr.state.State = ContainerStatePaused + case "running": + ctr.state.State = ContainerStateRunning + case "stopped": + ctr.state.State = ContainerStateStopped + default: + return errors.Wrapf(ErrInternal, "unrecognized status returned by runc for container %s: %s", + ctr.ID(), state.Status) + } + + if ctr.state.State == ContainerStateStopped { + exitFile := filepath.Join(r.exitsDir, ctr.ID()) + var fi os.FileInfo + err = kwait.ExponentialBackoff( + kwait.Backoff{ + Duration: 500 * time.Millisecond, + Factor: 1.2, + Steps: 6, + }, + func() (bool, error) { + var err error + fi, err = os.Stat(exitFile) + if err != nil { + // wait longer + return false, nil + } + return true, nil + }) + if err != nil { + ctr.state.ExitCode = -1 + ctr.state.FinishedTime = time.Now() + return errors.Wrapf(err, "no exit file for container %s found", ctr.ID()) + } + + ctr.state.FinishedTime = getFinishedTime(fi) + statusCodeStr, err := ioutil.ReadFile(exitFile) + if err != nil { + return errors.Wrapf(err, "failed to read exit file for container %s", ctr.ID()) + } + statusCode, err := strconv.Atoi(string(statusCodeStr)) + if err != nil { + return errors.Wrapf(err, "error convertaing exit status code for container %s to int", + ctr.ID()) + } + ctr.state.ExitCode = int32(statusCode) + + oomFilePath := filepath.Join(ctr.bundlePath(), "oom") + if _, err = os.Stat(oomFilePath); err == nil { + ctr.state.OOMKilled = true + } + + } + + return nil } // startContainer starts the given container -// remove nolint when function is complete -func (r *OCIRuntime) startContainer(ctr *Container) error { //nolint +// Sets time the container was started, but does not save it. +func (r *OCIRuntime) startContainer(ctr *Container) error { // TODO: streams should probably *not* be our STDIN/OUT/ERR - redirect to buffers? - err := utils.ExecCmdWithStdStreams(os.Stdin, os.Stdout, os.Stderr, r.path, "start", ctr.ID()) + if err := utils.ExecCmdWithStdStreams(os.Stdin, os.Stdout, os.Stderr, r.path, "start", ctr.ID()); err != nil { + return err + } - // TODO record start time in container struct + ctr.state.StartedTime = time.Now() - return err + return nil } diff --git a/libpod/runtime_ctr.go b/libpod/runtime_ctr.go index aa8ff7d88..fd92a3411 100644 --- a/libpod/runtime_ctr.go +++ b/libpod/runtime_ctr.go @@ -91,12 +91,8 @@ func (r *Runtime) RemoveContainer(c *Container, force bool) error { return ErrRuntimeStopped } - if !c.valid { - return ErrCtrRemoved - } - // Update the container to get current state - if err := r.state.UpdateContainer(c); err != nil { + if err := c.syncContainer(); err != nil { return err } @@ -225,9 +221,3 @@ func (r *Runtime) removeMultipleContainers(containers []storage.Container) error } return nil } - -// ContainerConfigToDisk saves a container's nonvolatile configuration to disk -// remove nolint when implemented -func (r *Runtime) containerConfigToDisk(ctr *Container) error { //nolint - return ErrNotImplemented -} diff --git a/libpod/sql_state.go b/libpod/sql_state.go index 466ad66a5..893223914 100644 --- a/libpod/sql_state.go +++ b/libpod/sql_state.go @@ -103,7 +103,9 @@ func (s *SQLState) Container(id string) (*Container, error) { containerState.MountPoint, containerState.StartedTime, containerState.FinishedTime, - containerState.ExitCode + containerState.ExitCode, + containerState.OomKilled, + containerState.Pid FROM containers INNER JOIN containerState ON containers.Id = containerState.Id @@ -136,7 +138,9 @@ func (s *SQLState) LookupContainer(idOrName string) (*Container, error) { containerState.MountPoint, containerState.StartedTime, containerState.FinishedTime, - containerState.ExitCode + containerState.ExitCode, + containerState.OomKilled, + containerState.Pid FROM containers INNER JOIN containerState ON containers.Id = containerState.Id @@ -220,7 +224,7 @@ func (s *SQLState) AddContainer(ctr *Container) (err error) { ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? );` addCtrState = `INSERT INTO containerState VALUES ( - ?, ?, ?, ?, ?, ?, ?, ? + ?, ?, ?, ?, ?, ?, ?, ?, ?, ? );` ) @@ -278,7 +282,9 @@ func (s *SQLState) AddContainer(ctr *Container) (err error) { ctr.state.Mountpoint, timeToSQL(ctr.state.StartedTime), timeToSQL(ctr.state.FinishedTime), - ctr.state.ExitCode) + ctr.state.ExitCode, + boolToSQL(ctr.state.OOMKilled), + ctr.state.PID) if err != nil { return errors.Wrapf(err, "error adding container %s state to database", ctr.ID()) } @@ -315,7 +321,9 @@ func (s *SQLState) UpdateContainer(ctr *Container) error { Mountpoint, StartedTime, FinishedTime, - ExitCode + ExitCode, + OomKilled, + Pid FROM containerState WHERE ID=?;` var ( @@ -326,6 +334,8 @@ func (s *SQLState) UpdateContainer(ctr *Container) error { startedTimeString string finishedTimeString string exitCode int32 + oomKilled int + pid int ) if !s.valid { @@ -344,7 +354,9 @@ func (s *SQLState) UpdateContainer(ctr *Container) error { &mountpoint, &startedTimeString, &finishedTimeString, - &exitCode) + &exitCode, + &oomKilled, + &pid) if err != nil { // The container may not exist in the database if err == sql.ErrNoRows { @@ -364,6 +376,8 @@ func (s *SQLState) UpdateContainer(ctr *Container) error { newState.RunDir = runDir newState.Mountpoint = mountpoint newState.ExitCode = exitCode + newState.OOMKilled = boolFromSQL(oomKilled) + newState.PID = pid if newState.Mountpoint != "" { newState.Mounted = true @@ -396,7 +410,9 @@ func (s *SQLState) SaveContainer(ctr *Container) error { Mountpoint=?, StartedTime=?, FinishedTime=?, - ExitCode=? + ExitCode=?, + OomKilled=?, + Pid=? WHERE Id=?;` s.lock.Lock() @@ -431,6 +447,8 @@ func (s *SQLState) SaveContainer(ctr *Container) error { timeToSQL(ctr.state.StartedTime), timeToSQL(ctr.state.FinishedTime), ctr.state.ExitCode, + boolToSQL(ctr.state.OOMKilled), + ctr.state.PID, ctr.ID()) if err != nil { return errors.Wrapf(err, "error updating container %s state in database", ctr.ID()) @@ -521,7 +539,9 @@ func (s *SQLState) AllContainers() ([]*Container, error) { containerState.MountPoint, containerState.StartedTime, containerState.FinishedTime, - containerState.ExitCode + containerState.ExitCode, + containerState.OomKilled, + containerState.Pid FROM containers INNER JOIN containerState ON containers.Id = containerState.Id;` diff --git a/libpod/sql_state_internal.go b/libpod/sql_state_internal.go index 698b0433c..b6816e008 100644 --- a/libpod/sql_state_internal.go +++ b/libpod/sql_state_internal.go @@ -61,7 +61,10 @@ func prepareDB(db *sql.DB) (err error) { StartedTime TEXT NUT NULL, FinishedTime TEXT NOT NULL, ExitCode INTEGER NOT NULL, + OomKilled INTEGER NOT NULL, + Pid INTEGER NOT NULL, CHECK (State>0), + CHECK (OomKilled IN (0, 1)), FOREIGN KEY (Id) REFERENCES containers(Id) DEFERRABLE INITIALLY DEFERRED ); ` @@ -149,6 +152,8 @@ func ctrFromScannable(row scannable, runtime *Runtime, specsDir string) (*Contai startedTimeString string finishedTimeString string exitCode int32 + oomKilled int + pid int ) err := row.Scan( @@ -169,7 +174,9 @@ func ctrFromScannable(row scannable, runtime *Runtime, specsDir string) (*Contai &mountpoint, &startedTimeString, &finishedTimeString, - &exitCode) + &exitCode, + &oomKilled, + &pid) if err != nil { if err == sql.ErrNoRows { return nil, ErrNoSuchCtr @@ -197,6 +204,8 @@ func ctrFromScannable(row scannable, runtime *Runtime, specsDir string) (*Contai ctr.state.RunDir = runDir ctr.state.Mountpoint = mountpoint ctr.state.ExitCode = exitCode + ctr.state.OOMKilled = boolFromSQL(oomKilled) + ctr.state.PID = pid // TODO should we store this in the database separately instead? if ctr.state.Mountpoint != "" { |