Add an API for Attach over HTTP API

The new APIv2 branch provides an HTTP-based remote API to Podman.
The requirements of this are, unfortunately, incompatible with
the existing Attach API. For non-terminal attach, we need append
a header to what was copied from the container, to multiplex
STDOUT and STDERR; to do this with the old API, we'd need to copy
into an intermediate buffer first, to handle the headers.

To avoid this, provide a new API to handle all aspects of
terminal and non-terminal attach, including closing the hijacked
HTTP connection. This might be a bit too specific, but for now,
it seems to be the simplest approach.

At the same time, add a Resize endpoint. This needs to be a
separate endpoint, so our existing channel approach does not work
here.

I wanted to rework the rest of attach at the same time (some
parts of it, particularly how we start the Attach session and how
we do resizing, are (in my opinion) handled much better here.
That may still be on the table, but I wanted to avoid breaking
existing APIs in this already massive change.

Signed-off-by: Matthew Heon <matthew.heon@pm.me>

14 files changed, 950 insertions, 2 deletions

diff --git a/go.mod b/go.mod
index dff9ab4f3..1f89eb89f 100644
--- a/go.mod
+++ b/go.mod
@@ -11,6 +11,7 @@ require (
 	github.com/containernetworking/cni v0.7.2-0.20190904153231-83439463f784
 	github.com/containernetworking/plugins v0.8.2
 	github.com/containers/buildah v1.13.1
+	github.com/containers/conmon v2.0.9+incompatible
 	github.com/containers/image/v5 v5.1.0
 	github.com/containers/psgo v1.4.0
 	github.com/containers/storage v1.15.5
diff --git a/go.sum b/go.sum
index f3613ce2e..f1d6f1c78 100644
--- a/go.sum
+++ b/go.sum
@@ -81,6 +81,8 @@ github.com/containers/common v0.0.3 h1:C2Zshb0w720FqPa42MCRuiGfbW0kwbURRwvK1EWIC
 github.com/containers/common v0.0.3/go.mod h1:CaOgMRiwi2JJHISMZ6VPPZhQYFUDRv3YYVss2RqUCMg=
 github.com/containers/common v0.0.7 h1:eKYZLKfJ2d/RNDgecLDFv45cHb4imYzIcrQHx1Y029M=
 github.com/containers/common v0.0.7/go.mod h1:lhWV3MLhO1+KGE2x6v9+K38MxpjXGso+edmpkFnCOqI=
+github.com/containers/conmon v2.0.9+incompatible h1:YcEgk0Ny1WBdH35M2LKe2cG6FiQqzDdVaURw84XvS7A=
+github.com/containers/conmon v2.0.9+incompatible/go.mod h1:hgwZ2mtuDrppv78a/cOBNiCm6O0UMWGx1mu7P00nu5I=
 github.com/containers/image/v5 v5.0.0 h1:arnXgbt1ucsC/ndtSpiQY87rA0UjhF+/xQnPzqdBDn4=
 github.com/containers/image/v5 v5.0.0/go.mod h1:MgiLzCfIeo8lrHi+4Lb8HP+rh513sm0Mlk6RrhjFOLY=
 github.com/containers/image/v5 v5.1.0 h1:5FjAvPJniamuNNIQHkh4PnsL+n+xzs6Aonzaz5dqTEo=
diff --git a/libpod/container_api.go b/libpod/container_api.go
index e36623529..d74a14f15 100644
--- a/libpod/container_api.go
+++ b/libpod/container_api.go
@@ -5,6 +5,7 @@ import (
 	"context"
 	"io"
 	"io/ioutil"
+	"net"
 	"os"
 	"time"
 
@@ -374,7 +375,9 @@ type AttachStreams struct {
 	AttachInput bool
 }
 
-// Attach attaches to a container
+// Attach attaches to a container.
+// This function returns when the attach finishes. It does not hold the lock for
+// the duration of its runtime, only using it at the beginning to verify state.
 func (c *Container) Attach(streams *AttachStreams, keys string, resize <-chan remotecommand.TerminalSize) error {
 	if !c.batched {
 		c.lock.Lock()
@@ -382,6 +385,7 @@ func (c *Container) Attach(streams *AttachStreams, keys string, resize <-chan re
 			c.lock.Unlock()
 			return err
 		}
+		// We are NOT holding the lock for the duration of the function.
 		c.lock.Unlock()
 	}
 
@@ -389,10 +393,71 @@ func (c *Container) Attach(streams *AttachStreams, keys string, resize <-chan re
 		return errors.Wrapf(define.ErrCtrStateInvalid, "can only attach to created or running containers")
 	}
 
-	defer c.newContainerEvent(events.Attach)
+	c.newContainerEvent(events.Attach)
 	return c.attach(streams, keys, resize, false, nil)
 }
 
+// HTTPAttach forwards an attach session over a hijacked HTTP session.
+// HTTPAttach will consume and close the included httpCon, which is expected to
+// be sourced from a hijacked HTTP connection.
+// The cancel channel is optional, and can be used to asyncronously cancel the
+// attach session.
+// The streams variable is only supported if the container was not a terminal,
+// and allows specifying which of the container's standard streams will be
+// forwarded to the client.
+// This function returns when the attach finishes. It does not hold the lock for
+// the duration of its runtime, only using it at the beginning to verify state.
+func (c *Container) HTTPAttach(httpCon net.Conn, httpBuf *bufio.ReadWriter, streams *HTTPAttachStreams, detachKeys *string, cancel <-chan bool) error {
+	if !c.batched {
+		c.lock.Lock()
+		if err := c.syncContainer(); err != nil {
+			c.lock.Unlock()
+
+			// Write any errors to the HTTP buffer before we close.
+			hijackWriteErrorAndClose(err, c.ID(), httpCon, httpBuf)
+
+			return err
+		}
+		// We are NOT holding the lock for the duration of the function.
+		c.lock.Unlock()
+	}
+
+	if !c.ensureState(define.ContainerStateCreated, define.ContainerStateRunning) {
+		toReturn := errors.Wrapf(define.ErrCtrStateInvalid, "can only attach to created or running containers")
+
+		// Write any errors to the HTTP buffer before we close.
+		hijackWriteErrorAndClose(toReturn, c.ID(), httpCon, httpBuf)
+
+		return toReturn
+	}
+
+	logrus.Infof("Performing HTTP Hijack attach to container %s", c.ID())
+
+	c.newContainerEvent(events.Attach)
+	return c.ociRuntime.HTTPAttach(c, httpCon, httpBuf, streams, detachKeys, cancel)
+}
+
+// AttachResize resizes the container's terminal, which is displayed by Attach
+// and HTTPAttach.
+func (c *Container) AttachResize(newSize remotecommand.TerminalSize) error {
+	if !c.batched {
+		c.lock.Lock()
+		defer c.lock.Unlock()
+
+		if err := c.syncContainer(); err != nil {
+			return err
+		}
+	}
+
+	if !c.ensureState(define.ContainerStateCreated, define.ContainerStateRunning) {
+		return errors.Wrapf(define.ErrCtrStateInvalid, "can only resize created or running containers")
+	}
+
+	logrus.Infof("Resizing TTY of container %s", c.ID())
+
+	return c.ociRuntime.AttachResize(c, newSize)
+}
+
 // Mount mounts a container's filesystem on the host
 // The path where the container has been mounted is returned
 func (c *Container) Mount() (string, error) {
diff --git a/libpod/oci.go b/libpod/oci.go
index 05a2f37db..2ea61851f 100644
--- a/libpod/oci.go
+++ b/libpod/oci.go
@@ -1,6 +1,9 @@
 package libpod
 
 import (
+	"bufio"
+	"net"
+
 	"k8s.io/client-go/tools/remotecommand"
 )
 
@@ -47,6 +50,23 @@ type OCIRuntime interface {
 	// UnpauseContainer unpauses the given container.
 	UnpauseContainer(ctr *Container) error
 
+	// HTTPAttach performs an attach intended to be transported over HTTP.
+	// For terminal attach, the container's output will be directly streamed
+	// to output; otherwise, STDOUT and STDERR will be multiplexed, with
+	// a header prepended as follows: 1-byte STREAM (0, 1, 2 for STDIN,
+	// STDOUT, STDERR), 3 null (0x00) bytes, 4-byte big endian length.
+	// If a cancel channel is provided, it can be used to asyncronously
+	// termninate the attach session. Detach keys, if given, will also cause
+	// the attach session to be terminated if provided via the STDIN
+	// channel. If they are not provided, the default detach keys will be
+	// used instead. Detach keys of "" will disable detaching via keyboard.
+	// The streams parameter may be passed for containers that did not
+	// create a terminal and will determine which streams to forward to the
+	// client.
+	HTTPAttach(ctr *Container, httpConn net.Conn, httpBuf *bufio.ReadWriter, streams *HTTPAttachStreams, detachKeys *string, cancel <-chan bool) error
+	// AttachResize resizes the terminal in use by the given container.
+	AttachResize(ctr *Container, newSize remotecommand.TerminalSize) error
+
 	// ExecContainer executes a command in a running container.
 	// Returns an int (exit code), error channel (errors from attach), and
 	// error (errors that occurred attempting to start the exec session).
@@ -130,3 +150,12 @@ type ExecOptions struct {
 	// detach from the container.
 	DetachKeys string
 }
+
+// HTTPAttachStreams informs the HTTPAttach endpoint which of the container's
+// standard streams should be streamed to the client. If this is passed, at
+// least one of the streams must be set to true.
+type HTTPAttachStreams struct {
+	Stdin  bool
+	Stdout bool
+	Stderr bool
+}
diff --git a/libpod/oci_conmon_linux.go b/libpod/oci_conmon_linux.go
index 5ab0e73c4..0e8a64865 100644
--- a/libpod/oci_conmon_linux.go
+++ b/libpod/oci_conmon_linux.go
@@ -5,8 +5,11 @@ package libpod
 import (
 	"bufio"
 	"bytes"
+	"encoding/binary"
 	"fmt"
+	"io"
 	"io/ioutil"
+	"net"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -17,6 +20,7 @@ import (
 	"text/template"
 	"time"
 
+	conmonConfig "github.com/containers/conmon/runner/config"
 	"github.com/containers/libpod/libpod/config"
 	"github.com/containers/libpod/libpod/define"
 	"github.com/containers/libpod/pkg/cgroups"
@@ -33,6 +37,13 @@ import (
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/sys/unix"
+	"k8s.io/client-go/tools/remotecommand"
+)
+
+const (
+	// This is Conmon's STDIO_BUF_SIZE. I don't believe we have access to it
+	// directly from the Go cose, so const it here
+	bufferSize = conmonConfig.BufSize
 )
 
 // ConmonOCIRuntime is an OCI runtime managed by Conmon.
@@ -465,6 +476,123 @@ func (r *ConmonOCIRuntime) UnpauseContainer(ctr *Container) error {
 	return utils.ExecCmdWithStdStreams(os.Stdin, os.Stdout, os.Stderr, env, r.path, "resume", ctr.ID())
 }
 
+// HTTPAttach performs an attach for the HTTP API.
+// This will consume, and automatically close, the hijacked HTTP session.
+// It is not necessary to close it independently.
+// The cancel channel is not closed; it is up to the caller to do so after
+// this function returns.
+// If this is a container with a terminal, we will stream raw. If it is not, we
+// will stream with an 8-byte header to multiplex STDOUT and STDERR.
+func (r *ConmonOCIRuntime) HTTPAttach(ctr *Container, httpConn net.Conn, httpBuf *bufio.ReadWriter, streams *HTTPAttachStreams, detachKeys *string, cancel <-chan bool) (deferredErr error) {
+	isTerminal := false
+	if ctr.config.Spec.Process != nil {
+		isTerminal = ctr.config.Spec.Process.Terminal
+	}
+
+	// Ensure that our contract of closing the HTTP connection is honored.
+	defer hijackWriteErrorAndClose(deferredErr, ctr.ID(), httpConn, httpBuf)
+
+	if streams != nil {
+		if isTerminal {
+			return errors.Wrapf(define.ErrInvalidArg, "cannot specify which streams to attach as container %s has a terminal", ctr.ID())
+		}
+		if !streams.Stdin && !streams.Stdout && !streams.Stderr {
+			return errors.Wrapf(define.ErrInvalidArg, "must specify at least one stream to attach to")
+		}
+	}
+
+	attachSock, err := r.AttachSocketPath(ctr)
+	if err != nil {
+		return err
+	}
+	socketPath := buildSocketPath(attachSock)
+
+	conn, err := net.DialUnix("unixpacket", nil, &net.UnixAddr{Name: socketPath, Net: "unixpacket"})
+	if err != nil {
+		return errors.Wrapf(err, "failed to connect to container's attach socket: %v", socketPath)
+	}
+	defer func() {
+		if err := conn.Close(); err != nil {
+			logrus.Errorf("unable to close container %s attach socket: %q", ctr.ID(), err)
+		}
+	}()
+
+	logrus.Debugf("Successfully connected to container %s attach socket %s", ctr.ID(), socketPath)
+
+	detachString := define.DefaultDetachKeys
+	if detachKeys != nil {
+		detachString = *detachKeys
+	}
+	detach, err := processDetachKeys(detachString)
+	if err != nil {
+		return err
+	}
+
+	// Make a channel to pass errors back
+	errChan := make(chan error)
+
+	attachStdout := true
+	attachStderr := true
+	attachStdin := true
+	if streams != nil {
+		attachStdout = streams.Stdout
+		attachStderr = streams.Stderr
+		attachStdin = streams.Stdin
+	}
+
+	// Handle STDOUT/STDERR
+	go func() {
+		var err error
+		if isTerminal {
+			logrus.Debugf("Performing terminal HTTP attach for container %s", ctr.ID())
+			err = httpAttachTerminalCopy(conn, httpBuf, ctr.ID())
+		} else {
+			logrus.Debugf("Performing non-terminal HTTP attach for container %s", ctr.ID())
+			err = httpAttachNonTerminalCopy(conn, httpBuf, ctr.ID(), attachStdin, attachStdout, attachStderr)
+		}
+		errChan <- err
+		logrus.Debugf("STDOUT/ERR copy completed")
+	}()
+	// Next, STDIN. Avoid entirely if attachStdin unset.
+	if attachStdin {
+		go func() {
+			_, err := utils.CopyDetachable(conn, httpBuf, detach)
+			logrus.Debugf("STDIN copy completed")
+			errChan <- err
+		}()
+	}
+
+	if cancel != nil {
+		select {
+		case err := <-errChan:
+			return err
+		case <-cancel:
+			return nil
+		}
+	} else {
+		var connErr error = <-errChan
+		return connErr
+	}
+}
+
+// AttachResize resizes the terminal used by the given container.
+func (r *ConmonOCIRuntime) AttachResize(ctr *Container, newSize remotecommand.TerminalSize) error {
+	// TODO: probably want a dedicated function to get ctl file path?
+	controlPath := filepath.Join(ctr.bundlePath(), "ctl")
+	controlFile, err := os.OpenFile(controlPath, unix.O_WRONLY, 0)
+	if err != nil {
+		return errors.Wrapf(err, "could not open ctl file for terminal resize")
+	}
+	defer controlFile.Close()
+
+	logrus.Debugf("Received a resize event for container %s: %+v", ctr.ID(), newSize)
+	if _, err = fmt.Fprintf(controlFile, "%d %d %d\n", 1, newSize.Height, newSize.Width); err != nil {
+		return errors.Wrapf(err, "failed to write to ctl file to resize terminal")
+	}
+
+	return nil
+}
+
 // ExecContainer executes a command in a running container
 // TODO: Split into Create/Start/Attach/Wait
 func (r *ConmonOCIRuntime) ExecContainer(c *Container, sessionID string, options *ExecOptions) (int, chan error, error) {
@@ -1465,3 +1593,139 @@ func (r *ConmonOCIRuntime) getOCIRuntimeVersion() (string, error) {
 	}
 	return strings.TrimSuffix(output, "\n"), nil
 }
+
+// Copy data from container to HTTP connection, for terminal attach.
+// Container is the container's attach socket connection, http is a buffer for
+// the HTTP connection. cid is the ID of the container the attach session is
+// running for (used solely for error messages).
+func httpAttachTerminalCopy(container *net.UnixConn, http *bufio.ReadWriter, cid string) error {
+	buf := make([]byte, bufferSize)
+	for {
+		numR, err := container.Read(buf)
+		if numR > 0 {
+			switch buf[0] {
+			case AttachPipeStdout:
+				// Do nothing
+			default:
+				logrus.Errorf("Received unexpected attach type %+d, discarding %d bytes", buf[0], numR)
+				continue
+			}
+
+			numW, err2 := http.Write(buf[1:numR])
+			if err2 != nil {
+				if err != nil {
+					logrus.Errorf("Error reading container %s STDOUT: %v", cid, err)
+				}
+				return err2
+			} else if numW+1 != numR {
+				return io.ErrShortWrite
+			}
+			// We need to force the buffer to write immediately, so
+			// there isn't a delay on the terminal side.
+			if err2 := http.Flush(); err2 != nil {
+				if err != nil {
+					logrus.Errorf("Error reading container %s STDOUT: %v", cid, err)
+				}
+				return err2
+			}
+		}
+		if err != nil {
+			if err == io.EOF {
+				return nil
+			}
+			return err
+		}
+	}
+}
+
+// Copy data from a container to an HTTP connection, for non-terminal attach.
+// Appends a header to multiplex input.
+func httpAttachNonTerminalCopy(container *net.UnixConn, http *bufio.ReadWriter, cid string, stdin, stdout, stderr bool) error {
+	buf := make([]byte, bufferSize)
+	for {
+		numR, err := container.Read(buf)
+		if numR > 0 {
+			headerBuf := []byte{0, 0, 0, 0}
+
+			// Practically speaking, we could make this buf[0] - 1,
+			// but we need to validate it anyways...
+			switch buf[0] {
+			case AttachPipeStdin:
+				headerBuf[0] = 0
+				if !stdin {
+					continue
+				}
+			case AttachPipeStdout:
+				if !stdout {
+					continue
+				}
+				headerBuf[0] = 1
+			case AttachPipeStderr:
+				if !stderr {
+					continue
+				}
+				headerBuf[0] = 2
+			default:
+				logrus.Errorf("Received unexpected attach type %+d, discarding %d bytes", buf[0], numR)
+				continue
+			}
+
+			// Get big-endian length and append.
+			// Subtract 1 because we strip the first byte (used for
+			// multiplexing by Conmon).
+			lenBuf := []byte{0, 0, 0, 0}
+			binary.BigEndian.PutUint32(lenBuf, uint32(numR-1))
+			headerBuf = append(headerBuf, lenBuf...)
+
+			numH, err2 := http.Write(headerBuf)
+			if err2 != nil {
+				if err != nil {
+					logrus.Errorf("Error reading container %s standard streams: %v", cid, err)
+				}
+
+				return err2
+			}
+			// Hardcoding header length is pretty gross, but
+			// fast. Should be safe, as this is a fixed part
+			// of the protocol.
+			if numH != 8 {
+				if err != nil {
+					logrus.Errorf("Error reading container %s standard streams: %v", cid, err)
+				}
+
+				return io.ErrShortWrite
+			}
+
+			numW, err2 := http.Write(buf[1:numR])
+			if err2 != nil {
+				if err != nil {
+					logrus.Errorf("Error reading container %s standard streams: %v", cid, err)
+				}
+
+				return err2
+			} else if numW+1 != numR {
+				if err != nil {
+					logrus.Errorf("Error reading container %s standard streams: %v", cid, err)
+				}
+
+				return io.ErrShortWrite
+			}
+			// We need to force the buffer to write immediately, so
+			// there isn't a delay on the terminal side.
+			if err2 := http.Flush(); err2 != nil {
+				if err != nil {
+					logrus.Errorf("Error reading container %s STDOUT: %v", cid, err)
+				}
+				return err2
+			}
+		}
+		if err != nil {
+			if err == io.EOF {
+				return nil
+			}
+
+			return err
+		}
+	}
+
+}
diff --git a/libpod/oci_missing.go b/libpod/oci_missing.go
index 0faa1805b..ff7eea625 100644
--- a/libpod/oci_missing.go
+++ b/libpod/oci_missing.go
@@ -1,13 +1,16 @@
 package libpod
 
 import (
+	"bufio"
 	"fmt"
+	"net"
 	"path/filepath"
 	"sync"
 
 	"github.com/containers/libpod/libpod/define"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
+	"k8s.io/client-go/tools/remotecommand"
 )
 
 var (
@@ -107,6 +110,16 @@ func (r *MissingRuntime) UnpauseContainer(ctr *Container) error {
 	return r.printError()
 }
 
+// HTTPAttach is not available as the runtime is missing
+func (r *MissingRuntime) HTTPAttach(ctr *Container, httpConn net.Conn, httpBuf *bufio.ReadWriter, streams *HTTPAttachStreams, detachKeys *string, cancel <-chan bool) error {
+	return r.printError()
+}
+
+// AttachResize is not available as the runtime is missing
+func (r *MissingRuntime) AttachResize(ctr *Container, newSize remotecommand.TerminalSize) error {
+	return r.printError()
+}
+
 // ExecContainer is not available as the runtime is missing
 func (r *MissingRuntime) ExecContainer(ctr *Container, sessionID string, options *ExecOptions) (int, chan error, error) {
 	return -1, nil, r.printError()
diff --git a/libpod/util.go b/libpod/util.go
index 30e5cd4c3..f79d6c09b 100644
--- a/libpod/util.go
+++ b/libpod/util.go
@@ -1,7 +1,9 @@
 package libpod
 
 import (
+	"bufio"
 	"fmt"
+	"io"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -16,6 +18,7 @@ import (
 	"github.com/fsnotify/fsnotify"
 	spec "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
 )
 
 // Runtime API constants
@@ -231,3 +234,20 @@ func checkDependencyContainer(depCtr, ctr *Container) error {
 
 	return nil
 }
+
+// hijackWriteErrorAndClose writes an error to a hijacked HTTP session and
+// closes it. Intended to HTTPAttach function.
+// If error is nil, it will not be written; we'll only close the connection.
+func hijackWriteErrorAndClose(toWrite error, cid string, httpCon io.Closer, httpBuf *bufio.ReadWriter) {
+	if toWrite != nil {
+		if _, err := httpBuf.Write([]byte(toWrite.Error())); err != nil {
+			logrus.Errorf("Error writing error %q to container %s HTTP attach connection: %v", toWrite, cid, err)
+		} else if err := httpBuf.Flush(); err != nil {
+			logrus.Errorf("Error flushing HTTP buffer for container %s HTTP attach connection: %v", cid, err)
+		}
+	}
+
+	if err := httpCon.Close(); err != nil {
+		logrus.Errorf("Error closing container %s HTTP attach connection: %v", cid, err)
+	}
+}
diff --git a/pkg/api/handlers/containers_attach.go b/pkg/api/handlers/containers_attach.go
new file mode 100644
index 000000000..eb306348b
--- /dev/null
+++ b/pkg/api/handlers/containers_attach.go
@@ -0,0 +1,159 @@
+package handlers
+
+import (
+	"net/http"
+
+	"github.com/containers/libpod/libpod"
+	"github.com/containers/libpod/libpod/define"
+	"github.com/containers/libpod/pkg/api/handlers/utils"
+	"github.com/gorilla/mux"
+	"github.com/gorilla/schema"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"k8s.io/client-go/tools/remotecommand"
+)
+
+func AttachContainer(w http.ResponseWriter, r *http.Request) {
+	runtime := r.Context().Value("runtime").(*libpod.Runtime)
+	decoder := r.Context().Value("decoder").(*schema.Decoder)
+
+	query := struct {
+		DetachKeys string `schema:"detachKeys"`
+		Logs       bool   `schema:"logs"`
+		Stream     bool   `schema:"stream"`
+		Stdin      bool   `schema:"stdin"`
+		Stdout     bool   `schema:"stdout"`
+		Stderr     bool   `schema:"stderr"`
+	}{}
+	if err := decoder.Decode(&query, r.URL.Query()); err != nil {
+		utils.Error(w, "Error parsing parameters", http.StatusBadRequest, err)
+		return
+	}
+
+	muxVars := mux.Vars(r)
+
+	// Detach keys: explicitly set to "" is very different from unset
+	// TODO: Our format for parsing these may be different from Docker.
+	var detachKeys *string
+	if _, found := muxVars["detachKeys"]; found {
+		detachKeys = &query.DetachKeys
+	}
+
+	streams := new(libpod.HTTPAttachStreams)
+	streams.Stdout = true
+	streams.Stderr = true
+	streams.Stdin = true
+	useStreams := false
+	if _, found := muxVars["stdin"]; found {
+		streams.Stdin = query.Stdin
+		useStreams = true
+	}
+	if _, found := muxVars["stdout"]; found {
+		streams.Stdout = query.Stdout
+		useStreams = true
+	}
+	if _, found := muxVars["stderr"]; found {
+		streams.Stderr = query.Stderr
+		useStreams = true
+	}
+	if !useStreams {
+		streams = nil
+	}
+	if useStreams && !streams.Stdout && !streams.Stderr && !streams.Stdin {
+		utils.Error(w, "Parameter conflict", http.StatusBadRequest, errors.Errorf("at least one of stdin, stdout, stderr must be true"))
+		return
+	}
+
+	// TODO: Investigate supporting these.
+	// Logs replays container logs over the attach socket.
+	// Stream seems to break things up somehow? Not 100% clear.
+	if query.Logs {
+		utils.Error(w, "Unsupported parameter", http.StatusBadRequest, errors.Errorf("the logs parameter to attach is not presently supported"))
+		return
+	}
+	// We only support stream=true or unset
+	if _, found := muxVars["stream"]; found && query.Stream {
+		utils.Error(w, "Unsupported parameter", http.StatusBadRequest, errors.Errorf("the stream parameter to attach is not presently supported"))
+		return
+	}
+
+	name := getName(r)
+	ctr, err := runtime.LookupContainer(name)
+	if err != nil {
+		utils.ContainerNotFound(w, name, err)
+		return
+	}
+
+	state, err := ctr.State()
+	if err != nil {
+		utils.InternalServerError(w, err)
+		return
+	}
+	if !(state == define.ContainerStateCreated || state == define.ContainerStateRunning) {
+		utils.InternalServerError(w, errors.Wrapf(define.ErrCtrStateInvalid, "can only attach to created or running containers"))
+		return
+	}
+
+	// Hijack the connection
+	hijacker, ok := w.(http.Hijacker)
+	if !ok {
+		utils.InternalServerError(w, errors.Errorf("unable to hijack connection"))
+		return
+	}
+
+	w.WriteHeader(http.StatusSwitchingProtocols)
+
+	connection, buffer, err := hijacker.Hijack()
+	if err != nil {
+		utils.InternalServerError(w, errors.Wrapf(err, "error hijacking connection"))
+		return
+	}
+
+	logrus.Debugf("Hijack for attach of container %s successful", ctr.ID())
+
+	// Perform HTTP attach.
+	// HTTPAttach will handle everything about the connection from here on
+	// (including closing it and writing errors to it).
+	if err := ctr.HTTPAttach(connection, buffer, streams, detachKeys, nil); err != nil {
+		// We can't really do anything about errors anymore. HTTPAttach
+		// should be writing them to the connection.
+		logrus.Errorf("Error attaching to container %s: %v", ctr.ID(), err)
+	}
+
+	logrus.Debugf("Attach for container %s completed successfully", ctr.ID())
+}
+
+func ResizeContainer(w http.ResponseWriter, r *http.Request) {
+	runtime := r.Context().Value("runtime").(*libpod.Runtime)
+	decoder := r.Context().Value("decoder").(*schema.Decoder)
+
+	query := struct {
+		Height uint16 `schema:"h"`
+		Width  uint16 `schema:"w"`
+	}{}
+	if err := decoder.Decode(&query, r.URL.Query()); err != nil {
+		// This is not a 400, despite the fact that is should be, for
+		// compatibility reasons.
+		utils.InternalServerError(w, errors.Wrapf(err, "error parsing query options"))
+		return
+	}
+
+	name := getName(r)
+	ctr, err := runtime.LookupContainer(name)
+	if err != nil {
+		utils.ContainerNotFound(w, name, err)
+		return
+	}
+
+	newSize := remotecommand.TerminalSize{
+		Width:  query.Width,
+		Height: query.Height,
+	}
+	if err := ctr.AttachResize(newSize); err != nil {
+		utils.InternalServerError(w, err)
+		return
+	}
+	// This is not a 204, even though we write nothing, for compatibility
+	// reasons.
+	utils.WriteResponse(w, http.StatusOK, "")
+}
diff --git a/pkg/api/server/register_containers.go b/pkg/api/server/register_containers.go
index b275fa4d1..dbe194cd4 100644
--- a/pkg/api/server/register_containers.go
+++ b/pkg/api/server/register_containers.go
@@ -428,6 +428,91 @@ func (s *APIServer) RegisterContainersHandlers(r *mux.Router) error {
 	//   '500':
 	//      "$ref": "#/responses/InternalError"
 	r.HandleFunc(VersionedPath("/containers/{name:..*}/wait"), APIHandler(s.Context, generic.WaitContainer)).Methods(http.MethodPost)
+	// swagger:operation POST /containers/{nameOrID}/attach compat attach
+	// ---
+	// tags:
+	//   - containers (compat)
+	// summary: Attach to a container
+	// description: Hijacks the connection to forward the container's standard streams to the client.
+	// parameters:
+	//  - in: path
+	//    name: nameOrID
+	//    required: true
+	//    description: the name or ID of the container
+	//  - in: query
+	//    name: detachKeys
+	//    required: false
+	//    type: string
+	//    description: keys to use for detaching from the container
+	//  - in: query
+	//    name: logs
+	//    required: false
+	//    type: bool
+	//    description: Not yet supported
+	//  - in: query
+	//    name: stream
+	//    required: false
+	//    type: bool
+	//    default: true
+	//    description: If passed, must be set to true; stream=false is not yet supported
+	//  - in: query
+	//    name: stdout
+	//    required: false
+	//    type: bool
+	//    description: Attach to container STDOUT
+	//  - in: query
+	//    name: stderr
+	//    required: false
+	//    type: bool
+	//    description: Attach to container STDERR
+	//  - in: query
+	//    name: stdin
+	//    required: false
+	//    type: bool
+	//    description: Attach to container STDIN
+	// produces:
+	// - application/json
+	// responses:
+	//   '101':
+	//     description: No error, connection has been hijacked for transporting streams.
+	//   '400':
+	//       "$ref": "#/responses/BadParamError"
+	//   '404':
+	//       "$ref": "#/responses/NoSuchContainer"
+	//   '500':
+	//       "$ref": "#/responses/InternalError"
+	r.HandleFunc(VersionedPath("/containers/{name:..*}/attach"), APIHandler(s.Context, handlers.AttachContainer)).Methods(http.MethodPost)
+	// swagger:operation POST /containers/{nameOrID}/resize compat resize
+	// ---
+	// tags:
+	//  - containers (compat)
+	// summary: Resize a container's TTY
+	// description: Resize the terminal attached to a container (for use with Attach).
+	// parameters:
+	//  - in: path
+	//    name: nameOrID
+	//    required: true
+	//    description: the name or ID of the container
+	//  - in: query
+	//    name: h
+	//    type: int
+	//    required: false
+	//    description: Height to set for the terminal, in characters
+	//  - in: query
+	//    name: w
+	//    type: int
+	//    required: false
+	//    description: Width to set for the terminal, in characters
+	// produces:
+	// - application/json
+	// responses:
+	//   '200':
+	//        description: no error
+	//   '404':
+	//       "$ref": "#/responses/NoSuchContainer"
+	//   '500':
+	//       "$ref": "#/responses/InternalError"
+	r.HandleFunc(VersionedPath("/containers/{name:..*}/resize"), APIHandler(s.Context, handlers.ResizeContainer)).Methods(http.MethodPost)
 
 	/*
 		libpod endpoints
@@ -823,5 +908,90 @@ func (s *APIServer) RegisterContainersHandlers(r *mux.Router) error {
 	//   '500':
 	//      "$ref": "#/responses/InternalError"
 	r.HandleFunc(VersionedPath("/libpod/containers/{name:..*}/stop"), APIHandler(s.Context, handlers.StopContainer)).Methods(http.MethodPost)
+	// swagger:operation POST /libpod/containers/{nameOrID}/attach libpod attach
+	// ---
+	// tags:
+	//   - containers
+	// summary: Attach to a container
+	// description: Hijacks the connection to forward the container's standard streams to the client.
+	// parameters:
+	//  - in: path
+	//    name: nameOrID
+	//    required: true
+	//    description: the name or ID of the container
+	//  - in: query
+	//    name: detachKeys
+	//    required: false
+	//    type: string
+	//    description: keys to use for detaching from the container
+	//  - in: query
+	//    name: logs
+	//    required: false
+	//    type: bool
+	//    description: Not yet supported
+	//  - in: query
+	//    name: stream
+	//    required: false
+	//    type: bool
+	//    default: true
+	//    description: If passed, must be set to true; stream=false is not yet supported
+	//  - in: query
+	//    name: stdout
+	//    required: false
+	//    type: bool
+	//    description: Attach to container STDOUT
+	//  - in: query
+	//    name: stderr
+	//    required: false
+	//    type: bool
+	//    description: Attach to container STDERR
+	//  - in: query
+	//    name: stdin
+	//    required: false
+	//    type: bool
+	//    description: Attach to container STDIN
+	// produces:
+	// - application/json
+	// responses:
+	//   '101':
+	//     description: No error, connection has been hijacked for transporting streams.
+	//   '400':
+	//       "$ref": "#/responses/BadParamError"
+	//   '404':
+	//       "$ref": "#/responses/NoSuchContainer"
+	//   '500':
+	//       "$ref": "#/responses/InternalError"
+	r.HandleFunc(VersionedPath("/libpod/containers/{name:..*}/attach"), APIHandler(s.Context, handlers.AttachContainer)).Methods(http.MethodPost)
+	// swagger:operation POST /libpod/containers/{nameOrID}/resize libpod resize
+	// ---
+	// tags:
+	//  - containers
+	// summary: Resize a container's TTY
+	// description: Resize the terminal attached to a container (for use with Attach).
+	// parameters:
+	//  - in: path
+	//    name: nameOrID
+	//    required: true
+	//    description: the name or ID of the container
+	//  - in: query
+	//    name: h
+	//    type: int
+	//    required: false
+	//    description: Height to set for the terminal, in characters
+	//  - in: query
+	//    name: w
+	//    type: int
+	//    required: false
+	//    description: Width to set for the terminal, in characters
+	// produces:
+	// - application/json
+	// responses:
+	//   '200':
+	//        description: no error
+	//   '404':
+	//       "$ref": "#/responses/NoSuchContainer"
+	//   '500':
+	//       "$ref": "#/responses/InternalError"
+	r.HandleFunc(VersionedPath("/libpod/containers/{name:..*}/resize"), APIHandler(s.Context, handlers.ResizeContainer)).Methods(http.MethodPost)
 	return nil
 }
diff --git a/vendor/github.com/containers/conmon/LICENSE b/vendor/github.com/containers/conmon/LICENSE
new file mode 100644
index 000000000..28df615ce
--- /dev/null
+++ b/vendor/github.com/containers/conmon/LICENSE
@@ -0,0 +1,190 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   Copyright 2018-2019 github.com/containers authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/vendor/github.com/containers/conmon/runner/config/config.go b/vendor/github.com/containers/conmon/runner/config/config.go
new file mode 100644
index 000000000..cb70e9325
--- /dev/null
+++ b/vendor/github.com/containers/conmon/runner/config/config.go
@@ -0,0 +1,19 @@
+package config
+
+const (
+	// BufSize is the size of buffers passed in to sockets
+	BufSize = 8192
+	// ConnSockBufSize is the size of the socket used for
+	// to attach to the container
+	ConnSockBufSize = 32768
+	// WinResizeEvent is the event code the caller program will
+	// send along the ctrl fd to signal conmon to resize
+	// the pty window
+	WinResizeEvent = 1
+	// ReopenLogsEvent is the event code the caller program will
+	// send along the ctrl fd to signal conmon to reopen the log files
+	ReopenLogsEvent = 2
+	// TimedOutMessage is the message sent back to the caller by conmon
+	// when a container times out
+	TimedOutMessage = "command timed out"
+)
diff --git a/vendor/github.com/containers/conmon/runner/config/config_unix.go b/vendor/github.com/containers/conmon/runner/config/config_unix.go
new file mode 100644
index 000000000..5caaca7cc
--- /dev/null
+++ b/vendor/github.com/containers/conmon/runner/config/config_unix.go
@@ -0,0 +1,7 @@
+// +build !windows
+
+package config
+
+const (
+	ContainerAttachSocketDir = "/var/run/crio"
+)
diff --git a/vendor/github.com/containers/conmon/runner/config/config_windows.go b/vendor/github.com/containers/conmon/runner/config/config_windows.go
new file mode 100644
index 000000000..2161b9441
--- /dev/null
+++ b/vendor/github.com/containers/conmon/runner/config/config_windows.go
@@ -0,0 +1,7 @@
+// +build windows
+
+package config
+
+const (
+	ContainerAttachSocketDir = "C:\\crio\\run\\"
+)
diff --git a/vendor/modules.txt b/vendor/modules.txt
index fa01fbeec..ebac0089b 100644
--- a/vendor/modules.txt
+++ b/vendor/modules.txt
@@ -78,6 +78,8 @@ github.com/containers/buildah/util
 # github.com/containers/common v0.0.7
 github.com/containers/common/pkg/cgroups
 github.com/containers/common/pkg/unshare
+# github.com/containers/conmon v2.0.9+incompatible
+github.com/containers/conmon/runner/config
 # github.com/containers/image/v5 v5.1.0
 github.com/containers/image/v5/copy
 github.com/containers/image/v5/directory