Merge pull request #2874 from baude/varlinkterm

Add the ability to attach remotely to a container

11 files changed, 459 insertions, 56 deletions

diff --git a/API.md b/API.md
index 219e061d7..95507d189 100755
--- a/API.md
+++ b/API.md
@@ -3,6 +3,10 @@ Podman Service Interface and API description.  The master version of this docume
 in the [API.md](https://github.com/containers/libpod/blob/master/API.md) file in the upstream libpod repository.
 ## Index
 
+[func Attach(name: string) ](#Attach)
+
+[func AttachControl(name: string) ](#AttachControl)
+
 [func BuildImage(build: BuildInfo) MoreResponse](#BuildImage)
 
 [func BuildImageHierarchyMap(name: string) string](#BuildImageHierarchyMap)
@@ -135,6 +139,8 @@ in the [API.md](https://github.com/containers/libpod/blob/master/API.md) file in
 
 [func SendFile(type: string, length: int) string](#SendFile)
 
+[func Spec(name: string) string](#Spec)
+
 [func StartContainer(name: string) string](#StartContainer)
 
 [func StartPod(name: string) string](#StartPod)
@@ -252,6 +258,16 @@ in the [API.md](https://github.com/containers/libpod/blob/master/API.md) file in
 [error WantsMoreRequired](#WantsMoreRequired)
 
 ## Methods
+### <a name="Attach"></a>func Attach
+<div style="background-color: #E8E8E8; padding: 15px; margin: 10px; border-radius: 10px;">
+
+method Attach(name: [string](https://godoc.org/builtin#string)) </div>
+
+### <a name="AttachControl"></a>func AttachControl
+<div style="background-color: #E8E8E8; padding: 15px; margin: 10px; border-radius: 10px;">
+
+method AttachControl(name: [string](https://godoc.org/builtin#string)) </div>
+
 ### <a name="BuildImage"></a>func BuildImage
 <div style="background-color: #E8E8E8; padding: 15px; margin: 10px; border-radius: 10px;">
 
@@ -945,6 +961,11 @@ search results per registry.
 
 method SendFile(type: [string](https://godoc.org/builtin#string), length: [int](https://godoc.org/builtin#int)) [string](https://godoc.org/builtin#string)</div>
 Sendfile allows a remote client to send a file to the host
+### <a name="Spec"></a>func Spec
+<div style="background-color: #E8E8E8; padding: 15px; margin: 10px; border-radius: 10px;">
+
+method Spec(name: [string](https://godoc.org/builtin#string)) [string](https://godoc.org/builtin#string)</div>
+Spec returns the oci spec for a container.  This call is for development of Podman only and generally should not be used.
 ### <a name="StartContainer"></a>func StartContainer
 <div style="background-color: #E8E8E8; padding: 15px; margin: 10px; border-radius: 10px;">
 
diff --git a/cmd/podman/attach.go b/cmd/podman/attach.go
index f326f53c3..2fa05a3b1 100644
--- a/cmd/podman/attach.go
+++ b/cmd/podman/attach.go
@@ -1,11 +1,7 @@
 package main
 
 import (
-	"os"
-
 	"github.com/containers/libpod/cmd/podman/cliconfig"
-	"github.com/containers/libpod/cmd/podman/libpodruntime"
-	"github.com/containers/libpod/libpod"
 	"github.com/containers/libpod/pkg/adapter"
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
@@ -39,49 +35,21 @@ func init() {
 	flags.BoolVar(&attachCommand.SigProxy, "sig-proxy", true, "Proxy received signals to the process")
 	flags.BoolVarP(&attachCommand.Latest, "latest", "l", false, "Act on the latest container podman is aware of")
 	markFlagHiddenForRemoteClient("latest", flags)
+	// TODO allow for passing of a new deatch keys
+	markFlagHiddenForRemoteClient("detach-keys", flags)
 }
 
 func attachCmd(c *cliconfig.AttachValues) error {
-	args := c.InputArgs
-	var ctr *libpod.Container
-
 	if len(c.InputArgs) > 1 || (len(c.InputArgs) == 0 && !c.Latest) {
 		return errors.Errorf("attach requires the name or id of one running container or the latest flag")
 	}
-
-	runtime, err := libpodruntime.GetRuntime(&c.PodmanCommand)
-	if err != nil {
-		return errors.Wrapf(err, "error creating libpod runtime")
-	}
-	defer runtime.Shutdown(false)
-
-	if c.Latest {
-		ctr, err = runtime.GetLatestContainer()
-	} else {
-		ctr, err = runtime.LookupContainer(args[0])
-	}
-
-	if err != nil {
-		return errors.Wrapf(err, "unable to exec into %s", args[0])
+	if remoteclient && len(c.InputArgs) != 1 {
+		return errors.Errorf("attach requires the name or id of one running container")
 	}
-
-	conState, err := ctr.State()
+	runtime, err := adapter.GetRuntime(&c.PodmanCommand)
 	if err != nil {
-		return errors.Wrapf(err, "unable to determine state of %s", args[0])
-	}
-	if conState != libpod.ContainerStateRunning {
-		return errors.Errorf("you can only attach to running containers")
-	}
-
-	inputStream := os.Stdin
-	if c.NoStdin {
-		inputStream = nil
+		return errors.Wrapf(err, "error creating runtime")
 	}
-
-	// If the container is in a pod, also set to recursively start dependencies
-	if err := adapter.StartAttachCtr(getContext(), ctr, os.Stdout, os.Stderr, inputStream, c.DetachKeys, c.SigProxy, false, ctr.PodID() != ""); err != nil && errors.Cause(err) != libpod.ErrDetach {
-		return errors.Wrapf(err, "error attaching to container %s", ctr.ID())
-	}
-
-	return nil
+	defer runtime.Shutdown(false)
+	return runtime.Attach(getContext(), c)
 }
diff --git a/cmd/podman/commands.go b/cmd/podman/commands.go
index 7c660f7cb..4d3df6896 100644
--- a/cmd/podman/commands.go
+++ b/cmd/podman/commands.go
@@ -11,7 +11,6 @@ const remoteclient = false
 // Commands that the local client implements
 func getMainCommands() []*cobra.Command {
 	rootCommands := []*cobra.Command{
-		_attachCommand,
 		_commitCommand,
 		_execCommand,
 		_generateCommand,
@@ -47,7 +46,6 @@ func getImageSubCommands() []*cobra.Command {
 func getContainerSubCommands() []*cobra.Command {
 
 	return []*cobra.Command{
-		_attachCommand,
 		_checkpointCommand,
 		_cleanupCommand,
 		_commitCommand,
diff --git a/cmd/podman/container.go b/cmd/podman/container.go
index d1c42f673..380d1f250 100644
--- a/cmd/podman/container.go
+++ b/cmd/podman/container.go
@@ -50,6 +50,7 @@ var (
 
 	// Commands that are universally implemented.
 	containerCommands = []*cobra.Command{
+		_attachCommand,
 		_containerExistsCommand,
 		_contInspectSubCommand,
 		_diffCommand,
diff --git a/cmd/podman/main.go b/cmd/podman/main.go
index 7c765a0e0..35cef793d 100644
--- a/cmd/podman/main.go
+++ b/cmd/podman/main.go
@@ -38,6 +38,7 @@ var (
 // Commands that the remote and local client have
 // implemented.
 var mainCommands = []*cobra.Command{
+	_attachCommand,
 	_buildCommand,
 	_diffCommand,
 	_createCommand,
diff --git a/cmd/podman/varlink/io.podman.varlink b/cmd/podman/varlink/io.podman.varlink
index d8905326c..ae830f3e6 100644
--- a/cmd/podman/varlink/io.podman.varlink
+++ b/cmd/podman/varlink/io.podman.varlink
@@ -658,8 +658,9 @@ method PauseContainer(name: string) -> (container: string)
 # See also [PauseContainer](#PauseContainer).
 method UnpauseContainer(name: string) -> (container: string)
 
-# This method has not be implemented yet.
-# method AttachToContainer() -> (notimplemented: NotImplemented)
+method Attach(name: string) -> ()
+
+method AttachControl(name: string) -> ()
 
 # GetAttachSockets takes the name or ID of an existing container.  It returns file paths for two sockets needed
 # to properly communicate with a container.  The first is the actual I/O socket that the container uses.  The
@@ -1154,6 +1155,9 @@ method PodStateData(name: string) -> (config: string)
 # This call is for the development of Podman only and should not be used.
 method CreateFromCC(in: []string) -> (id: string)
 
+# Spec returns the oci spec for a container.  This call is for development of Podman only and generally should not be used.
+method Spec(name: string) -> (config: string)
+
 # Sendfile allows a remote client to send a file to the host
 method SendFile(type: string, length: int) -> (file_handle: string)
 
diff --git a/pkg/adapter/containers.go b/pkg/adapter/containers.go
index 8ce506542..a9b3232e7 100644
--- a/pkg/adapter/containers.go
+++ b/pkg/adapter/containers.go
@@ -407,3 +407,39 @@ func (r *LocalRuntime) Ps(c *cliconfig.PsValues, opts shared.PsOptions) ([]share
 	logrus.Debugf("Setting maximum workers to %d", maxWorkers)
 	return shared.GetPsContainerOutput(r.Runtime, opts, c.Filter, maxWorkers)
 }
+
+// Attach ...
+func (r *LocalRuntime) Attach(ctx context.Context, c *cliconfig.AttachValues) error {
+	var (
+		ctr *libpod.Container
+		err error
+	)
+
+	if c.Latest {
+		ctr, err = r.Runtime.GetLatestContainer()
+	} else {
+		ctr, err = r.Runtime.LookupContainer(c.InputArgs[0])
+	}
+
+	if err != nil {
+		return errors.Wrapf(err, "unable to exec into %s", c.InputArgs[0])
+	}
+
+	conState, err := ctr.State()
+	if err != nil {
+		return errors.Wrapf(err, "unable to determine state of %s", ctr.ID())
+	}
+	if conState != libpod.ContainerStateRunning {
+		return errors.Errorf("you can only attach to running containers")
+	}
+
+	inputStream := os.Stdin
+	if c.NoStdin {
+		inputStream = nil
+	}
+	// If the container is in a pod, also set to recursively start dependencies
+	if err := StartAttachCtr(ctx, ctr, os.Stdout, os.Stderr, inputStream, c.DetachKeys, c.SigProxy, false, ctr.PodID() != ""); err != nil && errors.Cause(err) != libpod.ErrDetach {
+		return errors.Wrapf(err, "error attaching to container %s", ctr.ID())
+	}
+	return nil
+}
diff --git a/pkg/adapter/containers_remote.go b/pkg/adapter/containers_remote.go
index 424c431df..1ae39749f 100644
--- a/pkg/adapter/containers_remote.go
+++ b/pkg/adapter/containers_remote.go
@@ -6,19 +6,25 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"io"
+	"os"
 	"strconv"
 	"syscall"
 	"time"
 
 	"github.com/containers/libpod/cmd/podman/cliconfig"
 	"github.com/containers/libpod/cmd/podman/shared"
-	"github.com/pkg/errors"
-	"github.com/sirupsen/logrus"
-
 	iopodman "github.com/containers/libpod/cmd/podman/varlink"
 	"github.com/containers/libpod/libpod"
 	"github.com/containers/libpod/pkg/inspect"
+	"github.com/containers/libpod/pkg/varlinkapi/virtwriter"
+	"github.com/docker/docker/pkg/term"
+	"github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
 	"github.com/varlink/go/varlink"
+	"golang.org/x/crypto/ssh/terminal"
+	"k8s.io/client-go/tools/remotecommand"
 )
 
 // Inspect returns an inspect struct from varlink
@@ -71,6 +77,19 @@ func (r *LocalRuntime) ContainerState(name string) (*libpod.ContainerState, erro
 
 }
 
+// Spec obtains the container spec.
+func (r *LocalRuntime) Spec(name string) (*specs.Spec, error) {
+	reply, err := iopodman.Spec().Call(r.Conn, name)
+	if err != nil {
+		return nil, err
+	}
+	data := specs.Spec{}
+	if err := json.Unmarshal([]byte(reply), &data); err != nil {
+		return nil, err
+	}
+	return &data, nil
+}
+
 // LookupContainer gets basic information about container over a varlink
 // connection and then translates it to a *Container
 func (r *LocalRuntime) LookupContainer(idOrName string) (*Container, error) {
@@ -79,10 +98,6 @@ func (r *LocalRuntime) LookupContainer(idOrName string) (*Container, error) {
 		return nil, err
 	}
 	config := r.Config(idOrName)
-	if err != nil {
-		return nil, err
-	}
-
 	return &Container{
 		remoteContainer{
 			r,
@@ -322,18 +337,32 @@ func (r *LocalRuntime) CreateContainer(ctx context.Context, c *cliconfig.CreateV
 
 // Run creates a container overvarlink and then starts it
 func (r *LocalRuntime) Run(ctx context.Context, c *cliconfig.RunValues, exitCode int) (int, error) {
+	// FIXME
+	// podman-remote run -it alpine ls DOES NOT WORK YET
+	// podman-remote run -it alpine /bin/sh does, i suspect there is some sort of
+	// timing issue between the socket availability and terminal setup and the command
+	// being run.
+
 	// TODO the exit codes for run need to be figured out for remote connections
-	if !c.Bool("detach") {
-		return 0, errors.New("the remote client only supports detached containers")
-	}
 	results := shared.NewIntermediateLayer(&c.PodmanCommand)
 	cid, err := iopodman.CreateContainer().Call(r.Conn, results.MakeVarlink())
 	if err != nil {
 		return 0, err
 	}
-	fmt.Println(cid)
 	_, err = iopodman.StartContainer().Call(r.Conn, cid)
-	return 0, err
+	if err != nil {
+		return 0, err
+	}
+	errChan, err := r.attach(ctx, os.Stdin, os.Stdout, cid)
+	if err != nil {
+		return 0, err
+	}
+	if c.Bool("detach") {
+		fmt.Println(cid)
+		return 0, err
+	}
+	finalError := <-errChan
+	return 0, finalError
 }
 
 func ReadExitFile(runtimeTmp, ctrID string) (int, error) {
@@ -411,3 +440,102 @@ func (r *LocalRuntime) Ps(c *cliconfig.PsValues, opts shared.PsOptions) ([]share
 	}
 	return psContainers, nil
 }
+
+func (r *LocalRuntime) attach(ctx context.Context, stdin, stdout *os.File, cid string) (chan error, error) {
+	var (
+		oldTermState *term.State
+	)
+	errChan := make(chan error)
+	spec, err := r.Spec(cid)
+	if err != nil {
+		return nil, err
+	}
+	resize := make(chan remotecommand.TerminalSize)
+	haveTerminal := terminal.IsTerminal(int(os.Stdin.Fd()))
+
+	// Check if we are attached to a terminal. If we are, generate resize
+	// events, and set the terminal to raw mode
+	if haveTerminal && spec.Process.Terminal {
+		logrus.Debugf("Handling terminal attach")
+
+		subCtx, cancel := context.WithCancel(ctx)
+		defer cancel()
+
+		resizeTty(subCtx, resize)
+		oldTermState, err = term.SaveState(os.Stdin.Fd())
+		if err != nil {
+			return nil, errors.Wrapf(err, "unable to save terminal state")
+		}
+
+		logrus.SetFormatter(&RawTtyFormatter{})
+		term.SetRawTerminal(os.Stdin.Fd())
+
+	}
+
+	_, err = iopodman.Attach().Send(r.Conn, varlink.Upgrade, cid)
+	if err != nil {
+		restoreTerminal(oldTermState)
+		return nil, err
+	}
+
+	// These are the varlink sockets
+	reader := r.Conn.Reader
+	writer := r.Conn.Writer
+
+	// These are the special writers that encode input from the client.
+	varlinkStdinWriter := virtwriter.NewVirtWriteCloser(writer, virtwriter.ToStdin)
+	varlinkResizeWriter := virtwriter.NewVirtWriteCloser(writer, virtwriter.TerminalResize)
+
+	go func() {
+		// Read from the wire and direct to stdout or stderr
+		err := virtwriter.Reader(reader, stdout, os.Stderr, nil, nil)
+		defer restoreTerminal(oldTermState)
+		errChan <- err
+	}()
+
+	go func() {
+		for termResize := range resize {
+			b, err := json.Marshal(termResize)
+			if err != nil {
+				defer restoreTerminal(oldTermState)
+				errChan <- err
+			}
+			_, err = varlinkResizeWriter.Write(b)
+			if err != nil {
+				defer restoreTerminal(oldTermState)
+				errChan <- err
+			}
+		}
+	}()
+
+	// Takes stdinput and sends it over the wire after being encoded
+	go func() {
+		if _, err := io.Copy(varlinkStdinWriter, stdin); err != nil {
+			defer restoreTerminal(oldTermState)
+			errChan <- err
+		}
+
+	}()
+	return errChan, nil
+
+}
+
+// Attach to a remote terminal
+func (r *LocalRuntime) Attach(ctx context.Context, c *cliconfig.AttachValues) error {
+	ctr, err := r.LookupContainer(c.InputArgs[0])
+	if err != nil {
+		return nil
+	}
+	if ctr.state.State != libpod.ContainerStateRunning {
+		return errors.New("you can only attach to running containers")
+	}
+	inputStream := os.Stdin
+	if c.NoStdin {
+		inputStream = nil
+	}
+	errChan, err := r.attach(ctx, inputStream, os.Stdout, c.InputArgs[0])
+	if err != nil {
+		return err
+	}
+	return <-errChan
+}
diff --git a/pkg/varlinkapi/attach.go b/pkg/varlinkapi/attach.go
new file mode 100644
index 000000000..53c4d1ff6
--- /dev/null
+++ b/pkg/varlinkapi/attach.go
@@ -0,0 +1,75 @@
+// +build varlink
+
+package varlinkapi
+
+import (
+	"io"
+
+	"github.com/containers/libpod/cmd/podman/varlink"
+	"github.com/containers/libpod/libpod"
+	"github.com/containers/libpod/pkg/varlinkapi/virtwriter"
+	"k8s.io/client-go/tools/remotecommand"
+)
+
+// Close is method to close the writer
+
+// Attach ...
+func (i *LibpodAPI) Attach(call iopodman.VarlinkCall, name string) error {
+	var finalErr error
+	resize := make(chan remotecommand.TerminalSize)
+	errChan := make(chan error)
+
+	if !call.WantsUpgrade() {
+		return call.ReplyErrorOccurred("client must use upgraded connection to attach")
+	}
+	ctr, err := i.Runtime.LookupContainer(name)
+	if err != nil {
+		return call.ReplyErrorOccurred(err.Error())
+	}
+
+	// These are the varlink sockets
+	reader := call.Call.Reader
+	writer := call.Call.Writer
+
+	// This pipe is used to pass stdin from the client to the input stream
+	// once the msg has been "decoded"
+	pr, pw := io.Pipe()
+
+	stdoutWriter := virtwriter.NewVirtWriteCloser(writer, virtwriter.ToStdout)
+	// TODO if runc ever starts passing stderr, we can too
+	//stderrWriter := NewVirtWriteCloser(writer, ToStderr)
+
+	streams := libpod.AttachStreams{
+		OutputStream: stdoutWriter,
+		InputStream:  pr,
+		// Runc eats the error stream
+		ErrorStream:  stdoutWriter,
+		AttachInput:  true,
+		AttachOutput: true,
+		// Runc eats the error stream
+		AttachError: true,
+	}
+
+	go func() {
+		if err := virtwriter.Reader(reader, nil, nil, pw, resize); err != nil {
+			errChan <- err
+		}
+	}()
+
+	go func() {
+		// TODO allow for customizable detach keys
+		if err := ctr.Attach(&streams, "", resize); err != nil {
+			errChan <- err
+		}
+	}()
+
+	select {
+	// Blocking on an error
+	case finalErr = <-errChan:
+		// Need to close up shop
+		_ = finalErr
+	}
+	quitWriter := virtwriter.NewVirtWriteCloser(writer, virtwriter.Quit)
+	_, err = quitWriter.Write([]byte("HANG-UP"))
+	return call.Writer.Flush()
+}
diff --git a/pkg/varlinkapi/containers.go b/pkg/varlinkapi/containers.go
index 816a72953..17792ccfe 100644
--- a/pkg/varlinkapi/containers.go
+++ b/pkg/varlinkapi/containers.go
@@ -634,6 +634,22 @@ func (i *LibpodAPI) GetContainerStatsWithHistory(call iopodman.VarlinkCall, prev
 	return call.ReplyGetContainerStatsWithHistory(cStats)
 }
 
+// Spec ...
+func (i *LibpodAPI) Spec(call iopodman.VarlinkCall, name string) error {
+	ctr, err := i.Runtime.LookupContainer(name)
+	if err != nil {
+		return call.ReplyErrorOccurred(err.Error())
+	}
+
+	spec := ctr.Spec()
+	b, err := json.Marshal(spec)
+	if err != nil {
+		return call.ReplyErrorOccurred(err.Error())
+	}
+
+	return call.ReplySpec(string(b))
+}
+
 // GetContainersLogs is the varlink endpoint to obtain one or more container logs
 func (i *LibpodAPI) GetContainersLogs(call iopodman.VarlinkCall, names []string, follow, latest bool, since string, tail int64, timestamps bool) error {
 	var wg sync.WaitGroup
diff --git a/pkg/varlinkapi/virtwriter/virtwriter.go b/pkg/varlinkapi/virtwriter/virtwriter.go
new file mode 100644
index 000000000..3adaf6e17
--- /dev/null
+++ b/pkg/varlinkapi/virtwriter/virtwriter.go
@@ -0,0 +1,155 @@
+package virtwriter
+
+import (
+	"bufio"
+	"encoding/binary"
+	"encoding/json"
+	"errors"
+	"io"
+	"os"
+
+	"k8s.io/client-go/tools/remotecommand"
+)
+
+// SocketDest is the "key" to where IO should go on the varlink
+// multiplexed socket
+type SocketDest int
+
+const (
+	// ToStdout indicates traffic should go stdout
+	ToStdout SocketDest = iota
+	// ToStdin indicates traffic came from stdin
+	ToStdin SocketDest = iota
+	// ToStderr indicates traffuc should go to stderr
+	ToStderr SocketDest = iota
+	// TerminalResize indicates a terminal resize event has occurred
+	// and data should be passed to resizer
+	TerminalResize SocketDest = iota
+	// Quit and detach
+	Quit SocketDest = iota
+)
+
+// IntToSocketDest returns a socketdest based on integer input
+func IntToSocketDest(i int) SocketDest {
+	switch i {
+	case ToStdout.Int():
+		return ToStdout
+	case ToStderr.Int():
+		return ToStderr
+	case ToStdin.Int():
+		return ToStdin
+	case TerminalResize.Int():
+		return TerminalResize
+	case Quit.Int():
+		return Quit
+	default:
+		return ToStderr
+	}
+}
+
+// Int returns the integer representation of the socket dest
+func (sd SocketDest) Int() int {
+	return int(sd)
+}
+
+// VirtWriteCloser are writers for attach which include the dest
+// of the data
+type VirtWriteCloser struct {
+	writer *bufio.Writer
+	dest   SocketDest
+}
+
+// NewVirtWriteCloser is a constructor
+func NewVirtWriteCloser(w *bufio.Writer, dest SocketDest) VirtWriteCloser {
+	return VirtWriteCloser{w, dest}
+}
+
+// Close is a required method for a writecloser
+func (v VirtWriteCloser) Close() error {
+	return nil
+}
+
+// Write prepends a header to the input message.  The header is
+// 8bytes.  Position one contains the destination.  Positions
+// 5,6,7,8 are a big-endian encoded uint32 for len of the message.
+func (v VirtWriteCloser) Write(input []byte) (int, error) {
+	header := []byte{byte(v.dest), 0, 0, 0}
+	// Go makes us define the byte for big endian
+	mlen := make([]byte, 4)
+	binary.BigEndian.PutUint32(mlen, uint32(len(input)))
+	// append the message len to the header
+	msg := append(header, mlen...)
+	// append the message to the header
+	msg = append(msg, input...)
+	_, err := v.writer.Write(msg)
+	if err != nil {
+		return 0, err
+	}
+	err = v.writer.Flush()
+	return len(input), err
+}
+
+// Reader decodes the content that comes over the wire and directs it to the proper destination.
+func Reader(r *bufio.Reader, output, errput *os.File, input *io.PipeWriter, resize chan remotecommand.TerminalSize) error {
+	var saveb []byte
+	var eom int
+	for {
+		readb := make([]byte, 32*1024)
+		n, err := r.Read(readb)
+		// TODO, later may be worth checking in len of the read is 0
+		if err != nil {
+			return err
+		}
+		b := append(saveb, readb[0:n]...)
+		// no sense in reading less than the header len
+		for len(b) > 7 {
+			eom = int(binary.BigEndian.Uint32(b[4:8])) + 8
+			// The message and header are togther
+			if len(b) >= eom {
+				out := append([]byte{}, b[8:eom]...)
+
+				switch IntToSocketDest(int(b[0])) {
+				case ToStdout:
+					n, err := output.Write(out)
+					if err != nil {
+						return err
+					}
+					if n < len(out) {
+						return errors.New("short write error occurred on stdout")
+					}
+				case ToStderr:
+					n, err := errput.Write(out)
+					if err != nil {
+						return err
+					}
+					if n < len(out) {
+						return errors.New("short write error occurred on stderr")
+					}
+				case ToStdin:
+					n, err := input.Write(out)
+					if err != nil {
+						return err
+					}
+					if n < len(out) {
+						return errors.New("short write error occurred on stdin")
+					}
+				case TerminalResize:
+					// Resize events come over in bytes, need to be reserialized
+					resizeEvent := remotecommand.TerminalSize{}
+					if err := json.Unmarshal(out, &resizeEvent); err != nil {
+						return err
+					}
+					resize <- resizeEvent
+				case Quit:
+					return nil
+				}
+				b = b[eom:]
+			} else {
+				//	We do not have the header and full message, need to slurp again
+				saveb = b
+				break
+			}
+		}
+	}
+	return nil
+}