VMware vSphere Integrated Containers provider (#206)

* Add Virtual Kubelet provider for VIC

Initial virtual kubelet provider for VMware VIC.  This provider currently
handles creating and starting of a pod VM via the VIC portlayer and persona
server.  Image store handling via the VIC persona server.  This provider
currently requires the feature/wolfpack branch of VIC.

* Added pod stop and delete.  Also added node capacity.

Added the ability to stop and delete pod VMs via VIC.  Also retrieve
node capacity information from the VCH.

* Cleanup and readme file

Some file clean up and added a Readme.md markdown file for the VIC
provider.

* Cleaned up errors, added function comments, moved operation code

1. Cleaned up error handling.  Set standard for creating errors.
2. Added method prototype comments for all interface functions.
3. Moved PodCreator, PodStarter, PodStopper, and PodDeleter to a new folder.

* Add mocking code and unit tests for podcache, podcreator, and podstarter

Used the unit test framework used in VIC to handle assertions in the provider's
unit test.  Mocking code generated using OSS project mockery, which is compatible
with the testify assertion framework.

* Vendored packages for the VIC provider

Requires feature/wolfpack branch of VIC and a few specific commit sha of
projects used within VIC.

* Implementation of POD Stopper and Deleter unit tests (#4)

* Updated files for initial PR

vendor/github.com/vmware/vic/lib/portlayer/attach/bind.go

@@ -0,0 +1,37 @@
+// Copyright 2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package attach
+
+import (
+	"fmt"
+
+	"github.com/vmware/vic/lib/portlayer/exec"
+	"github.com/vmware/vic/pkg/trace"
+)
+
+// Bind sets the *Connected fields of the VirtualSerialPort
+func Bind(h interface{}, id string) (interface{}, error) {
+	defer trace.End(trace.Begin(""))
+
+	handle, ok := h.(*exec.Handle)
+	if !ok {
+		return nil, fmt.Errorf("Type assertion failed for %#+v", handle)
+	}
+	if handle.MigrationError != nil {
+		return nil, fmt.Errorf("Migration failed %s", handle.MigrationError)
+	}
+
+	return toggle(handle, id, true)
+}

vendor/github.com/vmware/vic/lib/portlayer/attach/common.go

@@ -0,0 +1,135 @@
+// Copyright 2016 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package attach
+
+import (
+	"fmt"
+	"net"
+
+	"github.com/vmware/govmomi/object"
+	"github.com/vmware/govmomi/vim25/types"
+	"github.com/vmware/vic/lib/constants"
+	"github.com/vmware/vic/lib/migration/feature"
+	"github.com/vmware/vic/lib/portlayer/exec"
+
+	log "github.com/Sirupsen/logrus"
+)
+
+func lookupVCHIP() (net.IP, error) {
+	// FIXME: THERE MUST BE ANOTHER WAY
+	// following is from Create@exec.go
+	ips, err := net.LookupIP(constants.ManagementHostName)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(ips) == 0 {
+		return nil, fmt.Errorf("No IP found on %s", constants.ManagementHostName)
+	}
+
+	if len(ips) > 1 {
+		return nil, fmt.Errorf("Multiple IPs found on %s: %#v", constants.ManagementHostName, ips)
+	}
+	return ips[0], nil
+}
+
+func toggle(handle *exec.Handle, id string, connected bool) (*exec.Handle, error) {
+	// check to see whether id is in Execs, if so set its RunBlock property to connected
+	session, ok := handle.ExecConfig.Execs[id]
+	if ok {
+		if err := compatible(handle); err != nil {
+			return nil, err
+		}
+
+		if session.Attach {
+			session.RunBlock = connected
+		}
+	}
+
+	// get the virtual device list
+	devices := object.VirtualDeviceList(handle.Config.Hardware.Device)
+
+	// select the virtual serial ports
+	serials := devices.SelectByBackingInfo((*types.VirtualSerialPortURIBackingInfo)(nil))
+	if len(serials) == 0 {
+		return nil, fmt.Errorf("Unable to find a device with desired backing")
+	}
+	if len(serials) > 1 {
+		return nil, fmt.Errorf("Multiple matches found with desired backing")
+	}
+	serial := serials[0]
+
+	ip, err := lookupVCHIP()
+	if err != nil {
+		return nil, err
+	}
+
+	log.Debugf("Found a device with desired backing: %#v", serial)
+
+	c := serial.GetVirtualDevice().Connectable
+	b := serial.GetVirtualDevice().Backing.(*types.VirtualSerialPortURIBackingInfo)
+
+	serviceURI := fmt.Sprintf("tcp://127.0.0.1:%d", constants.AttachServerPort)
+	proxyURI := fmt.Sprintf("telnet://%s:%d", ip, constants.SerialOverLANPort)
+
+	if b.ProxyURI == proxyURI && c.Connected == connected {
+		log.Debugf("Already in the desired state, (connected: %t, proxyURI: %s)", connected, proxyURI)
+		return handle, nil
+	}
+
+	// set the values
+	log.Debugf("Setting Connected to %t", connected)
+	c.Connected = connected
+	if connected && handle.ExecConfig.Sessions[handle.ExecConfig.ID].Attach {
+		log.Debugf("Setting the start connected state to %t", connected)
+		c.StartConnected = handle.ExecConfig.Sessions[handle.ExecConfig.ID].Attach
+	}
+
+	log.Debugf("Setting ServiceURI to %s", serviceURI)
+	b.ServiceURI = serviceURI
+
+	log.Debugf("Setting the ProxyURI to %s", proxyURI)
+	b.ProxyURI = proxyURI
+
+	config := &types.VirtualDeviceConfigSpec{
+		Device:    serial,
+		Operation: types.VirtualDeviceConfigSpecOperationEdit,
+	}
+	handle.Spec.DeviceChange = append(handle.Spec.DeviceChange, config)
+
+	// check to see whether id is in Sessions, if so set its RunBlock property to connected
+	// if attach happens before start then this property will be persist in the vmx
+	// if attach happens after start then this propery will be thrown away by commit (one cannot change persistent extraconfig values if the vm is powered on)
+	session, ok = handle.ExecConfig.Sessions[id]
+	if ok {
+		if session.Attach {
+			session.RunBlock = connected
+		}
+	}
+
+	return handle, nil
+}
+
+func compatible(h interface{}) error {
+	if handle, ok := h.(*exec.Handle); ok {
+		if handle.DataVersion < feature.ExecSupportedVersion {
+			return fmt.Errorf("attaching exec tasks not supported for this container")
+		}
+
+		return nil
+	}
+
+	return fmt.Errorf("Type assertion failed for %#+v", h)
+}

vendor/github.com/vmware/vic/lib/portlayer/attach/communication/connector.go

@@ -0,0 +1,484 @@
+// Copyright 2016-2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package communication
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"sync"
+	"time"
+
+	log "github.com/Sirupsen/logrus"
+
+	"github.com/vmware/vic/lib/tether/msgs"
+	"github.com/vmware/vic/pkg/errors"
+	"github.com/vmware/vic/pkg/serial"
+	"github.com/vmware/vic/pkg/trace"
+
+	"golang.org/x/crypto/ssh"
+	"golang.org/x/sync/singleflight"
+)
+
+const (
+	VersionString = "SSH-2.0-VIC"
+	ClientTimeout = 10 * time.Second
+)
+
+// Connector defines the connection and interactions
+type Connector struct {
+	mutex        sync.RWMutex
+	cond         *sync.Cond
+	interactions map[string]*LazySessionInteractor
+
+	listener net.Listener
+	// Quit channel for serve
+	done chan struct{}
+
+	// deduplication of incoming calls
+	fg singleflight.Group
+
+	// graceful shutdown
+	wg sync.WaitGroup
+}
+
+// NewConnector returns a new Connector
+func NewConnector(listener net.Listener) *Connector {
+	defer trace.End(trace.Begin(""))
+
+	connector := &Connector{
+		interactions: make(map[string]*LazySessionInteractor),
+		listener:     listener,
+		done:         make(chan struct{}),
+	}
+	connector.cond = sync.NewCond(connector.mutex.RLocker())
+
+	return connector
+}
+
+// SessionIfAlive returns SessionInteractor or error
+func (c *Connector) SessionIfAlive(ctx context.Context, id string) (SessionInteractor, error) {
+	c.mutex.RLock()
+	v, ok := c.interactions[id]
+	c.mutex.RUnlock()
+
+	if !ok {
+		return nil, fmt.Errorf("attach connector: no such connection in the map")
+	}
+	// we have an entry in the map, let's check its status
+	var conn SessionInteractor
+	var err error
+
+	conn, err = v.Initialize()
+	if err != nil {
+		goto Error
+	}
+
+	log.Debugf("attach connector: Pinging for %s", id)
+	if err = conn.Ping(); err != nil {
+		goto Error
+	}
+
+	log.Debugf("attach connector: Unblocking for %s", id)
+	if err = conn.Unblock(); err != nil {
+		goto Error
+	}
+	log.Debugf("attach connector: Unblocked %s, returning", id)
+
+	return conn, nil
+
+Error:
+	log.Debugf("attach connector: liveness check failed, removing %s from connection map", id)
+
+	c.mutex.Lock()
+	delete(c.interactions, id)
+	c.mutex.Unlock()
+
+	return nil, err
+}
+
+// Interaction returns the interactor corresponding to the specified ID. If the connection doesn't exist
+// the method will wait for the specified timeout, returning when the connection is created
+// or the timeout expires, whichever occurs first
+func (c *Connector) Interaction(ctx context.Context, id string) (SessionInteractor, error) {
+	defer trace.End(trace.Begin(id))
+
+	// make sure that we have only one call in-flight for each ID at any given time
+	si, err, shared := c.fg.Do(id, func() (interface{}, error) {
+		return c.interaction(ctx, id)
+	})
+	if err != nil {
+		c.fg.Forget(id)
+		return nil, err
+	}
+	if shared {
+		log.Debugf("Eliminated duplicated calls to Interaction for %s", id)
+	}
+	return si.(SessionInteractor), nil
+}
+
+func (c *Connector) interaction(ctx context.Context, id string) (SessionInteractor, error) {
+	defer trace.End(trace.Begin(id))
+
+	conn, err := c.SessionIfAlive(ctx, id)
+	if conn != nil && err == nil {
+		return conn, nil
+	}
+
+	if ctx.Err() == context.DeadlineExceeded {
+		return nil, fmt.Errorf("attach connector: no such connection")
+	}
+
+	result := make(chan SessionInteractor, 1)
+	go func() {
+		ok := false
+		var v *LazySessionInteractor
+
+		c.mutex.RLock()
+		defer c.mutex.RUnlock()
+
+		for !ok && ctx.Err() == nil {
+			v, ok = c.interactions[id]
+			if ok {
+				conn, err := v.Initialize()
+				if conn != nil && err == nil {
+					// no need to test this connection as we just created it, unblock if needed
+					log.Debugf("attach connector: Unblocking for %s", id)
+					err = conn.Unblock()
+					if err == nil {
+						log.Debugf("attach connector: Unblocked %s, returning", id)
+
+						result <- conn
+						return
+					}
+				}
+				if err != nil {
+					log.Error(err)
+				}
+				ok = false
+			}
+
+			// block until cond is updated
+			log.Infof("attach connector:  Connection not found yet for %s", id)
+			c.cond.Wait()
+		}
+		log.Debugf("attach connector:  Giving up on connection for %s", id)
+	}()
+
+	select {
+	case client := <-result:
+		log.Debugf("attach connector: Found connection for %s: %p", id, client)
+		return client, nil
+	case <-ctx.Done():
+		err := fmt.Errorf("attach connector: Connection not found error for id:%s: %s", id, ctx.Err())
+		log.Error(err)
+		// wake up the result gofunc before returning
+		c.mutex.RLock()
+		c.cond.Broadcast()
+		c.mutex.RUnlock()
+
+		return nil, err
+	}
+}
+
+// RemoveInteraction removes the session the inteactions map
+func (c *Connector) RemoveInteraction(id string) error {
+	defer trace.End(trace.Begin(id))
+
+	var err error
+
+	c.mutex.Lock()
+	v, ok := c.interactions[id]
+	if ok {
+		log.Debugf("attach connector: Removing %s from the connection map", id)
+		delete(c.interactions, id)
+		c.fg.Forget(id)
+	}
+	c.mutex.Unlock()
+
+	// the !ok case, but let's check the actual condition that impacts us
+	if v == nil {
+		return nil
+	}
+
+	conn := v.SessionInteractor()
+	if conn != nil {
+		err = conn.Close()
+	}
+	return err
+}
+
+// Start starts the connector
+func (c *Connector) Start() {
+	defer trace.End(trace.Begin(""))
+
+	c.wg.Add(1)
+	go c.serve()
+}
+
+// Stop stops the connector
+func (c *Connector) Stop() {
+	defer trace.End(trace.Begin(""))
+
+	c.listener.Close()
+	close(c.done)
+	c.wg.Wait()
+}
+
+// Starts the connector listening on the specified source
+// TODO: should have mechanism for stopping this, and probably handing off the interactions to another
+// routine to insert into the map
+func (c *Connector) serve() {
+	defer c.wg.Done()
+	for {
+		if c.listener == nil {
+			log.Debugf("attach connector: listener closed")
+			break
+		}
+
+		// check to see whether we should stop accepting new connections and exit
+		select {
+		case <-c.done:
+			log.Debugf("attach connector: done closed")
+			return
+		default:
+		}
+
+		conn, err := c.listener.Accept()
+		if err != nil {
+			log.Errorf("Error waiting for incoming connection: %s", errors.ErrorStack(err))
+			continue
+		}
+		log.Debugf("attach connector: Received incoming connection")
+
+		go c.processIncoming(conn)
+	}
+}
+
+// takes the base connection, determines the ID of the source and stashes it in the map
+func (c *Connector) processIncoming(conn net.Conn) {
+	var err error
+	defer func() {
+		if err != nil && conn != nil {
+			conn.Close()
+		}
+	}()
+
+	log.Debugf("Initiating ssh handshake with new connection attempt")
+	for {
+		if conn == nil {
+			log.Infof("attach connector: connection closed")
+			return
+		}
+
+		// TODO needs timeout handling.  This could take 30s.
+
+		// Timeout for client handshake should be reasonably small.
+		// Server will try to drain a buffer and if the buffer doesn't contain
+		// 2 or more bytes it will just wait, so client should timeout.
+		// However, if timeout is too short, client will flood server with Syn requests.
+		ctx, cancel := context.WithTimeout(context.TODO(), time.Second)
+		defer cancel()
+
+		deadline, ok := ctx.Deadline()
+		if ok {
+			conn.SetReadDeadline(deadline)
+		}
+		if err = serial.HandshakeClient(conn); err == nil {
+			conn.SetReadDeadline(time.Time{})
+			log.Debugf("HandshakeClient: connection handshake established")
+			cancel()
+			break
+		}
+
+		switch e := err.(type) {
+		case *serial.HandshakeError:
+			log.Debugf("HandshakeClient: %v", e)
+			continue
+		case *net.OpError:
+			if e.Temporary() || e.Timeout() {
+				// if it's a passing error or timeout then try again
+				continue
+			}
+			// if it's not a temporary condition, then treat it as a transport error
+			log.Errorf("HandshakeClient: transport op-error: %v", e)
+			conn.Close()
+			return
+		default: // includes the io.EOF case
+			// treat everything unknown as transport errror
+			log.Errorf("HandshakeClient: transport error: %v (%T)", e, e)
+			conn.Close()
+			return
+		}
+	}
+
+	callback := func(hostname string, remote net.Addr, key ssh.PublicKey) error {
+		return nil
+	}
+
+	config := &ssh.ClientConfig{
+		User:            "daemon",
+		HostKeyCallback: callback,
+		ClientVersion:   VersionString,
+		Timeout:         ClientTimeout,
+	}
+
+	// create the SSH connection
+	clientConn, chans, reqs, err := ssh.NewClientConn(conn, "", config)
+	if err != nil {
+		log.Errorf("SSH connection could not be established: %s", errors.ErrorStack(err))
+		return
+	}
+
+	// ask the IDs
+	ids, err := ContainerIDs(clientConn)
+	if err != nil {
+		log.Errorf("SSH connection could not be established: %s", errors.ErrorStack(err))
+		return
+	}
+
+	// Handle global requests
+	go c.reqs(reqs, clientConn, ids)
+	// Handle channel open messages
+	go c.chans(chans)
+
+	// create the connections
+	c.ids(clientConn, ids)
+
+	return
+}
+
+// ids iterates over the gived ids and
+// - calls Ping for existing connections
+// - calls NewSSHInteraction for new connections and fills the connection map
+func (c *Connector) ids(conn ssh.Conn, ids []string) {
+	for _, id := range ids {
+		// needed for following closure - https://golang.org/doc/faq#closures_and_goroutines
+		id := id
+
+		c.mutex.RLock()
+		v, ok := c.interactions[id]
+		c.mutex.RUnlock()
+
+		if ok {
+			si, err := v.Initialize()
+			if si != nil && err == nil {
+				if err := si.Ping(); err == nil {
+					log.Debugf("Connection %s found and alive", id)
+
+					continue
+				}
+			}
+			log.Warnf("Connection found but it wasn't alive. Creating a new one")
+		}
+
+		// this is a new connection so learn the version
+		version, err := ContainerVersion(conn)
+		if err != nil {
+			log.Errorf("SSH version could not be learned (id=%s): %s", id, errors.ErrorStack(err))
+			return
+		}
+
+		lazy := &LazySessionInteractor{
+			fn: func() (SessionInteractor, error) {
+				defer trace.End(trace.Begin(id))
+
+				return NewSSHInteraction(conn, id, version)
+			},
+		}
+
+		log.Infof("Established connection with container VM: %s", id)
+
+		c.mutex.Lock()
+
+		c.interactions[id] = lazy
+
+		c.cond.Broadcast()
+		c.mutex.Unlock()
+	}
+}
+
+// reqs is the global request channel of the portlayer side of the connection
+// we keep a list of sessions associated with this connection and drop them from the map when the global mux exits
+func (c *Connector) reqs(reqs <-chan *ssh.Request, conn ssh.Conn, ids []string) {
+	defer trace.End(trace.Begin(""))
+
+	var pending func()
+
+	// list of session ids mux'ed on this connection
+	droplist := make(map[string]struct{})
+
+	// fill the map with the initial ids
+	for _, id := range ids {
+		droplist[id] = struct{}{}
+	}
+
+	for req := range reqs {
+		ok := true
+
+		log.Infof("received global request type %v", req.Type)
+		switch req.Type {
+		case msgs.ContainersReq:
+			pending = func() {
+				ids := msgs.ContainersMsg{}
+				if err := ids.Unmarshal(req.Payload); err != nil {
+					log.Errorf("Unmarshal failed with %s", err)
+					return
+				}
+				c.ids(conn, ids.IDs)
+
+				// drop the drop list to clear no longer active sessions from the map
+				droplist = make(map[string]struct{})
+
+				// fill the droplist with the latest info
+				for _, id := range ids.IDs {
+					droplist[id] = struct{}{}
+				}
+			}
+		default:
+			ok = false
+		}
+
+		// make sure that errors get send back if we failed
+		if req.WantReply {
+			log.Infof("Sending global request reply %t", ok)
+			if err := req.Reply(ok, nil); err != nil {
+				log.Warnf("Failed to reply a request back")
+			}
+		}
+
+		// run any pending work now that a reply has been sent
+		if pending != nil {
+			log.Debug("Invoking pending work for global mux")
+			go pending()
+			pending = nil
+		}
+	}
+
+	// global mux closed so it is time to do cleanup
+	for id := range droplist {
+		log.Infof("Droping %s from connection map", id)
+		c.RemoveInteraction(id)
+	}
+}
+
+// this is the channel mux for the ssh channel . It is configured to reject everything (required)
+func (c *Connector) chans(chans <-chan ssh.NewChannel) {
+	defer trace.End(trace.Begin(""))
+
+	for ch := range chans {
+		ch.Reject(ssh.UnknownChannelType, fmt.Sprintf("unknown channel type: %v", ch.ChannelType()))
+	}
+}

vendor/github.com/vmware/vic/lib/portlayer/attach/communication/interactor.go

@@ -0,0 +1,253 @@
+// Copyright 2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package communication
+
+import (
+	"fmt"
+	"io"
+	"sync"
+
+	log "github.com/Sirupsen/logrus"
+	"github.com/matryer/resync"
+
+	"golang.org/x/crypto/ssh"
+
+	"github.com/vmware/vic/lib/migration/feature"
+	"github.com/vmware/vic/lib/tether/msgs"
+	"github.com/vmware/vic/pkg/trace"
+)
+
+const (
+	attachChannelType = "attach"
+)
+
+// SessionInteractor defines the interaction interface
+type SessionInteractor interface {
+	// Send specific signal
+	Signal(signal ssh.Signal) error
+	// Stdout stream
+	Stdout() io.Reader
+	// Stderr stream
+	Stderr() io.Reader
+	// Stdin stream
+	Stdin() io.WriteCloser
+	Close() error
+
+	// Resize the terminal
+	Resize(cols, rows, widthpx, heightpx uint32) error
+
+	CloseStdin() error
+
+	Ping() error
+	Unblock() error
+}
+
+// interaction implements SessionInteractor using SSH
+type interaction struct {
+	channel ssh.Channel
+
+	// to serialize unblock requests
+	mu sync.Mutex
+	// avoid spamming unblock messages
+	unblocked resync.Once
+
+	// current feature version that the container provides
+	version uint32
+}
+
+// ContainerVersion asks the version of the containers on the other hand and return them to the caller
+func ContainerVersion(conn ssh.Conn) (uint32, error) {
+	defer trace.End(trace.Begin(""))
+
+	ok, reply, err := conn.SendRequest(msgs.VersionReq, true, nil)
+	if !ok && err == nil {
+		log.Warnf("VersionReq not supported by the container")
+		return 0, nil
+	}
+	if !ok || err != nil {
+		return 0, fmt.Errorf("failed to get container version from remote: %s", err)
+	}
+
+	version := msgs.VersionMsg{}
+	if err = version.Unmarshal(reply); err != nil {
+		return 0, fmt.Errorf("failed to unmarshal version from remote: %s", err)
+	}
+
+	return version.Version, nil
+}
+
+// ContainerIDs asks the ids of the containers on the other hand and return them to the caller
+func ContainerIDs(conn ssh.Conn) ([]string, error) {
+	defer trace.End(trace.Begin(""))
+
+	ok, reply, err := conn.SendRequest(msgs.ContainersReq, true, nil)
+	if !ok || err != nil {
+		return nil, fmt.Errorf("failed to get container IDs from remote: %s", err)
+	}
+
+	ids := msgs.ContainersMsg{}
+	if err = ids.Unmarshal(reply); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal ids from remote: %s", err)
+	}
+
+	return ids.IDs, nil
+}
+
+// NewSSHInteraction returns a stream connection to the requested session
+// The ssh conn is assumed to be connected to the Executor hosting the session
+func NewSSHInteraction(conn ssh.Conn, id string, version uint32) (SessionInteractor, error) {
+	defer trace.End(trace.Begin(id))
+
+	channel, _, err := conn.OpenChannel(attachChannelType, []byte(id))
+	if err != nil {
+		return nil, err
+	}
+
+	i := &interaction{
+		channel: channel,
+		version: version,
+	}
+	return i, nil
+}
+
+func (t *interaction) Signal(signal ssh.Signal) error {
+	defer trace.End(trace.Begin(""))
+
+	msg := msgs.SignalMsg{Signal: signal}
+	ok, err := t.channel.SendRequest(msgs.SignalReq, true, msg.Marshal())
+	if err == nil && !ok {
+		return fmt.Errorf("unknown error")
+	}
+
+	if err != nil {
+		return fmt.Errorf("signal error: %s", err)
+	}
+
+	return nil
+}
+
+func (t *interaction) CloseStdin() error {
+	defer trace.End(trace.Begin(""))
+
+	// configure remote to relay EOFs
+	ok, err := t.channel.SendRequest(msgs.CloseStdinReq, true, nil)
+	if err == nil && !ok {
+		return fmt.Errorf("unknown error closing stdin")
+	}
+
+	if err != nil {
+		return fmt.Errorf("close stdin request error: %s", err)
+	}
+
+	// send inline EOF on the stdin stream
+	err = t.channel.CloseWrite()
+	if err != nil {
+		return fmt.Errorf("close stdin error: %s", err)
+	}
+
+	return nil
+}
+
+func (t *interaction) Stdout() io.Reader {
+	defer trace.End(trace.Begin(""))
+
+	return t.channel
+}
+
+func (t *interaction) Stderr() io.Reader {
+	defer trace.End(trace.Begin(""))
+
+	return t.channel.Stderr()
+}
+
+func (t *interaction) Stdin() io.WriteCloser {
+	defer trace.End(trace.Begin(""))
+
+	return t.channel
+}
+
+func (t *interaction) Close() error {
+	defer trace.End(trace.Begin(""))
+
+	return t.channel.Close()
+}
+
+// Resize resizes the terminal.
+func (t *interaction) Resize(cols, rows, widthpx, heightpx uint32) error {
+	defer trace.End(trace.Begin(""))
+
+	msg := msgs.WindowChangeMsg{
+		Columns:  cols,
+		Rows:     rows,
+		WidthPx:  widthpx,
+		HeightPx: heightpx,
+	}
+	ok, err := t.channel.SendRequest(msgs.WindowChangeReq, true, msg.Marshal())
+	if err == nil && !ok {
+		return fmt.Errorf("unknown error")
+	}
+
+	if err != nil {
+		return fmt.Errorf("resize error: %s", err)
+	}
+	return nil
+}
+
+// Ping checks the liveliness of the connection
+func (t *interaction) Ping() error {
+	defer trace.End(trace.Begin(""))
+
+	if t.version < feature.ExecSupportedVersion {
+		log.Warnf("Running container does not support Ping request, skipping.")
+		return nil
+	}
+
+	ok, err := t.channel.SendRequest(msgs.PingReq, true, []byte(msgs.PingMsg))
+	if !ok || err != nil {
+		return fmt.Errorf("failed to ping the other side: %s", err)
+	}
+
+	return nil
+}
+
+// Unblock sends an unblock msg
+func (t *interaction) Unblock() error {
+	defer trace.End(trace.Begin(""))
+
+	var ok bool
+	var err error
+	var reset bool
+
+	if t.version < feature.ExecSupportedVersion {
+		log.Warnf("Running container does not support Unblock request, skipping.")
+		return nil
+	}
+
+	t.mu.Lock()
+	defer t.mu.Unlock()
+
+	t.unblocked.Do(func() {
+		if ok, err = t.channel.SendRequest(msgs.UnblockReq, true, []byte(msgs.UnblockMsg)); !ok || err != nil {
+			log.Errorf("failed to unblock the other side: %s", err)
+			// #5038: resync package is not reentrant so we need to call Reset after this
+			reset = true
+		}
+	})
+
+	if reset {
+		t.unblocked.Reset()
+	}
+	return err
+}

vendor/github.com/vmware/vic/lib/portlayer/attach/communication/lazy.go

@@ -0,0 +1,60 @@
+// Copyright 2016-2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package communication
+
+import (
+	"sync"
+)
+
+// LazyInitializer defines the function that returns SessionInteractor
+type LazyInitializer func() (SessionInteractor, error)
+
+// LazySessionInteractor holds lazily initialized SessionInteractor
+type LazySessionInteractor struct {
+	mu sync.Mutex
+	si SessionInteractor
+	fn LazyInitializer
+}
+
+// Initialize either returns either already initialized connection or returns the connection after initializing it
+func (l *LazySessionInteractor) Initialize() (SessionInteractor, error) {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+
+	if l.si != nil {
+		return l.si, nil
+	}
+
+	if l.si == nil && l.fn == nil {
+		panic("both si and fn are nil")
+	}
+
+	var err error
+
+	// l.si is nil but l.fn is not
+	l.si, err = l.fn()
+	if err != nil {
+		return nil, err
+	}
+	return l.si, nil
+}
+
+// SessionInteractor returns either an initialized connection, or nil if it was never initialized
+func (l *LazySessionInteractor) SessionInteractor() SessionInteractor {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+
+	return l.si
+}

vendor/github.com/vmware/vic/lib/portlayer/attach/communication/lazy_test.go

@@ -0,0 +1,55 @@
+// Copyright 2016-2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package communication
+
+import (
+	"fmt"
+	"reflect"
+	"testing"
+)
+
+func TestLazySessionInteractor_Initialize(t *testing.T) {
+	type fields struct {
+		si SessionInteractor
+		fn LazyInitializer
+	}
+	tests := []struct {
+		name    string
+		fields  fields
+		want    SessionInteractor
+		wantErr bool
+	}{
+		{"FnIsNil", fields{si: &interaction{}}, &interaction{}, false},
+		{"SiIsNil", fields{si: nil, fn: func() (SessionInteractor, error) { return &interaction{}, nil }}, &interaction{}, false},
+		{"FnAndSIAreNotNil", fields{si: &interaction{}, fn: func() (SessionInteractor, error) { return nil, fmt.Errorf("failure") }}, &interaction{}, false},
+		{"SiIsNilFnWillFail", fields{si: nil, fn: func() (SessionInteractor, error) { return nil, fmt.Errorf("failure") }}, nil, true},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			l := &LazySessionInteractor{
+				si: tt.fields.si,
+				fn: tt.fields.fn,
+			}
+			got, err := l.Initialize()
+			if (err != nil) != tt.wantErr {
+				t.Errorf("LazySessionInteractor.Initialize() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("LazySessionInteractor.Initialize() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

vendor/github.com/vmware/vic/lib/portlayer/attach/communication/server.go

@@ -0,0 +1,120 @@
+// Copyright 2016-2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package communication
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"sync"
+
+	log "github.com/Sirupsen/logrus"
+
+	"github.com/vmware/vic/pkg/errors"
+	"github.com/vmware/vic/pkg/trace"
+)
+
+// Server waits for TCP client connections on serialOverLANPort, then
+// once connected, attempts to negotiate an SSH connection to the attached
+// client.  The client is the ssh server.
+type Server struct {
+	port int
+	ip   string
+
+	m sync.RWMutex
+	l *net.TCPListener
+	c *Connector
+}
+
+// NewServer returns a Server instance
+func NewServer(ip string, port int) *Server {
+	defer trace.End(trace.Begin(""))
+
+	return &Server{
+		ip:   ip,
+		port: port,
+	}
+}
+
+// Start starts the connector with given listener
+func (n *Server) Start() error {
+	defer trace.End(trace.Begin(""))
+
+	n.m.Lock()
+	defer n.m.Unlock()
+
+	addr, err := net.ResolveTCPAddr("tcp", fmt.Sprintf("%s:%d", n.ip, n.port))
+	if err != nil {
+		return fmt.Errorf("Attach server error %s:%d: %s", n.ip, n.port, errors.ErrorStack(err))
+	}
+
+	n.l, err = net.ListenTCP("tcp", addr)
+	if err != nil {
+		return fmt.Errorf("Attach server error %s: %s", addr, errors.ErrorStack(err))
+	}
+
+	log.Infof("Attach server listening on %s:%d", n.ip, n.port)
+
+	// starts serving requests immediately
+	n.c = NewConnector(n.l)
+	n.c.Start()
+
+	return nil
+}
+
+// Stop stops the connector
+func (n *Server) Stop() error {
+	defer trace.End(trace.Begin(""))
+
+	n.m.Lock()
+	defer n.m.Unlock()
+
+	err := n.l.Close()
+	n.c.Stop()
+
+	return err
+}
+
+// Addr returns the address of the underlying listener
+func (n *Server) Addr() string {
+	defer trace.End(trace.Begin(""))
+
+	n.m.RLock()
+	defer n.m.RUnlock()
+
+	return n.l.Addr().String()
+}
+
+// Interaction returns the session interface for the given container.  If the container
+// cannot be found, this call will wait for the given timeout.
+// id is ID of the container.
+func (n *Server) Interaction(ctx context.Context, id string) (SessionInteractor, error) {
+	defer trace.End(trace.Begin(id))
+
+	n.m.RLock()
+	defer n.m.RUnlock()
+
+	return n.c.Interaction(ctx, id)
+}
+
+// RemoveInteraction removes the session interface from underlying connector
+func (n *Server) RemoveInteraction(id string) error {
+	defer trace.End(trace.Begin(id))
+
+	n.m.Lock()
+	defer n.m.Unlock()
+
+	return n.c.RemoveInteraction(id)
+}

vendor/github.com/vmware/vic/lib/portlayer/attach/communication/server_test.go

@@ -0,0 +1,185 @@
+// Copyright 2016 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package communication
+
+import (
+	"net"
+	"sync"
+	"testing"
+	"time"
+
+	"context"
+
+	"golang.org/x/crypto/ssh"
+	"golang.org/x/crypto/ssh/testdata"
+
+	log "github.com/Sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
+
+	"github.com/vmware/vic/lib/migration/feature"
+	"github.com/vmware/vic/lib/tether/msgs"
+	"github.com/vmware/vic/pkg/serial"
+)
+
+// Start the server, make 200 client connections, test they connect, then Stop.
+func TestAttachStartStop(t *testing.T) {
+	log.SetLevel(log.InfoLevel)
+	if testing.Verbose() {
+		log.SetLevel(log.DebugLevel)
+	}
+	s := NewServer("localhost", 0)
+
+	var wg sync.WaitGroup
+
+	dial := func() {
+		defer wg.Done()
+
+		c, err := net.Dial("tcp", s.l.Addr().String())
+		if !assert.NoError(t, err) || !assert.NotNil(t, c) {
+			return
+		}
+		defer c.Close()
+
+		buf := make([]byte, 1)
+		c.SetReadDeadline(time.Now().Add(time.Second))
+		c.Read(buf)
+
+		// This will pass if the client has written a second syn packet by the time it's called. As such we set an
+		// unbounded readdeadline on the connection.
+		// We can assert behaviours that take a while, but cannot reliably assert behaviours that require fast scheduling
+		// of lots of threads on all systems running the CI.
+		c.SetReadDeadline(time.Time{})
+		if !assert.NoError(t, serial.HandshakeServer(c), "Expected handshake to succeed on 2nd syn packet from client") {
+			return
+		}
+	}
+
+	assert.NoError(t, s.Start())
+
+	for i := 0; i < 100; i++ {
+		wg.Add(1)
+		go dial()
+	}
+
+	done := make(chan bool)
+	go func() {
+		wg.Wait()
+		close(done)
+	}()
+
+	select {
+	case <-done:
+	case <-time.After(10 * time.Second):
+		t.Fail()
+	}
+	assert.NoError(t, s.Stop())
+
+	_, err := net.Dial("tcp", s.Addr())
+	assert.Error(t, err)
+}
+
+func TestAttachSshSession(t *testing.T) {
+	log.SetLevel(log.InfoLevel)
+	if testing.Verbose() {
+		log.SetLevel(log.DebugLevel)
+	}
+	s := NewServer("localhost", 0)
+	assert.NoError(t, s.Start())
+	defer s.Stop()
+
+	expectedID := "foo"
+
+	// This should block until the ssh server returns its container ID
+	var wg sync.WaitGroup
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+
+		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		defer cancel()
+
+		_, err := s.c.Interaction(ctx, expectedID)
+		if !assert.NoError(t, err) {
+			return
+		}
+	}()
+
+	// Dial the attach server.  This is a TCP client
+	networkClientCon, err := net.Dial("tcp", s.Addr())
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	if !assert.NoError(t, serial.HandshakeServer(networkClientCon)) {
+		return
+	}
+
+	containerConfig := &ssh.ServerConfig{
+		NoClientAuth: true,
+	}
+
+	signer, err := ssh.ParsePrivateKey(testdata.PEMBytes["dsa"])
+	if !assert.NoError(t, err) {
+		return
+	}
+	containerConfig.AddHostKey(signer)
+
+	// create the SSH server on the client.  The attach server will ssh connect to this.
+	sshConn, chans, reqs, err := ssh.NewServerConn(networkClientCon, containerConfig)
+	if !assert.NoError(t, err) {
+		return
+	}
+	defer sshConn.Close()
+
+	// Service the incoming Channel channel.
+	wg.Add(2)
+	go func() {
+		defer wg.Done()
+		exit := 0
+		for req := range reqs {
+			if req.Type == msgs.ContainersReq {
+				msg := msgs.ContainersMsg{IDs: []string{expectedID}}
+				req.Reply(true, msg.Marshal())
+				exit++
+			}
+			if req.Type == msgs.VersionReq {
+				msg := msgs.VersionMsg{Version: feature.MaxPluginVersion - 1}
+				req.Reply(true, msg.Marshal())
+				exit++
+			}
+			if exit == 2 {
+				break
+			}
+		}
+	}()
+
+	go func() {
+		defer wg.Done()
+		for ch := range chans {
+			assert.Equal(t, ch.ChannelType(), attachChannelType)
+			_, reqs, _ = ch.Accept()
+			for req := range reqs {
+				if req.Type == msgs.UnblockReq {
+					req.Reply(true, nil)
+					break
+				}
+			}
+			break
+		}
+	}()
+
+	wg.Wait()
+}

vendor/github.com/vmware/vic/lib/portlayer/attach/join.go

@@ -0,0 +1,61 @@
+// Copyright 2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package attach
+
+import (
+	"fmt"
+
+	"github.com/vmware/govmomi/vim25/types"
+	"github.com/vmware/vic/lib/constants"
+	"github.com/vmware/vic/lib/portlayer/exec"
+	"github.com/vmware/vic/pkg/trace"
+)
+
+// Join adds network backed serial port to the caller and configures them
+func Join(h interface{}) (interface{}, error) {
+	defer trace.End(trace.Begin(""))
+
+	handle, ok := h.(*exec.Handle)
+	if !ok {
+		return nil, fmt.Errorf("Type assertion failed for %#+v", handle)
+	}
+
+	// Tether serial port - backed by network
+	serial := &types.VirtualSerialPort{
+		VirtualDevice: types.VirtualDevice{
+			Backing: &types.VirtualSerialPortURIBackingInfo{
+				VirtualDeviceURIBackingInfo: types.VirtualDeviceURIBackingInfo{
+					Direction: string(types.VirtualDeviceURIBackingOptionDirectionClient),
+					ProxyURI:  fmt.Sprintf("telnet://0.0.0.0:%d", constants.SerialOverLANPort),
+					// Set it to 0.0.0.0 during Join call, VCH IP will be set when we call Bind
+					ServiceURI: fmt.Sprintf("tcp://127.0.0.1:%d", constants.AttachServerPort),
+				},
+			},
+			Connectable: &types.VirtualDeviceConnectInfo{
+				Connected:         false,
+				StartConnected:    false,
+				AllowGuestControl: true,
+			},
+		},
+		YieldOnPoll: true,
+	}
+	config := &types.VirtualDeviceConfigSpec{
+		Device:    serial,
+		Operation: types.VirtualDeviceConfigSpecOperationAdd,
+	}
+	handle.Spec.DeviceChange = append(handle.Spec.DeviceChange, config)
+
+	return handle, nil
+}

vendor/github.com/vmware/vic/lib/portlayer/attach/unbind.go

@@ -0,0 +1,36 @@
+// Copyright 2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package attach
+
+import (
+	"fmt"
+
+	"github.com/vmware/vic/lib/portlayer/exec"
+	"github.com/vmware/vic/pkg/trace"
+)
+
+// Unbind unsets the *Connected fields of the VirtualSerialPort
+func Unbind(h interface{}, id string) (interface{}, error) {
+	defer trace.End(trace.Begin(""))
+
+	handle, ok := h.(*exec.Handle)
+	if !ok {
+		return nil, fmt.Errorf("Type assertion failed for %#+v", handle)
+	}
+	if handle.MigrationError != nil {
+		return nil, fmt.Errorf("Migration failed %s", handle.MigrationError)
+	}
+	return toggle(handle, id, false)
+}