VMware vSphere Integrated Containers provider (#206)

* Add Virtual Kubelet provider for VIC

Initial virtual kubelet provider for VMware VIC.  This provider currently
handles creating and starting of a pod VM via the VIC portlayer and persona
server.  Image store handling via the VIC persona server.  This provider
currently requires the feature/wolfpack branch of VIC.

* Added pod stop and delete.  Also added node capacity.

Added the ability to stop and delete pod VMs via VIC.  Also retrieve
node capacity information from the VCH.

* Cleanup and readme file

Some file clean up and added a Readme.md markdown file for the VIC
provider.

* Cleaned up errors, added function comments, moved operation code

1. Cleaned up error handling.  Set standard for creating errors.
2. Added method prototype comments for all interface functions.
3. Moved PodCreator, PodStarter, PodStopper, and PodDeleter to a new folder.

* Add mocking code and unit tests for podcache, podcreator, and podstarter

Used the unit test framework used in VIC to handle assertions in the provider's
unit test.  Mocking code generated using OSS project mockery, which is compatible
with the testify assertion framework.

* Vendored packages for the VIC provider

Requires feature/wolfpack branch of VIC and a few specific commit sha of
projects used within VIC.

* Implementation of POD Stopper and Deleter unit tests (#4)

* Updated files for initial PR

vendor/github.com/vmware/vic/lib/portlayer/exec/base.go

@@ -0,0 +1,440 @@
+// Copyright 2016-2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package exec
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"golang.org/x/crypto/ssh"
+
+	"github.com/vmware/govmomi/guest"
+	"github.com/vmware/govmomi/task"
+	"github.com/vmware/govmomi/vim25/mo"
+	"github.com/vmware/govmomi/vim25/soap"
+	"github.com/vmware/govmomi/vim25/types"
+	"github.com/vmware/vic/lib/config/executor"
+	"github.com/vmware/vic/lib/migration"
+	"github.com/vmware/vic/pkg/trace"
+	"github.com/vmware/vic/pkg/vsphere/extraconfig"
+	"github.com/vmware/vic/pkg/vsphere/extraconfig/vmomi"
+	"github.com/vmware/vic/pkg/vsphere/tasks"
+	"github.com/vmware/vic/pkg/vsphere/vm"
+)
+
+// NotYetExistError is returned when a call that requires a VM exist is made
+type NotYetExistError struct {
+	ID string
+}
+
+func (e NotYetExistError) Error() string {
+	return fmt.Sprintf("%s is not completely created", e.ID)
+}
+
+// containerBase holds fields common between Handle and Container. The fields and
+// methods in containerBase should not require locking as they're primary use is:
+// a. for read-only reference when used in Container
+// b. single use/no-concurrent modification when used in Handle
+type containerBase struct {
+	ExecConfig *executor.ExecutorConfig
+
+	// Migrated is used during in memory migration to assign whether an execConfig is viable for a commit phase
+	Migrated bool
+	// MigrationError means the errors happens during data migration, some operation might fail for we cannot extract the whole container configuration
+	MigrationError error
+	DataVersion    int
+
+	// original - can be pointers so long as refreshes
+	// use different instances of the structures
+	Config  *types.VirtualMachineConfigInfo
+	Runtime *types.VirtualMachineRuntimeInfo
+
+	// doesn't change so can be copied here
+	vm *vm.VirtualMachine
+}
+
+func newBase(vm *vm.VirtualMachine, c *types.VirtualMachineConfigInfo, r *types.VirtualMachineRuntimeInfo) *containerBase {
+	base := &containerBase{
+		ExecConfig: &executor.ExecutorConfig{},
+		Config:     c,
+		Runtime:    r,
+		vm:         vm,
+	}
+
+	// construct a working copy of the exec config
+	if c != nil && c.ExtraConfig != nil {
+		var migratedConf map[string]string
+		containerExecKeyValues := vmomi.OptionValueMap(c.ExtraConfig)
+		// #nosec: Errors unhandled.
+		base.DataVersion, _ = migration.ContainerDataVersion(containerExecKeyValues)
+		migratedConf, base.Migrated, base.MigrationError = migration.MigrateContainerConfig(containerExecKeyValues)
+		extraconfig.Decode(extraconfig.MapSource(migratedConf), base.ExecConfig)
+	}
+
+	return base
+}
+
+// String returns the string representation of ContainerBase
+func (c *containerBase) String() string {
+	return c.ExecConfig.ID
+}
+
+// VMReference will provide the vSphere vm managed object reference
+func (c *containerBase) VMReference() types.ManagedObjectReference {
+	var moref types.ManagedObjectReference
+	if c.vm != nil {
+		moref = c.vm.Reference()
+	}
+	return moref
+}
+
+// unlocked refresh of container state
+func (c *containerBase) refresh(op trace.Operation) error {
+	base, err := c.updates(op)
+	if err != nil {
+		op.Errorf("Update: unable to update container %s: %s", c, err)
+		return err
+	}
+
+	// copy over the new state
+	*c = *base
+	return nil
+}
+
+// updates acquires updates from the infrastructure without holding a lock
+func (c *containerBase) updates(op trace.Operation) (*containerBase, error) {
+	defer trace.End(trace.Begin(c.ExecConfig.ID, op))
+
+	var o mo.VirtualMachine
+
+	// make sure we have vm
+	if c.vm == nil {
+		return nil, NotYetExistError{c.ExecConfig.ID}
+	}
+
+	if c.Config != nil {
+		op.Debugf("Update: for %s, refreshing from change version %s", c, c.Config.ChangeVersion)
+	}
+
+	if err := c.vm.Properties(op, c.vm.Reference(), []string{"config", "runtime"}, &o); err != nil {
+		return nil, err
+	}
+
+	base := &containerBase{
+		vm:         c.vm,
+		Config:     o.Config,
+		Runtime:    &o.Runtime,
+		ExecConfig: &executor.ExecutorConfig{},
+	}
+
+	// Get the ExtraConfig
+	var migratedConf map[string]string
+	containerExecKeyValues := vmomi.OptionValueMap(o.Config.ExtraConfig)
+	if containerExecKeyValues["guestinfo.vice./common/id"] == "" {
+		return nil, fmt.Errorf("Update: change version %s failed assertion extraconfig id != nil", o.Config.ChangeVersion)
+	}
+
+	op.Debugf("Update: for %s, change version %s, extraconfig id: %+v", c, o.Config.ChangeVersion, containerExecKeyValues["guestinfo.vice./common/id"])
+	// #nosec: Errors unhandled.
+	base.DataVersion, _ = migration.ContainerDataVersion(containerExecKeyValues)
+	migratedConf, base.Migrated, base.MigrationError = migration.MigrateContainerConfig(containerExecKeyValues)
+	extraconfig.Decode(extraconfig.MapSource(migratedConf), base.ExecConfig)
+
+	return base, nil
+}
+
+func (c *containerBase) ReloadConfig(op trace.Operation) error {
+	defer trace.End(trace.Begin(c.ExecConfig.ID, op))
+
+	return c.startGuestProgram(op, "reload", "")
+}
+
+// WaitForExec waits exec'ed task to set started field or timeout
+func (c *containerBase) WaitForExec(op trace.Operation, id string) error {
+	defer trace.End(trace.Begin(id, op))
+
+	return c.waitForExec(op, id)
+}
+
+// WaitForSession waits non-exec'ed task to set started field or timeout
+func (c *containerBase) WaitForSession(ctx context.Context, id string) error {
+	defer trace.End(trace.Begin(id, ctx))
+
+	return c.waitForSession(ctx, id)
+}
+
+func (c *containerBase) startGuestProgram(op trace.Operation, name string, args string) error {
+	// make sure we have vm
+	if c.vm == nil {
+		return NotYetExistError{c.ExecConfig.ID}
+	}
+
+	defer trace.End(trace.Begin(c.ExecConfig.ID+":"+name, op))
+	o := guest.NewOperationsManager(c.vm.Client.Client, c.vm.Reference())
+	m, err := o.ProcessManager(op)
+	if err != nil {
+		return err
+	}
+
+	spec := types.GuestProgramSpec{
+		ProgramPath: name,
+		Arguments:   args,
+	}
+
+	auth := types.NamePasswordAuthentication{
+		Username: c.ExecConfig.ID,
+	}
+
+	_, err = m.StartProgram(op, &auth, &spec)
+
+	return err
+}
+
+func (c *containerBase) start(op trace.Operation) error {
+	defer trace.End(trace.Begin(c.ExecConfig.ID, op))
+
+	// make sure we have vm
+	if c.vm == nil {
+		return NotYetExistError{c.ExecConfig.ID}
+	}
+
+	// Power on
+	_, err := c.vm.WaitForResult(op, func(op context.Context) (tasks.Task, error) {
+		return c.vm.PowerOn(op)
+	})
+
+	return err
+}
+
+func (c *containerBase) stop(op trace.Operation, waitTime *int32) error {
+	// make sure we have vm
+	if c.vm == nil {
+		return NotYetExistError{c.ExecConfig.ID}
+	}
+
+	// get existing state and set to stopping
+	// if there's a failure we'll revert to existing
+
+	err := c.shutdown(op, waitTime)
+	if err == nil {
+		return nil
+	}
+
+	op.Warnf("stopping %s via hard power off due to: %s", c, err.Error())
+
+	return c.poweroff(op)
+}
+
+func (c *containerBase) kill(op trace.Operation) error {
+	// make sure we have vm
+	if c.vm == nil {
+		return NotYetExistError{c.ExecConfig.ID}
+	}
+
+	wait := 10 * time.Second // default
+	timeout, cancel := trace.WithTimeout(&op, wait, "kill")
+	defer cancel()
+
+	sig := string(ssh.SIGKILL)
+	timeout.Infof("sending kill -%s %s", sig, c)
+
+	err := c.startGuestProgram(timeout, "kill", sig)
+	if err == nil && timeout.Err() != nil {
+		timeout.Warnf("timeout (%s) waiting for %s to power off via SIG%s", wait, c, sig)
+	}
+	if err != nil {
+		timeout.Warnf("killing %s attempt resulted in: %s", c, err.Error())
+
+		if isInvalidPowerStateError(err) {
+			return nil
+		}
+	}
+
+	// Even if startGuestProgram failed above, it may actually have executed.  If the container came up and then
+	// we kill it before VC gets a chance to detect the toolbox, vSphere can execute the kill but report an
+	// error 3016 indicating the guest toolbox wasn't found.  If we then try to poweroff, it may throw vSphere
+	// into an invalid transition and will need to recover.  If we try to grab properties at this time, the
+	// power state may be incorrect.  We work around this by waiting on the power state, regardless of error
+	// from startGuestProgram. https://github.com/vmware/vic/issues/5803
+	timeout.Infof("waiting %s for %s to power off", wait, c)
+	err = c.vm.WaitForPowerState(timeout, types.VirtualMachinePowerStatePoweredOff)
+	if err == nil {
+		return nil // VM has powered off
+	}
+
+	timeout.Warnf("killing %s via hard power off", c)
+
+	// stop wait time is not applied for the hard kill
+	return c.poweroff(op)
+}
+
+func (c *containerBase) shutdown(op trace.Operation, waitTime *int32) error {
+	// make sure we have vm
+	if c.vm == nil {
+		return NotYetExistError{c.ExecConfig.ID}
+	}
+
+	wait := 10 * time.Second // default
+	if waitTime != nil && *waitTime > 0 {
+		wait = time.Duration(*waitTime) * time.Second
+	}
+
+	cs := c.ExecConfig.Sessions[c.ExecConfig.ID]
+	stop := []string{cs.StopSignal, string(ssh.SIGKILL)}
+	if stop[0] == "" {
+		stop[0] = string(ssh.SIGTERM)
+	}
+
+	var killed bool
+
+	for _, sig := range stop {
+		msg := fmt.Sprintf("sending kill -%s %s", sig, c)
+		op.Infof(msg)
+
+		timeout, cancel := trace.WithTimeout(&op, wait, "shutdown")
+		defer cancel()
+
+		err := c.startGuestProgram(timeout, "kill", sig)
+		if err != nil {
+			// Just warn and proceed to waiting for power state per issue https://github.com/vmware/vic/issues/5803
+			// Description above in function kill()
+			timeout.Warnf("%s: %s", msg, err)
+
+			// If the error tells us "The attempted operation cannot be performed in the current state (Powered off)" (InvalidPowerState),
+			// we can avoid hard poweroff (issues #6236 and #6252). Here we wait for the power state changes instead of return
+			// immediately to avoid excess vSphere queries
+			if isInvalidPowerStateError(err) {
+				killed = true
+			}
+		}
+
+		timeout.Infof("waiting %s for %s to power off", wait, c)
+		err = c.vm.WaitForPowerState(timeout, types.VirtualMachinePowerStatePoweredOff)
+		if err == nil {
+			return nil // VM has powered off
+		}
+
+		if timeout.Err() == nil {
+			return err // error other than timeout
+		}
+
+		timeout.Warnf("timeout (%s) waiting for %s to power off via SIG%s", wait, c, sig)
+
+		if killed {
+			return nil
+		}
+	}
+
+	return fmt.Errorf("failed to shutdown %s via kill signals %s", c, stop)
+}
+
+func (c *containerBase) poweroff(op trace.Operation) error {
+	// make sure we have vm
+	if c.vm == nil {
+		return NotYetExistError{c.ExecConfig.ID}
+	}
+
+	_, err := c.vm.WaitForResult(op, func(op context.Context) (tasks.Task, error) {
+		return c.vm.PowerOff(op)
+	})
+
+	if err != nil {
+
+		// It is possible the VM has finally shutdown in between, ignore the error in that case
+		if terr, ok := err.(task.Error); ok {
+			switch terr := terr.Fault().(type) {
+			case *types.InvalidPowerState:
+				if terr.ExistingState == types.VirtualMachinePowerStatePoweredOff {
+					op.Warnf("power off %s task skipped (state was already %s)", c, terr.ExistingState)
+					return nil
+				}
+				op.Warnf("invalid power state during power off: %s", terr.ExistingState)
+
+			case *types.GenericVmConfigFault:
+
+				// Check if the poweroff task was canceled due to a concurrent guest shutdown
+				if len(terr.FaultMessage) > 0 {
+					k := terr.FaultMessage[0].Key
+					if k == vmNotSuspendedKey || k == vmPoweringOffKey {
+						op.Infof("power off %s task skipped due to guest shutdown", c)
+						return nil
+					}
+				}
+				op.Warnf("generic vm config fault during power off: %#v", terr)
+
+			default:
+				op.Warnf("hard power off failed due to: %#v", terr)
+			}
+		}
+
+		return err
+	}
+
+	return nil
+}
+
+func (c *containerBase) waitForPowerState(ctx context.Context, max time.Duration, state types.VirtualMachinePowerState) (bool, error) {
+	defer trace.End(trace.Begin(c.ExecConfig.ID, ctx))
+
+	timeout, cancel := context.WithTimeout(ctx, max)
+	defer cancel()
+
+	err := c.vm.WaitForPowerState(timeout, state)
+	if err != nil {
+		return timeout.Err() != nil, err
+	}
+
+	return false, nil
+}
+
+func (c *containerBase) waitForSession(ctx context.Context, id string) error {
+	defer trace.End(trace.Begin(id, ctx))
+
+	// guestinfo key that we want to wait for
+	key := extraconfig.CalculateKeys(c.ExecConfig, fmt.Sprintf("Sessions.%s.Started", id), "")[0]
+	return c.waitFor(ctx, key)
+}
+
+func (c *containerBase) waitForExec(op trace.Operation, id string) error {
+	defer trace.End(trace.Begin(id, op))
+
+	// guestinfo key that we want to wait for
+	key := extraconfig.CalculateKeys(c.ExecConfig, fmt.Sprintf("Execs.%s.Started", id), "")[0]
+	return c.waitFor(op, key)
+}
+
+func (c *containerBase) waitFor(ctx context.Context, key string) error {
+	detail, err := c.vm.WaitForKeyInExtraConfig(ctx, key)
+	if err != nil {
+		return fmt.Errorf("unable to wait for process launch status: %s", err)
+	}
+
+	if detail != "true" {
+		return fmt.Errorf("%s", detail)
+	}
+
+	return nil
+}
+
+// isInvalidPowerStateError verifies if an error is the InvalidPowerStateError
+func isInvalidPowerStateError(err error) bool {
+	if soap.IsSoapFault(err) {
+		_, ok := soap.ToSoapFault(err).VimFault().(types.InvalidPowerState)
+		return ok
+	}
+
+	return false
+}

vendor/github.com/vmware/vic/lib/portlayer/exec/commit.go

@@ -0,0 +1,255 @@
+// Copyright 2016-2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package exec
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/vmware/govmomi/vim25/types"
+	"github.com/vmware/vic/lib/portlayer/event/events"
+	"github.com/vmware/vic/pkg/retry"
+	"github.com/vmware/vic/pkg/trace"
+	"github.com/vmware/vic/pkg/vsphere/session"
+	"github.com/vmware/vic/pkg/vsphere/tasks"
+	"github.com/vmware/vic/pkg/vsphere/vm"
+)
+
+// Commit executes the requires steps on the handle
+func Commit(op trace.Operation, sess *session.Session, h *Handle, waitTime *int32) error {
+	defer trace.End(trace.Begin(h.ExecConfig.ID, op))
+
+	c := Containers.Container(h.ExecConfig.ID)
+	creation := h.vm == nil
+	if creation {
+		if h.Spec == nil {
+			return fmt.Errorf("a spec must be provided for create operations")
+		}
+
+		if sess == nil {
+			// session must not be nil
+			return fmt.Errorf("no session provided for create operations")
+		}
+
+		// the only permissible operation is to create a VM
+		if h.Spec == nil {
+			return fmt.Errorf("only create operations can be committed without an existing VM")
+		}
+
+		if c != nil {
+			return fmt.Errorf("a container already exists in the cache with this ID")
+		}
+
+		var res *types.TaskInfo
+		var err error
+		if sess.IsVC() && Config.VirtualApp.ResourcePool != nil {
+			// Create the vm
+			res, err = tasks.WaitForResult(op, func(op context.Context) (tasks.Task, error) {
+				return Config.VirtualApp.CreateChildVM(op, *h.Spec.Spec(), nil)
+			})
+		} else {
+			// Create the vm
+			res, err = tasks.WaitForResult(op, func(op context.Context) (tasks.Task, error) {
+				return sess.VMFolder.CreateVM(op, *h.Spec.Spec(), Config.ResourcePool, nil)
+			})
+		}
+
+		if err != nil {
+			op.Errorf("An error occurred while waiting for a creation operation to complete. Spec was %+v", *h.Spec.Spec())
+			return err
+		}
+
+		h.vm = vm.NewVirtualMachine(op, sess, res.Result.(types.ManagedObjectReference))
+		h.vm.DisableDestroy(op)
+		c = newContainer(&h.containerBase)
+		Containers.Put(c)
+		// inform of creation irrespective of remaining operations
+		publishContainerEvent(op, c.ExecConfig.ID, time.Now().UTC(), events.ContainerCreated)
+
+		// clear the spec as we've acted on it - this prevents a reconfigure from occurring in follow-on
+		// processing
+		h.Spec = nil
+	}
+
+	// if we're stopping the VM, do so before the reconfigure to preserve the extraconfig
+	if h.TargetState() == StateStopped {
+		if h.Runtime == nil {
+			op.Warnf("Commit called with incomplete runtime state for %s", h.ExecConfig.ID)
+		}
+
+		if h.Runtime != nil && h.Runtime.PowerState == types.VirtualMachinePowerStatePoweredOff {
+			op.Infof("Dropping duplicate power off operation for %s", h.ExecConfig.ID)
+		} else {
+			// stop the container
+			if err := c.stop(op, waitTime); err != nil {
+				return err
+			}
+
+			// we must refresh now to get the new ChangeVersion - this is used to gate on powerstate in the reconfigure
+			// because we cannot set the ExtraConfig if the VM is powered on. There is still a race here unfortunately because
+			// tasks don't appear to contain the new ChangeVersion
+			h.refresh(op)
+
+			// inform of state change irrespective of remaining operations - but allow remaining operations to complete first
+			// to avoid data race on container config
+			defer publishContainerEvent(op, h.ExecConfig.ID, time.Now().UTC(), events.ContainerStopped)
+		}
+	}
+
+	// reconfigure operation
+	if h.Spec != nil {
+		if h.Runtime == nil {
+			op.Errorf("Refusing to perform reconfigure operation with incomplete runtime state for %s", h.ExecConfig.ID)
+		} else {
+			// ensure that our logic based on Runtime state remains valid
+
+			// NOTE: this inline refresh can be removed when switching away from guestinfo where we have non-persistence issues
+			// when updating ExtraConfig via the API with a powered on VM - we therefore have to be absolutely certain about the
+			// power state to decide if we can continue without nilifying extraconfig
+			//
+			// For the power off path this depends on handle.refresh() having been called to update the ChangeVersion
+			s := h.Spec.Spec()
+
+			op.Infof("Reconfigure: attempting update to %s with change version %q (%s)", h.ExecConfig.ID, s.ChangeVersion, h.Runtime.PowerState)
+
+			// nilify ExtraConfig if container configuration is migrated
+			// in this case, VCH and container are in different version. Migrated configuration cannot be written back to old container, to avoid data loss in old version's container
+			if h.Migrated {
+				op.Debugf("Reconfigure: dropping extraconfig as configuration of container %s is migrated", h.ExecConfig.ID)
+				s.ExtraConfig = nil
+			}
+
+			// address the race between power operation and refresh of config (and therefore ChangeVersion) in StateStopped block above
+			if s.ExtraConfig != nil && h.TargetState() == StateStopped && h.Runtime.PowerState != types.VirtualMachinePowerStatePoweredOff {
+				detail := fmt.Sprintf("Reconfigure: collision of concurrent operations - expected power state poweredOff, found %s", h.Runtime.PowerState)
+				op.Warnf(detail)
+
+				// log out current vm power state and runtime power state got from refresh, to see if there is anything mismatch,
+				// cause in issue #6127, we see the runtime power state is not updated even after 1 minute
+				ps, _ := h.vm.PowerState(op)
+				op.Debugf("Container %s power state: %s, runtime power state: %s", h.ExecConfig.ID, ps, h.Runtime.PowerState)
+				// this should cause a second attempt at the power op. This could result repeated contention that fails to resolve, but the randomness in the backoff and the tight timing
+				// to hit this scenario should mean it will resolve in a reasonable timeframe.
+				return ConcurrentAccessError{errors.New(detail)}
+			}
+
+			_, err := h.vm.WaitForResult(op, func(op context.Context) (tasks.Task, error) {
+				return h.vm.Reconfigure(op, *s)
+			})
+			if err != nil {
+				op.Errorf("Reconfigure: failed update to %s with change version %s: %+v", h.ExecConfig.ID, s.ChangeVersion, err)
+
+				// Check whether we get ConcurrentAccess and wrap it if needed
+				if f, ok := err.(types.HasFault); ok {
+					switch f.Fault().(type) {
+					case *types.ConcurrentAccess:
+						op.Errorf("Reconfigure: failed update to %s due to ConcurrentAccess, our change version %s", h.ExecConfig.ID, s.ChangeVersion)
+
+						return ConcurrentAccessError{err}
+					}
+				}
+				return err
+			}
+
+			op.Infof("Reconfigure: committed update to %s with change version: %s", h.ExecConfig.ID, s.ChangeVersion)
+
+			// trigger a configuration reload in the container if needed
+			err = reloadConfig(op, h, c)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	// best effort update of container cache using committed state - this will not reflect the power on below, however
+	// this is primarily for updating ExtraConfig state.
+	if !creation {
+		defer c.RefreshFromHandle(op, h)
+	}
+
+	if h.TargetState() == StateRunning {
+		if h.Runtime != nil && h.Runtime.PowerState == types.VirtualMachinePowerStatePoweredOn {
+			op.Infof("Dropping duplicate power on operation for %s", h.ExecConfig.ID)
+			return nil
+		}
+
+		if h.Runtime == nil && !creation {
+			op.Warnf("Commit called with incomplete runtime state for %s", h.ExecConfig.ID)
+		}
+
+		// start the container
+		if err := c.start(op); err != nil {
+			// We observed that PowerOn_Task could get stuck on VC time to time even though the VM was starting fine on the host ESXi.
+			// Eventually the task was getting timed out (After 20 min.) and that was setting the container state back to Stopped.
+			// During that time VC was not generating any other event so the persona listener was getting nothing.
+			// This new event is for signaling the eventmonitor so that it can autoremove the container after this failure.
+			publishContainerEvent(op, h.ExecConfig.ID, time.Now().UTC(), events.ContainerFailed)
+			return err
+		}
+
+		// publish started event
+		publishContainerEvent(op, h.ExecConfig.ID, time.Now().UTC(), events.ContainerStarted)
+	}
+
+	return nil
+}
+
+// HELPER FUNCTIONS BELOW
+
+// reloadConfig is responsible for triggering a guest_reconfigure in order to perform an operation on a running cVM
+// this function needs to be resilient to intermittent config errors and task errors, but will pass concurrent
+// modification issues back immediately.
+func reloadConfig(op trace.Operation, h *Handle, c *Container) error {
+
+	op.Infof("Attempting to perform a guest reconfigure operation on (%s)", h.ExecConfig.ID)
+	retryFunc := func() error {
+		if h.reload && h.Runtime != nil && h.Runtime.PowerState == types.VirtualMachinePowerStatePoweredOn {
+			err := c.ReloadConfig(op)
+
+			if err != nil {
+				op.Debugf("Error occurred during an attempt to reload the container config for an exec operation: (%s)", err)
+
+				// we will request the powerstate directly(this could be very costly without the vmomi gateway)
+				state, err := c.vm.PowerState(op)
+				if err != nil && state == types.VirtualMachinePowerStatePoweredOff {
+					// TODO: probably should make this error a specific type such as PowerOffDuringExecError( or a better name ofcourse)
+					return fmt.Errorf("container(%s) was powered down during the requested operation.", h.ExecConfig.ID)
+				}
+				return err
+			}
+			return nil
+		}
+
+		// nothing to be done.
+		return nil
+	}
+
+	err := retry.Do(retryFunc, isIntermittentFailure)
+	if err != nil {
+		op.Debugf("Failed an exec operation with err: %s", err)
+		return err
+	}
+	return nil
+}
+
+// TODO: refactor later, I need to test this and we need to unify the Task package and the retry Package(make task use retry imo)
+// right now this just looks silly...
+func isIntermittentFailure(err error) bool {
+	// in the future commit should be using the trace.operation for these calls and this function can act as a passthrough.
+	op := trace.NewOperation(context.TODO(), "")
+	return tasks.IsRetryError(op, err)
+}

vendor/github.com/vmware/vic/lib/portlayer/exec/config.go

@@ -0,0 +1,56 @@
+// Copyright 2016 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package exec
+
+import (
+	"net/url"
+
+	"github.com/vmware/govmomi/object"
+	"github.com/vmware/vic/lib/config"
+	"github.com/vmware/vic/lib/config/executor"
+	"github.com/vmware/vic/lib/portlayer/event"
+)
+
+var Config Configuration
+
+// Configuration is a slice of the VCH config that is relevant to the exec part of the port layer
+type Configuration struct {
+	// Turn on debug logging
+	DebugLevel int `vic:"0.1" scope:"read-only" key:"init/diagnostics/debug"`
+
+	SysLogConfig *executor.SysLogConfig `vic:"0.1" scope:"read-only" key:"init/diagnostics/syslog"`
+
+	// Port Layer - exec
+	config.Container `vic:"0.1" scope:"read-only" key:"container"`
+
+	// Resource pool is the working version of the compute resource config
+	ResourcePool *object.ResourcePool
+	// Parent resource will be a VirtualApp on VC
+	VirtualApp *object.VirtualApp
+
+	// For now throw the Event Manager here
+	EventManager event.EventManager
+
+	// Information about the VCH resource pool and about the real host that we want
+	// tol retrieve just once.
+	VCHMhz          int64
+	VCHMemoryLimit  int64
+	HostOS          string
+	HostOSVersion   string
+	HostProductName string //'VMware vCenter Server' or 'VMare ESXi'
+
+	// Datastore URLs for image stores - the top layer is [0], the bottom layer is [len-1]
+	ImageStores []url.URL `vic:"0.1" scope:"read-only" key:"storage/image_stores"`
+}

vendor/github.com/vmware/vic/lib/portlayer/exec/container.go

@@ -0,0 +1,896 @@
+// Copyright 2016-2018 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package exec
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"path"
+	"strings"
+	"sync"
+	"syscall"
+	"time"
+
+	"github.com/vmware/govmomi/vim25/mo"
+	"github.com/vmware/govmomi/vim25/soap"
+	"github.com/vmware/govmomi/vim25/types"
+
+	"github.com/vmware/vic/lib/constants"
+	"github.com/vmware/vic/lib/iolog"
+	"github.com/vmware/vic/lib/portlayer/event/events"
+	stateevents "github.com/vmware/vic/lib/portlayer/event/events/vsphere"
+	"github.com/vmware/vic/pkg/errors"
+	"github.com/vmware/vic/pkg/trace"
+	"github.com/vmware/vic/pkg/uid"
+	"github.com/vmware/vic/pkg/vsphere/disk"
+	"github.com/vmware/vic/pkg/vsphere/session"
+	"github.com/vmware/vic/pkg/vsphere/sys"
+	"github.com/vmware/vic/pkg/vsphere/tasks"
+	"github.com/vmware/vic/pkg/vsphere/vm"
+
+	log "github.com/Sirupsen/logrus"
+	"github.com/google/uuid"
+)
+
+type State int
+
+const (
+	StateUnknown State = iota
+	StateStarting
+	StateRunning
+	StateStopping
+	StateStopped
+	StateSuspending
+	StateSuspended
+	StateCreated
+	StateCreating
+	StateRemoving
+	StateRemoved
+
+	containerLogName = "output.log"
+
+	vmNotSuspendedKey = "msg.suspend.powerOff.notsuspended"
+	vmPoweringOffKey  = "msg.rpc.error.poweringoff"
+)
+
+func (s State) String() string {
+	switch s {
+	case StateCreated:
+		return "Created"
+	case StateStarting:
+		return "Starting"
+	case StateRunning:
+		return "Running"
+	case StateRemoving:
+		return "Removing"
+	case StateRemoved:
+		return "Removed"
+	case StateStopping:
+		return "Stopping"
+	case StateStopped:
+		return "Stopped"
+	case StateUnknown:
+		return "Unknown"
+	}
+	return ""
+}
+
+// NotFoundError is returned when a types.ManagedObjectNotFound is returned from a vmomi call
+type NotFoundError struct {
+	err error
+}
+
+func (r NotFoundError) Error() string {
+	return "VM has either been deleted or has not been fully created"
+}
+
+func IsNotFoundError(err error) bool {
+	if soap.IsSoapFault(err) {
+		fault := soap.ToSoapFault(err).VimFault()
+		if _, ok := fault.(types.ManagedObjectNotFound); ok {
+			return true
+		}
+	}
+	return false
+}
+
+// RemovePowerError is returned when attempting to remove a containerVM that is powered on
+type RemovePowerError struct {
+	err error
+}
+
+func (r RemovePowerError) Error() string {
+	return r.err.Error()
+}
+
+// ConcurrentAccessError is returned when concurrent calls tries to modify same object
+type ConcurrentAccessError struct {
+	err error
+}
+
+func (r ConcurrentAccessError) Error() string {
+	return r.err.Error()
+}
+
+func IsConcurrentAccessError(err error) bool {
+	_, ok := err.(ConcurrentAccessError)
+	return ok
+}
+
+type DevicesInUseError struct {
+	Devices []string
+}
+
+func (e DevicesInUseError) Error() string {
+	return fmt.Sprintf("device %s in use", strings.Join(e.Devices, ","))
+}
+
+// Container is used to return data about a container during inspection calls
+// It is a copy rather than a live reflection and does not require locking
+type ContainerInfo struct {
+	containerBase
+
+	state State
+
+	// Size of the leaf (unused)
+	VMUnsharedDisk int64
+}
+
+// Container is used for an entry in the container cache - this is a "live" representation
+// of containers in the infrastructure.
+// DANGEROUS USAGE CONSTRAINTS:
+//   None of the containerBase fields should be partially updated - consider them immutable once they're
+//   part of a cache entry
+//   i.e. Do not make changes in containerBase.ExecConfig - only swap, under lock, the pointer for a
+//   completely new ExecConfig.
+//   This constraint allows us to avoid deep copying those structs every time a container is inspected
+type Container struct {
+	m sync.Mutex
+
+	ContainerInfo
+
+	logFollowers []io.Closer
+
+	newStateEvents map[State]chan struct{}
+}
+
+// newContainer constructs a Container suitable for adding to the cache
+// it's state is set from the Runtime.PowerState field, or StateCreated if that is not
+// viable
+// This copies (shallow) the containerBase that's provided
+func newContainer(base *containerBase) *Container {
+	c := &Container{
+		ContainerInfo: ContainerInfo{
+			containerBase: *base,
+			state:         StateCreated,
+		},
+		newStateEvents: make(map[State]chan struct{}),
+	}
+
+	// if this is a creation path, then Runtime will be nil
+	if base.Runtime != nil {
+		// set state
+		switch base.Runtime.PowerState {
+		case types.VirtualMachinePowerStatePoweredOn:
+			// the containerVM is poweredOn, so set state to starting
+			// then check to see if a start was successful
+			c.state = StateStarting
+			// If any sessions successfully started then set to running
+			for _, s := range base.ExecConfig.Sessions {
+				if s.Started != "" {
+					c.state = StateRunning
+					break
+				}
+			}
+		case types.VirtualMachinePowerStatePoweredOff:
+			// check if any of the sessions was started
+			for _, s := range base.ExecConfig.Sessions {
+				if s.Started != "" {
+					c.state = StateStopped
+					break
+				}
+			}
+		case types.VirtualMachinePowerStateSuspended:
+			c.state = StateSuspended
+			log.Warnf("container VM %s: invalid power state %s", base.vm.Reference(), base.Runtime.PowerState)
+		}
+	}
+
+	return c
+}
+
+func GetContainer(ctx context.Context, id uid.UID) *Handle {
+	// get from the cache
+	container := Containers.Container(id.String())
+	if container != nil {
+		return container.NewHandle(ctx)
+	}
+
+	return nil
+}
+
+func (c *ContainerInfo) String() string {
+	return c.ExecConfig.ID
+}
+
+// State returns the state at the time the ContainerInfo object was created
+func (c *ContainerInfo) State() State {
+	return c.state
+}
+
+func (c *Container) String() string {
+	return c.ExecConfig.ID
+}
+
+// Info returns a copy of the public container configuration that
+// is consistent and copied under lock
+func (c *Container) Info() *ContainerInfo {
+	c.m.Lock()
+	defer c.m.Unlock()
+
+	info := c.ContainerInfo
+	return &info
+}
+
+// CurrentState returns current state.
+func (c *Container) CurrentState() State {
+	c.m.Lock()
+	defer c.m.Unlock()
+	return c.state
+}
+
+// SetState changes container state.
+func (c *Container) SetState(op trace.Operation, s State) State {
+	c.m.Lock()
+	defer c.m.Unlock()
+	return c.updateState(op, s)
+}
+
+func (c *Container) updateState(op trace.Operation, s State) State {
+	op.Debugf("Updating container %s state: %s->%s", c, c.state, s)
+	prevState := c.state
+	if s != c.state {
+		c.state = s
+		if ch, ok := c.newStateEvents[s]; ok {
+			delete(c.newStateEvents, s)
+			close(ch)
+		}
+	}
+	return prevState
+}
+
+// transitionState changes the container state to finalState if the current state is initialState
+// and returns an error otherwise.
+func (c *Container) transitionState(op trace.Operation, initialState, finalState State) error {
+	c.m.Lock()
+	defer c.m.Unlock()
+
+	if c.state == initialState {
+		c.state = finalState
+		op.Debugf("Set container %s state: %s->%s", c, initialState, finalState)
+		return nil
+	}
+
+	return fmt.Errorf("container state is %s and was not changed to %s", c.state, finalState)
+}
+
+var closedEventChannel = func() <-chan struct{} {
+	a := make(chan struct{})
+	close(a)
+	return a
+}()
+
+// WaitForState subscribes a caller to an event returning
+// a channel that will be closed when an expected state is set.
+// If expected state is already set the caller will receive a closed channel immediately.
+func (c *Container) WaitForState(s State) <-chan struct{} {
+	c.m.Lock()
+	defer c.m.Unlock()
+
+	if s == c.state {
+		return closedEventChannel
+	}
+
+	if ch, ok := c.newStateEvents[s]; ok {
+		return ch
+	}
+
+	eventChan := make(chan struct{})
+	c.newStateEvents[s] = eventChan
+	return eventChan
+}
+
+func (c *Container) NewHandle(ctx context.Context) *Handle {
+	// Call property collector to fill the data
+	if c.vm != nil {
+		op := trace.FromContext(ctx, "NewHandle")
+		// FIXME: this should be calling the cache to decide if a refresh is needed
+		if err := c.Refresh(op); err != nil {
+			op.Errorf("refreshing container %s failed: %s", c, err)
+			return nil // nil indicates error
+		}
+	}
+
+	// return a handle that represents zero changes over the current configuration
+	// for this container
+	return newHandle(c)
+}
+
+// Refresh updates config and runtime info, holding a lock only while swapping
+// the new data for the old
+func (c *Container) Refresh(op trace.Operation) error {
+	c.m.Lock()
+	defer c.m.Unlock()
+
+	if err := c.refresh(op); err != nil {
+		return err
+	}
+
+	// conditionally sync state (see issue 4872, 6372)
+	event := stateevents.NewStateEvent(op, c.containerBase.Runtime.PowerState, c.VMReference())
+	state := eventedState(op, event, c.state)
+
+	// trigger internal event publishing if c.state -> state is a transition we care about
+	// this will update container state and trigger follow up port layer events as needed
+	c.onEvent(op, state, event)
+
+	return nil
+}
+
+func (c *Container) refresh(op trace.Operation) error {
+	return c.containerBase.refresh(op)
+}
+
+// RefreshFromHandle updates config and runtime info, holding a lock only while swapping
+// the new data for the old
+func (c *Container) RefreshFromHandle(op trace.Operation, h *Handle) {
+	c.m.Lock()
+	defer c.m.Unlock()
+
+	if c.Config != nil && (h.Config == nil || h.Config.ChangeVersion != c.Config.ChangeVersion) {
+		op.Warnf("container and handle ChangeVersions do not match for %s: %s != %s", c, c.Config.ChangeVersion, h.Config.ChangeVersion)
+		return
+	}
+
+	// power off doesn't necessarily cause a change version increment and bug1898149 occasionally impacts power on
+	if c.Runtime != nil && (h.Runtime == nil || h.Runtime.PowerState != c.Runtime.PowerState) {
+		op.Warnf("container and handle PowerStates do not match: %s != %s", c.Runtime.PowerState, h.Runtime.PowerState)
+		return
+	}
+
+	// copy over the new state
+	c.containerBase = h.containerBase
+	if c.Config != nil {
+		op.Debugf("Update: updated change version from handle: %s", c.Config.ChangeVersion)
+	}
+}
+
+// Start starts a container vm with the given params
+func (c *Container) start(op trace.Operation) error {
+	defer trace.End(trace.Begin(c.ExecConfig.ID, op))
+
+	if c.vm == nil {
+		return fmt.Errorf("vm not set")
+	}
+	// Set state to Starting
+	c.SetState(op, StateStarting)
+
+	err := c.containerBase.start(op)
+	if err != nil {
+		// change state to stopped because start task failed
+		c.SetState(op, StateStopped)
+
+		// check if locked disk error
+		devices := disk.LockedDisks(err)
+		if len(devices) > 0 {
+			for i := range devices {
+				// get device id from datastore file path
+				// FIXME: find a reasonable way to get device ID from datastore path in exec
+				devices[i] = strings.TrimSuffix(path.Base(devices[i]), ".vmdk")
+			}
+			return DevicesInUseError{devices}
+		}
+		return err
+	}
+
+	// wait task to set started field to something
+	op, cancel := trace.WithTimeout(&op, constants.PropertyCollectorTimeout, "WaitForSession")
+	defer cancel()
+
+	err = c.waitForSession(op, c.ExecConfig.ID)
+	if err != nil {
+		// leave this in state starting - if it powers off then the event
+		// will cause transition to StateStopped which is likely our original state
+		// if the container was just taking a very long time it'll eventually
+		// become responsive.
+
+		// TODO: mechanism to trigger reinspection of long term transitional states
+		return err
+	}
+
+	// Transition the state to Running only if it's Starting.
+	// The current state is already Stopped if the container's process has exited or
+	// a poweredoff event has been processed.
+	if err = c.transitionState(op, StateStarting, StateRunning); err != nil {
+		op.Debugf(err.Error())
+	}
+
+	return nil
+}
+
+func (c *Container) stop(op trace.Operation, waitTime *int32) error {
+	defer trace.End(trace.Begin(c.ExecConfig.ID, op))
+
+	defer c.onStop()
+
+	// get existing state and set to stopping
+	// if there's a failure we'll revert to existing
+	finalState := c.SetState(op, StateStopping)
+
+	err := c.containerBase.stop(op, waitTime)
+	if err != nil {
+		// we've got no idea what state the container is in at this point
+		// running is an _optimistic_ statement
+		// If the current state is Stopping, revert it to the old state.
+		if stateErr := c.transitionState(op, StateStopping, finalState); stateErr != nil {
+			op.Debugf(stateErr.Error())
+		}
+
+		return err
+	}
+
+	// Transition the state to Stopped only if it's Stopping.
+	if err = c.transitionState(op, StateStopping, StateStopped); err != nil {
+		op.Debugf(err.Error())
+	}
+
+	return nil
+}
+
+func (c *Container) Signal(op trace.Operation, num int64) error {
+	defer trace.End(trace.Begin(c.ExecConfig.ID, op))
+
+	if c.vm == nil {
+		return fmt.Errorf("vm not set")
+	}
+
+	if num == int64(syscall.SIGKILL) {
+		return c.containerBase.kill(op)
+	}
+
+	return c.startGuestProgram(op, "kill", fmt.Sprintf("%d", num))
+}
+
+func (c *Container) onStop() {
+	lf := c.logFollowers
+	c.logFollowers = nil
+
+	log.Debugf("Container(%s) closing %d log followers", c, len(lf))
+	for _, l := range lf {
+		// #nosec: Errors unhandled.
+		_ = l.Close()
+	}
+}
+
+func (c *Container) LogReader(op trace.Operation, tail int, follow bool, since int64) (io.ReadCloser, error) {
+	defer trace.End(trace.Begin(c.ExecConfig.ID, op))
+	c.m.Lock()
+	defer c.m.Unlock()
+
+	if c.vm == nil {
+		return nil, fmt.Errorf("vm not set")
+	}
+
+	url, err := c.vm.VMPathNameAsURL(op)
+	if err != nil {
+		return nil, err
+	}
+
+	name := fmt.Sprintf("%s/%s", url.Path, containerLogName)
+
+	var via string
+
+	if c.state == StateRunning && c.vm.IsVC() {
+		// #nosec: Errors unhandled.
+		hosts, _ := c.vm.Datastore.AttachedHosts(op)
+		if len(hosts) > 1 {
+			// In this case, we need download from the VM host as it owns the file lock
+			// #nosec: Errors unhandled.
+			h, _ := c.vm.HostSystem(op)
+			if h != nil {
+				// get a context that embeds the host as a value
+				ctx := c.vm.Datastore.HostContext(op, h)
+
+				// revert the govmomi returned context to the previous op
+				// the op was preserved as a value in the context
+				op = trace.FromContext(ctx, "LogReader")
+				via = fmt.Sprintf(" via %s", h.Reference())
+			}
+		}
+	}
+
+	op.Infof("pulling %s%s", name, via)
+
+	file, err := c.vm.Datastore.Open(op, name)
+	if err != nil {
+		return nil, err
+	}
+
+	if since > 0 {
+		err = file.TailFunc(tail, func(line int, message string) bool {
+			if tail <= line && tail != -1 {
+				return false
+			}
+
+			buf := bytes.NewBufferString(message)
+
+			entry, err := iolog.ParseLogEntry(buf)
+			if err != nil {
+				op.Errorf("Error parsing log entry: %s", err.Error())
+				return false
+			}
+
+			if entry.Timestamp.Unix() <= since {
+				return false
+			}
+
+			return true
+		})
+	} else if tail >= 0 {
+		err = file.Tail(tail)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if follow && c.state == StateRunning {
+		follower := file.Follow(time.Second)
+
+		c.logFollowers = append(c.logFollowers, follower)
+
+		return follower, nil
+	}
+
+	return file, nil
+}
+
+// Remove removes a containerVM after detaching the disks
+func (c *Container) Remove(op trace.Operation, sess *session.Session) error {
+	// op := trace.FromContext(ctx, "Remove")
+	defer trace.End(trace.Begin(c.ExecConfig.ID, op))
+	c.m.Lock()
+	defer c.m.Unlock()
+
+	if c.vm == nil {
+		return NotFoundError{}
+	}
+
+	// check state first
+	if c.state == StateRunning {
+		return RemovePowerError{fmt.Errorf("Container %s is powered on", c)}
+	}
+
+	// get existing state and set to removing
+	// if there's a failure we'll revert to existing
+	existingState := c.updateState(op, StateRemoving)
+
+	// get the folder the VM is in
+	url, err := c.vm.VMPathNameAsURL(op)
+	if err != nil {
+
+		// handle the out-of-band removal case
+		if IsNotFoundError(err) {
+			Containers.Remove(c.ExecConfig.ID)
+			return NotFoundError{}
+		}
+
+		op.Errorf("Failed to get datastore path for %s: %s", c, err)
+		c.updateState(op, existingState)
+		return err
+	}
+
+	ds, err := sess.Finder.Datastore(op, url.Host)
+	if err != nil {
+		return err
+	}
+
+	// enable Destroy
+	c.vm.EnableDestroy(op)
+
+	concurrent := false
+	// if DeleteExceptDisks succeeds on VC, it leaves the VM orphan so we need to call Unregister
+	// if DeleteExceptDisks succeeds on ESXi, no further action needed
+	// if DeleteExceptDisks fails, we should call Unregister and only return an error if that fails too
+	//		Unregister sometimes can fail with ManagedObjectNotFound so we ignore it
+	_, err = c.vm.WaitForResult(op, func(op context.Context) (tasks.Task, error) {
+		return c.vm.DeleteExceptDisks(op)
+	})
+	if err != nil {
+		f, ok := err.(types.HasFault)
+		if !ok {
+			op.Warnf("DeleteExceptDisks failed with non-fault error %s for %s.", err, c)
+
+			c.updateState(op, existingState)
+			return err
+		}
+
+		switch f.Fault().(type) {
+		case *types.InvalidState:
+			op.Warnf("container VM %s is in invalid state, unregistering", c)
+			if err := c.vm.Unregister(op); err != nil {
+				op.Errorf("Error while attempting to unregister container VM %s: %s", c, err)
+				return err
+			}
+		case *types.ConcurrentAccess:
+			// We are getting ConcurrentAccess errors from DeleteExceptDisks - even though we don't set ChangeVersion in that path
+			// We are ignoring the error because in reality the operation finishes successfully.
+			op.Warnf("DeleteExceptDisks failed with ConcurrentAccess error for %s. Ignoring it.", c)
+			concurrent = true
+		default:
+			op.Debugf("Unhandled fault while attempting to destroy vm %s: %#v", c, f.Fault())
+
+			c.updateState(op, existingState)
+			return err
+		}
+	}
+
+	if concurrent && c.vm.IsVC() {
+		if err := c.vm.Unregister(op); err != nil {
+			if !IsNotFoundError(err) {
+				op.Errorf("Error while attempting to unregister container VM %s: %s", c, err)
+				return err
+			}
+		}
+	}
+
+	// remove from datastore
+	fm := ds.NewFileManager(sess.Datacenter, true)
+	if err = fm.Delete(op, url.Path); err != nil {
+		// at this phase error doesn't matter. Just log it.
+		op.Debugf("Failed to delete %s, %s for %s", url, err, c)
+	}
+
+	//remove container from cache
+	Containers.Remove(c.ExecConfig.ID)
+	publishContainerEvent(op, c.ExecConfig.ID, time.Now(), events.ContainerRemoved)
+
+	return nil
+}
+
+// eventedState will determine the target container
+// state based on the current container state and the vsphere event
+func eventedState(op trace.Operation, e events.Event, current State) State {
+	switch e.String() {
+	case events.ContainerPoweredOn:
+		// are we in the process of starting
+		if current != StateStarting {
+			return StateRunning
+		}
+	case events.ContainerPoweredOff:
+		// are we in the process of stopping or just created
+		if current != StateStopping && current != StateCreated {
+			return StateStopped
+		}
+	case events.ContainerSuspended:
+		// are we in the process of suspending
+		if current != StateSuspending {
+			return StateSuspended
+		}
+	case events.ContainerRemoved:
+		if current != StateRemoving {
+			return StateRemoved
+		}
+	}
+
+	return current
+}
+
+func (c *Container) OnEvent(e events.Event) {
+	op := trace.NewOperation(context.Background(), "OnEvent")
+	defer trace.End(trace.Begin(fmt.Sprintf("eventID(%s) received for event: %s", e.EventID(), e.String()), op))
+	c.m.Lock()
+	defer c.m.Unlock()
+
+	if c.vm == nil {
+		op.Warnf("Event(%s) received for %s but no VM found", e.EventID(), e.Reference())
+		return
+	}
+
+	newState := eventedState(op, e, c.state)
+	c.onEvent(op, newState, e)
+}
+
+// determine if the containerVM has started - this could pick up stale data in the started field for an out-of-band
+// power change such as HA or user intervention where we have not had an opportunity to reset the entry.
+func cleanStart(op trace.Operation, c *Container) bool {
+	if len(c.ExecConfig.Sessions) == 0 {
+		op.Warnf("Container %c has no sessions stored in in-memory config", c.ExecConfig.ID)
+		// if no sessions, then nothing to wait for
+		return true
+	}
+
+	for _, session := range c.ExecConfig.Sessions {
+		if session.Started != "true" {
+			return false
+		}
+	}
+	return true
+}
+
+// onEvent determines what needs to be done when receiving a state update. It filters duplicate state transitions
+// and publishes container events as needed in addition to performing necessary manipulations.
+// newState - this is the new state determined by eventedState
+// e - the source event used to derive the new State and reason for the transition
+func (c *Container) onEvent(op trace.Operation, newState State, e events.Event) {
+	// does local data report full start
+	started := cleanStart(op, c)
+	// do we need a refresh
+	refresh := e.String() == events.ContainerRelocated
+	// if it's a state event we've already done a refresh to end up here and dont need another
+	_, stateEvent := e.(*stateevents.StateEvent)
+	// the event we're going to publish - may be overridden/transformed by more context aware logic below
+	// the incoming event is from the very coarse vSphere events
+	publishEventType := e.String()
+
+	if !stateEvent {
+		if (newState == StateStarting && !started) || newState == StateStopping {
+			// inherently transient state. Starting with started == true is just accounting that will
+			// happen below and doesn't need a refresh.
+			refresh = true
+		}
+
+		if newState == StateRunning && !started {
+			// if we cannot confirm fully initialized
+			refresh = true
+		}
+	}
+
+	if refresh {
+		op, cancel := trace.WithTimeout(&op, constants.PropertyCollectorTimeout, "vSphere event triggered refresh")
+		defer cancel()
+
+		if err := c.refresh(op); err != nil {
+			op.Errorf("Container(%s) event driven update failed: %s", c, err)
+		}
+	}
+
+	started = cleanStart(op, c)
+	// it doesn't matter how the event was translated, if we're not fully started then we're starting
+	// if we are then we're running. Only exception is that we don't transition from Running->Starting
+	if newState == StateRunning && !started && c.state != StateRunning {
+		newState = StateStarting
+	}
+	if newState == StateStarting && started {
+		newState = StateRunning
+	}
+
+	if newState != c.state {
+		switch newState {
+		case StateRunning:
+			// transform the PoweredOn event into Started
+			publishEventType = events.ContainerStarted
+			fallthrough
+
+		case StateStarting,
+			StateStopping,
+			StateStopped,
+			StateSuspended:
+
+			c.updateState(op, newState)
+			if newState == StateStopped {
+				c.onStop()
+			}
+		case StateRemoved:
+			if c.vm != nil && c.vm.IsFixing() {
+				// is fixing vm, which will be registered back soon, so do not remove from containers cache
+				op.Debugf("Container(%s) %s is being fixed - %s event ignored", c, newState)
+
+				// Received remove event triggered by unregister VM operation - leave
+				// fixing state now. In a loaded environment, the remove event may be
+				// received after vm.fixVM() has returned, at which point the container
+				// should still be in fixing state to avoid removing it from the cache below.
+				c.vm.LeaveFixingState()
+				// since we're leaving the container in cache, just return w/o allowing
+				// a container event to be propogated to subscribers
+				return
+			}
+			op.Debugf("Container(%s) %s via event activity", c, newState)
+			// if we are here the containerVM has been removed from vSphere, so lets remove it
+			// from the portLayer cache
+			Containers.Remove(c.ExecConfig.ID)
+			c.vm = nil
+		default:
+			return
+		}
+
+		op.Debugf("Container (%s) publishing event (state=%s, event=%s) from event %s", c, newState, publishEventType, e.String())
+		// regardless of state update success or failure publish the container event
+		publishContainerEvent(op, c.ExecConfig.ID, e.Created(), publishEventType)
+		return
+	}
+}
+
+// get the containerVMs from infrastructure for this resource pool
+func infraContainers(ctx context.Context, sess *session.Session) ([]*Container, error) {
+	defer trace.End(trace.Begin(""))
+	var rp mo.ResourcePool
+
+	// popluate the vm property of the vch resource pool
+	if err := Config.ResourcePool.Properties(ctx, Config.ResourcePool.Reference(), []string{"vm"}, &rp); err != nil {
+		name := Config.ResourcePool.Name()
+		log.Errorf("List failed to get %s resource pool child vms: %s", name, err)
+		return nil, err
+	}
+	vms, err := populateVMAttributes(ctx, sess, rp.Vm)
+	if err != nil {
+		return nil, err
+	}
+
+	return convertInfraContainers(ctx, sess, vms), nil
+}
+
+func instanceUUID(id string) (string, error) {
+	// generate VM instance uuid, which will be used to query back VM
+	u, err := sys.UUID()
+	if err != nil {
+		return "", err
+	}
+	namespace, err := uuid.Parse(u)
+	if err != nil {
+		return "", errors.Errorf("unable to parse VCH uuid: %s", err)
+	}
+	return uuid.NewSHA1(namespace, []byte(id)).String(), nil
+}
+
+// populate the vm attributes for the specified morefs
+func populateVMAttributes(ctx context.Context, sess *session.Session, refs []types.ManagedObjectReference) ([]mo.VirtualMachine, error) {
+	defer trace.End(trace.Begin(fmt.Sprintf("populating %d refs", len(refs))))
+	var vms []mo.VirtualMachine
+
+	// current attributes we care about
+	attrib := []string{"config", "runtime.powerState", "summary"}
+
+	// populate the vm properties
+	err := sess.Retrieve(ctx, refs, attrib, &vms)
+	return vms, err
+}
+
+// convert the infra containers to a container object
+func convertInfraContainers(ctx context.Context, sess *session.Session, vms []mo.VirtualMachine) []*Container {
+	defer trace.End(trace.Begin(fmt.Sprintf("converting %d containers", len(vms))))
+	var cons []*Container
+
+	for _, v := range vms {
+		vm := vm.NewVirtualMachine(ctx, sess, v.Reference())
+		base := newBase(vm, v.Config, &v.Runtime)
+		c := newContainer(base)
+
+		id := uid.Parse(c.ExecConfig.ID)
+		if id == uid.NilUID {
+			log.Warnf("skipping converting container VM %s: could not parse id", v.Reference())
+			continue
+		}
+
+		if v.Summary.Storage != nil {
+			c.VMUnsharedDisk = v.Summary.Storage.Unshared
+		}
+
+		cons = append(cons, c)
+	}
+
+	return cons
+}

vendor/github.com/vmware/vic/lib/portlayer/exec/container_cache.go

@@ -0,0 +1,138 @@
+// Copyright 2016-2018 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package exec
+
+import (
+	"sync"
+
+	"context"
+
+	"github.com/vmware/vic/pkg/uid"
+	"github.com/vmware/vic/pkg/vsphere/session"
+)
+
+/*
+* ContainerCache will provide an in-memory cache of containerVMs.  It will
+* be refreshed on portlayer start and updated via container lifecycle
+* operations (start, stop, rm) and well as in response to infrastructure
+* events
+ */
+type containerCache struct {
+	m sync.RWMutex
+
+	// cache by container id
+	cache map[string]*Container
+}
+
+var Containers *containerCache
+
+func NewContainerCache() {
+	// cache by the container ID and the vsphere
+	// managed object reference
+	Containers = &containerCache{
+		cache: make(map[string]*Container),
+	}
+}
+
+func (conCache *containerCache) Container(idOrRef string) *Container {
+	conCache.m.RLock()
+	defer conCache.m.RUnlock()
+	// find by id or moref
+	return conCache.cache[idOrRef]
+}
+
+func (conCache *containerCache) Containers(states []State) []*Container {
+	conCache.m.RLock()
+	defer conCache.m.RUnlock()
+	// cache contains 2 items for each container
+	capacity := len(conCache.cache) / 2
+	containers := make([]*Container, 0, capacity)
+
+	for id, con := range conCache.cache {
+		// is the key a proper ID?
+		if !isContainerID(id) {
+			continue
+		}
+
+		// no state filtering
+		if len(states) == 0 {
+			containers = append(containers, con)
+			continue
+		}
+
+		// filter by container state
+		// DO NOT use container.CurrentState as that can
+		// cause cache deadlocks
+		for _, state := range states {
+			if state == con.State() {
+				containers = append(containers, con)
+			}
+		}
+	}
+
+	return containers
+}
+
+// puts a container in the cache and will overwrite an existing container
+func (conCache *containerCache) Put(container *Container) {
+	// only add containers w/backing VMs
+	if container.vm == nil {
+		return
+	}
+
+	conCache.m.Lock()
+	defer conCache.m.Unlock()
+
+	conCache.put(container)
+}
+
+func (conCache *containerCache) put(container *Container) {
+	// add pointer to cache by container ID
+	conCache.cache[container.ExecConfig.ID] = container
+	conCache.cache[container.vm.Reference().String()] = container
+
+}
+
+func (conCache *containerCache) Remove(idOrRef string) {
+	conCache.m.Lock()
+	defer conCache.m.Unlock()
+	// find by id
+	container := conCache.cache[idOrRef]
+	if container != nil {
+		delete(conCache.cache, container.ExecConfig.ID)
+		delete(conCache.cache, container.vm.Reference().String())
+	}
+}
+
+func (conCache *containerCache) sync(ctx context.Context, sess *session.Session) error {
+	conCache.m.Lock()
+	defer conCache.m.Unlock()
+
+	cons, err := infraContainers(ctx, sess)
+	if err != nil {
+		return err
+	}
+
+	conCache.cache = make(map[string]*Container)
+	for _, c := range cons {
+		conCache.put(c)
+	}
+
+	return nil
+}
+
+func isContainerID(id string) bool {
+	return uid.Parse(id) != uid.NilUID
+}

vendor/github.com/vmware/vic/lib/portlayer/exec/container_cache_test.go

@@ -0,0 +1,79 @@
+// Copyright 2016 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package exec
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/vmware/vic/pkg/uid"
+	"github.com/vmware/vic/pkg/vsphere/vm"
+
+	"github.com/vmware/govmomi/object"
+	"github.com/vmware/govmomi/vim25/types"
+)
+
+func TestContainerCache(t *testing.T) {
+	NewContainerCache()
+	containerID := uid.New().String()
+
+	// create a new container
+	container := newTestContainer(containerID)
+
+	// put it in the cache
+	Containers.Put(container)
+	// still shouldn't have a container because there's no vm
+	assert.Equal(t, len(Containers.cache), 0)
+
+	// add a test vm
+	addTestVM(container)
+
+	// put in cache
+	Containers.Put(container)
+	// get all containers -- should have 1
+	assert.Equal(t, len(Containers.Containers(nil)), 1)
+	// Get specific container
+	cachedContainer := Containers.Container(containerID)
+	// did we find it?
+	assert.NotNil(t, cachedContainer)
+	// do we have this one in the cache?
+	assert.Equal(t, cachedContainer.ExecConfig.ID, containerID)
+	// remove the container
+	Containers.Remove(containerID)
+	assert.Equal(t, len(Containers.cache), 0)
+	// remove non-existent container
+	Containers.Remove("blahblah")
+}
+
+func TestIsContainerID(t *testing.T) {
+	validID := uid.New().String()
+	invalidID := "ABC-XZ_@"
+
+	assert.True(t, isContainerID(validID))
+	assert.False(t, isContainerID(invalidID))
+}
+
+// addTestVM will add a pseudo VM to the container
+func addTestVM(container *Container) {
+	mo := types.ManagedObjectReference{Type: "vm", Value: "12"}
+	v := object.NewVirtualMachine(nil, mo)
+	container.vm = vm.NewVirtualMachineFromVM(nil, nil, v)
+}
+
+func newTestContainer(id string) *Container {
+	h := TestHandle(id)
+	return newContainer(&h.containerBase)
+}

vendor/github.com/vmware/vic/lib/portlayer/exec/container_test.go

@@ -0,0 +1,58 @@
+// Copyright 2016 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package exec
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/vmware/vic/pkg/uid"
+)
+
+func TestStateStringer(t *testing.T) {
+
+	c := &Container{
+		ContainerInfo: ContainerInfo{
+			state: StateRunning,
+		},
+	}
+
+	assert.Equal(t, "Running", c.state.String())
+	c.state = StateStopped
+	assert.Equal(t, "Stopped", c.state.String())
+	c.state = StateStopping
+	assert.Equal(t, "Stopping", c.state.String())
+	c.state = StateRemoving
+	assert.Equal(t, "Removing", c.state.String())
+	c.state = StateStarting
+	assert.Equal(t, "Starting", c.state.String())
+	c.state = StateCreated
+	assert.Equal(t, "Created", c.state.String())
+}
+
+func NewContainer(id uid.UID) *Handle {
+	con := &Container{
+		ContainerInfo: ContainerInfo{
+			state: StateCreating,
+		},
+		newStateEvents: make(map[State]chan struct{}),
+	}
+
+	h := newHandle(con)
+	h.ExecConfig.ID = id.String()
+
+	return h
+}

vendor/github.com/vmware/vic/lib/portlayer/exec/exec.go

@@ -0,0 +1,156 @@
+// Copyright 2016-2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package exec
+
+import (
+	"fmt"
+	"sync"
+	"time"
+
+	"context"
+
+	log "github.com/Sirupsen/logrus"
+
+	"github.com/vmware/govmomi/find"
+	"github.com/vmware/govmomi/object"
+
+	"github.com/vmware/vic/lib/portlayer/event"
+	"github.com/vmware/vic/lib/portlayer/event/collector/vsphere"
+	"github.com/vmware/vic/lib/portlayer/event/events"
+	"github.com/vmware/vic/pkg/trace"
+	"github.com/vmware/vic/pkg/vsphere/extraconfig"
+	"github.com/vmware/vic/pkg/vsphere/session"
+)
+
+var (
+	initializer struct {
+		err  error
+		once sync.Once
+	}
+)
+
+func Init(ctx context.Context, sess *session.Session, source extraconfig.DataSource, _ extraconfig.DataSink) error {
+	log.Info("Beginning initialization of portlayer exec component")
+	initializer.once.Do(func() {
+		var err error
+		defer func() {
+			if err != nil {
+				initializer.err = err
+			}
+		}()
+		f := find.NewFinder(sess.Vim25(), false)
+
+		extraconfig.Decode(source, &Config)
+
+		log.Debugf("Decoded VCH config for execution: %#v", Config)
+		ccount := len(Config.ComputeResources)
+		if ccount != 1 {
+			err = fmt.Errorf("expected singular compute resource element, found %d", ccount)
+			log.Error(err)
+			return
+		}
+
+		cr := Config.ComputeResources[0]
+		var r object.Reference
+		r, err = f.ObjectReference(ctx, cr)
+		if err != nil {
+			err = fmt.Errorf("could not get resource pool or virtual app reference from %q: %s", cr.String(), err)
+			log.Error(err)
+			return
+		}
+		switch o := r.(type) {
+		case *object.VirtualApp:
+			Config.VirtualApp = o
+			Config.ResourcePool = o.ResourcePool
+		case *object.ResourcePool:
+			Config.ResourcePool = o
+		default:
+			err = fmt.Errorf("could not get resource pool or virtual app from reference %q: object type is wrong", cr.String())
+			log.Error(err)
+			return
+		}
+
+		// we want to monitor the cluster, so create a vSphere Event Collector
+		// The cluster managed object will either be a proper vSphere Cluster or
+		// a specific host when standalone mode
+		ec := vsphere.NewCollector(sess.Vim25(), sess.Cluster.Reference().String())
+
+		// start the collection of vsphere events
+		err = ec.Start()
+		if err != nil {
+			err = fmt.Errorf("%s failed to start: %s", ec.Name(), err)
+			log.Error(err)
+			return
+		}
+
+		// instantiate the container cache now
+		NewContainerCache()
+
+		// create the event manager &  register the existing collector
+		Config.EventManager = event.NewEventManager(ec)
+
+		// subscribe the exec layer to the event stream for Vm events
+		vmSub := Config.EventManager.Subscribe(events.NewEventType(vsphere.VMEvent{}).Topic(), "exec", func(e events.Event) {
+			if c := Containers.Container(e.Reference()); c != nil {
+				c.OnEvent(e)
+			}
+		})
+
+		// Grab the AboutInfo about our host environment
+		about := sess.Vim25().ServiceContent.About
+
+		vch := GetVCHstats(ctx)
+		Config.VCHMhz = vch.CPULimit
+		Config.VCHMemoryLimit = vch.MemoryLimit
+
+		Config.HostOS = about.OsType
+		Config.HostOSVersion = about.Version
+		Config.HostProductName = about.Name
+		log.Debugf("Host - OS (%s), version (%s), name (%s)", about.OsType, about.Version, about.Name)
+		log.Debugf("VCH limits - %d Mhz, %d MB", Config.VCHMhz, Config.VCHMemoryLimit)
+
+		// sync container cache
+		vmSub.Suspend(true)
+		defer vmSub.Resume()
+		log.Info("Syncing container cache")
+		if err = Containers.sync(ctx, sess); err != nil {
+			log.Errorf("Error encountered during container cache sync during init process: %s", err)
+			return
+		}
+	})
+
+	return initializer.err
+}
+
+// publishContainerEvent will publish a ContainerEvent to the vic event stream
+func publishContainerEvent(op trace.Operation, id string, created time.Time, eventType string) {
+	if Config.EventManager == nil || eventType == "" {
+		return
+	}
+
+	ce := &events.ContainerEvent{
+		BaseEvent: &events.BaseEvent{
+			// containerEvents are a construct of vic, so lets set the
+			// ID equal to the operation that created the event
+			ID:          op.ID(),
+			Ref:         id,
+			CreatedTime: created,
+			Event:       eventType,
+			Detail:      fmt.Sprintf("Container %s %s", id, eventType),
+		},
+	}
+
+	Config.EventManager.Publish(ce)
+}

vendor/github.com/vmware/vic/lib/portlayer/exec/exec_test.go

@@ -0,0 +1,180 @@
+// Copyright 2016-2018 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package exec
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/vmware/govmomi/vim25/types"
+	"github.com/vmware/vic/lib/portlayer/event"
+	"github.com/vmware/vic/lib/portlayer/event/collector/vsphere"
+	"github.com/vmware/vic/lib/portlayer/event/events"
+	"github.com/vmware/vic/pkg/trace"
+)
+
+var containerEvents []events.Event
+
+func TestEventedState(t *testing.T) {
+	id := "123439"
+	container := newTestContainer(id)
+	addTestVM(container)
+	op := trace.NewOperation(context.Background(), "test operations")
+	// poweredOn event
+	event := createVMEvent(container, StateRunning)
+	assert.EqualValues(t, StateStarting, eventedState(op, event, StateStarting))
+	assert.EqualValues(t, StateRunning, eventedState(op, event, StateRunning))
+	assert.EqualValues(t, StateRunning, eventedState(op, event, StateStopped))
+	assert.EqualValues(t, StateRunning, eventedState(op, event, StateSuspended))
+
+	// // powerOff event
+	event = createVMEvent(container, StateStopped)
+	assert.EqualValues(t, StateStopping, eventedState(op, event, StateStopping))
+	assert.EqualValues(t, StateStopped, eventedState(op, event, StateStopped))
+	assert.EqualValues(t, StateStopped, eventedState(op, event, StateRunning))
+
+	// // suspended event
+	event = createVMEvent(container, StateSuspended)
+	assert.EqualValues(t, StateSuspending, eventedState(op, event, StateSuspending))
+	assert.EqualValues(t, StateSuspended, eventedState(op, event, StateSuspended))
+	assert.EqualValues(t, StateSuspended, eventedState(op, event, StateRunning))
+
+	// removed event
+	event = createVMEvent(container, StateRemoved)
+	assert.EqualValues(t, StateRemoved, eventedState(op, event, StateRemoved))
+	assert.EqualValues(t, StateRemoved, eventedState(op, event, StateStopped))
+	assert.EqualValues(t, StateRemoving, eventedState(op, event, StateRemoving))
+}
+
+func TestPublishContainerEvent(t *testing.T) {
+
+	NewContainerCache()
+	containerEvents = make([]events.Event, 0)
+	Config = Configuration{}
+
+	mgr := event.NewEventManager()
+	Config.EventManager = mgr
+	mgr.Subscribe(events.NewEventType(events.ContainerEvent{}).Topic(), "testing", containerCallback)
+
+	op := trace.NewOperation(context.Background(), "test publish event operation")
+	// create new running container and place in cache
+	id := "123439"
+	container := newTestContainer(id)
+	addTestVM(container)
+	container.SetState(op, StateRunning)
+	Containers.Put(container)
+
+	publishContainerEvent(trace.NewOperation(context.Background(), "container"), id, time.Now().UTC(), events.ContainerPoweredOff)
+	time.Sleep(time.Millisecond * 30)
+
+	assert.Equal(t, 1, len(containerEvents))
+	assert.Equal(t, id, containerEvents[0].Reference())
+	assert.Equal(t, events.ContainerPoweredOff, containerEvents[0].String())
+}
+
+func TestVMRemovedEventCallback(t *testing.T) {
+
+	NewContainerCache()
+	containerEvents = make([]events.Event, 0)
+	Config = Configuration{}
+
+	mgr := event.NewEventManager()
+	Config.EventManager = mgr
+
+	// subscribe the exec layer to the event stream for VM events
+	mgr.Subscribe(events.NewEventType(&vsphere.VMEvent{}).Topic(), "testing", func(e events.Event) {
+		if c := Containers.Container(e.Reference()); c != nil {
+			c.OnEvent(e)
+		}
+	})
+
+	op := trace.NewOperation(context.Background(), "test removed event operation")
+	// create new running container and place in cache
+	id := "123439"
+	container := newTestContainer(id)
+	addTestVM(container)
+	container.SetState(op, StateRunning)
+	Containers.Put(container)
+
+	container.vm.EnterFixingState()
+	vmEvent := createVMEvent(container, StateRemoved)
+
+	mgr.Publish(vmEvent)
+	time.Sleep(time.Millisecond * 30)
+	assertMsg := "Container should have left fixing state in VM remove event handler"
+	assert.False(t, container.vm.IsFixing(), assertMsg)
+
+	mgr.Publish(vmEvent)
+	time.Sleep(time.Millisecond * 30)
+	assertMsg = "Container should be removed now that it has left fixing state"
+	assert.True(t, Containers.Container(id) == nil, assertMsg)
+}
+
+func containerCallback(ee events.Event) {
+	containerEvents = append(containerEvents, ee)
+}
+
+func createVMEvent(container *Container, state State) *vsphere.VMEvent {
+	// event to return
+	var vmEvent *vsphere.VMEvent
+	// basic event info
+	vme := types.Event{
+		CreatedTime: time.Now().UTC(),
+		Key:         int32(101),
+		Vm: &types.VmEventArgument{
+			Vm: container.vm.Reference(),
+		},
+	}
+
+	switch state {
+	case StateSuspended:
+		// suspended
+		vmwEve := &types.VmSuspendedEvent{
+			VmEvent: types.VmEvent{
+				Event: vme,
+			},
+		}
+		vmEvent = vsphere.NewVMEvent(vmwEve)
+	case StateStopped:
+		// poweredOff
+		vmwEve := &types.VmPoweredOffEvent{
+			VmEvent: types.VmEvent{
+				Event: vme,
+			},
+		}
+		vmEvent = vsphere.NewVMEvent(vmwEve)
+	case StateRemoved:
+		// removed
+		vmwEve := &types.VmRemovedEvent{
+			VmEvent: types.VmEvent{
+				Event: vme,
+			},
+		}
+		vmEvent = vsphere.NewVMEvent(vmwEve)
+	default:
+		// poweredOn
+		vmwEve := &types.VmPoweredOnEvent{
+			VmEvent: types.VmEvent{
+				Event: vme,
+			},
+		}
+		vmEvent = vsphere.NewVMEvent(vmwEve)
+	}
+
+	return vmEvent
+}

vendor/github.com/vmware/vic/lib/portlayer/exec/handle.go

@@ -0,0 +1,374 @@
+// Copyright 2016-2018 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package exec
+
+import (
+	"crypto/rand"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"sync"
+	"time"
+
+	"context"
+
+	log "github.com/Sirupsen/logrus"
+	"github.com/golang/groupcache/lru"
+
+	"github.com/vmware/govmomi/vim25/types"
+	"github.com/vmware/vic/lib/config/executor"
+	"github.com/vmware/vic/lib/constants"
+	"github.com/vmware/vic/lib/guest"
+	"github.com/vmware/vic/lib/portlayer/util"
+	"github.com/vmware/vic/lib/spec"
+	"github.com/vmware/vic/pkg/trace"
+	"github.com/vmware/vic/pkg/vsphere/extraconfig"
+	"github.com/vmware/vic/pkg/vsphere/extraconfig/vmomi"
+	"github.com/vmware/vic/pkg/vsphere/session"
+)
+
+// Resources describes the resource allocation for the containerVM
+type Resources struct {
+	NumCPUs  int64
+	MemoryMB int64
+}
+
+// ContainerCreateConfig defines the parameters for Create call
+type ContainerCreateConfig struct {
+	Metadata *executor.ExecutorConfig
+
+	Resources Resources
+}
+
+var handles *lru.Cache
+var handlesLock sync.Mutex
+
+const (
+	handleLen = 16
+	lruSize   = 1000
+)
+
+func init() {
+	handles = lru.New(lruSize)
+}
+
+type Handle struct {
+	// copy from container cache
+	containerBase
+
+	// The guest used to generate specific device types
+	Guest guest.Guest
+
+	// desired spec
+	Spec *spec.VirtualMachineConfigSpec
+	// desired changes to extraconfig
+	changes []types.BaseOptionValue
+
+	// desired state
+	targetState State
+
+	// should this change trigger a reload in the target container
+	reload bool
+
+	// allow for passing outside of the process
+	key string
+}
+
+func newHandleKey() string {
+	b := make([]byte, handleLen)
+	if _, err := io.ReadFull(rand.Reader, b); err != nil {
+		panic(err) // This shouldn't happen
+	}
+	return hex.EncodeToString(b)
+}
+
+// Added solely to support testing - need a better way to do this
+func TestHandle(id string) *Handle {
+	defer trace.End(trace.Begin("Handle.Create"))
+
+	h := newHandle(&Container{})
+	h.ExecConfig.ID = id
+
+	return h
+}
+
+// newHandle creates a handle for an existing container
+// con must not be nil
+func newHandle(con *Container) *Handle {
+	h := &Handle{
+		key:           newHandleKey(),
+		targetState:   StateUnknown,
+		containerBase: *newBase(con.vm, con.Config, con.Runtime),
+		// currently every operation has a spec, because even the power operations
+		// make changes to extraconfig for timestamps and session status
+		Spec: &spec.VirtualMachineConfigSpec{
+			VirtualMachineConfigSpec: &types.VirtualMachineConfigSpec{},
+		},
+	}
+
+	handlesLock.Lock()
+	defer handlesLock.Unlock()
+
+	handles.Add(h.key, h)
+
+	return h
+}
+
+func (h *Handle) TargetState() State {
+	return h.targetState
+}
+
+func (h *Handle) SetTargetState(s State) {
+	h.targetState = s
+}
+
+func (h *Handle) Reload() {
+	h.reload = true
+}
+
+// Rename updates the container name in ExecConfig as well as the vSphere display name
+func (h *Handle) Rename(op trace.Operation, newName string) *Handle {
+	defer trace.End(trace.Begin(newName))
+
+	h.ExecConfig.Name = newName
+
+	s := &spec.VirtualMachineConfigSpecConfig{
+		ID:   h.ExecConfig.ID,
+		Name: newName,
+	}
+
+	h.Spec.Spec().Name = util.DisplayName(op, s, Config.ContainerNameConvention)
+
+	return h
+}
+
+// GetHandle finds and returns the handle that is referred by key
+func GetHandle(key string) *Handle {
+	handlesLock.Lock()
+	defer handlesLock.Unlock()
+
+	if h, ok := handles.Get(key); ok {
+		return h.(*Handle)
+	}
+
+	return nil
+}
+
+// HandleFromInterface returns the Handle
+func HandleFromInterface(key interface{}) *Handle {
+	defer trace.End(trace.Begin(""))
+
+	if h, ok := key.(string); ok {
+		return GetHandle(h)
+	}
+
+	log.Errorf("Type assertion failed for %#+v", key)
+	return nil
+}
+
+// ReferenceFromHandle returns the reference of the given handle
+func ReferenceFromHandle(handle interface{}) interface{} {
+	defer trace.End(trace.Begin(""))
+
+	if h, ok := handle.(*Handle); ok {
+		return h.String()
+	}
+
+	log.Errorf("Type assertion failed for %#+v", handle)
+	return nil
+}
+
+func (h *Handle) String() string {
+	return h.key
+}
+
+func (h *Handle) Commit(op trace.Operation, sess *session.Session, waitTime *int32) error {
+
+	cfg := make(map[string]string)
+
+	// Set timestamps based on target state
+	switch h.TargetState() {
+	case StateRunning:
+		for _, sc := range h.ExecConfig.Sessions {
+			sc.StartTime = time.Now().UTC().Unix()
+			sc.Started = ""
+			sc.ExitStatus = 0
+		}
+	case StateStopped:
+		for _, sc := range h.ExecConfig.Sessions {
+			sc.StopTime = time.Now().UTC().Unix()
+			sc.Started = ""
+		}
+	}
+
+	s := h.Spec.Spec()
+	if h.Config != nil {
+		s.ChangeVersion = h.Config.ChangeVersion
+	}
+
+	// if runtime is nil, should be fresh container create
+	var filter int
+	if h.Runtime == nil || h.Runtime.PowerState == types.VirtualMachinePowerStatePoweredOff || h.TargetState() == StateStopped {
+		// any values set with VM powered off are inherently persistent
+		filter = ^extraconfig.NonPersistent
+	} else {
+		filter = extraconfig.NonPersistent | extraconfig.Hidden
+	}
+
+	extraconfig.Encode(extraconfig.ScopeFilterSink(uint(filter), extraconfig.MapSink(cfg)), h.ExecConfig)
+
+	// strip unmodified keys from the update
+	if h.Config != nil {
+		h.changes = append(s.ExtraConfig, vmomi.OptionValueUpdatesFromMap(h.Config.ExtraConfig, cfg)...)
+	} else {
+		h.changes = append(s.ExtraConfig, vmomi.OptionValueFromMap(cfg, true)...)
+	}
+	s.ExtraConfig = h.changes
+
+	if err := Commit(op, sess, h, waitTime); err != nil {
+		return err
+	}
+
+	h.Close()
+	return nil
+}
+
+// refresh is for internal use only - it's sole purpose at this time is to allow the stop path to update ChangeVersion
+// and corresponding state before performing any associated reconfigure
+func (h *Handle) refresh(op trace.Operation) {
+	// update Config and Runtime to reflect current state
+	h.containerBase.refresh(op)
+
+	// reapply extraconfig changes
+	s := h.Spec.Spec()
+
+	s.ExtraConfig = h.changes
+	s.ChangeVersion = h.Config.ChangeVersion
+}
+
+func (h *Handle) Close() {
+	handlesLock.Lock()
+	defer handlesLock.Unlock()
+
+	handles.Remove(h.key)
+}
+
+// Create returns a new handle that can be Committed to create a new container.
+// At this time the config is *not* deep copied so should not be changed once passed
+//
+// TODO: either deep copy the configuration, or provide an alternative means of passing the data that
+// avoids the need for the caller to unpack/repack the parameters
+func Create(ctx context.Context, vmomiSession *session.Session, config *ContainerCreateConfig) (*Handle, error) {
+	op := trace.FromContext(ctx, "Handle.Create")
+	defer trace.End(trace.Begin(config.Metadata.Name, op))
+
+	h := &Handle{
+		key:         newHandleKey(),
+		targetState: StateCreated,
+		containerBase: containerBase{
+			ExecConfig: config.Metadata,
+		},
+	}
+
+	// configure with debug
+	h.ExecConfig.Diagnostics.DebugLevel = Config.DebugLevel
+	h.ExecConfig.Diagnostics.SysLogConfig = Config.SysLogConfig
+
+	// Convert the management hostname to IP
+	ips, err := net.LookupIP(constants.ManagementHostName)
+	if err != nil {
+		log.Errorf("Unable to look up %s during create of %s: %s", constants.ManagementHostName, config.Metadata.ID, err)
+		return nil, err
+	}
+
+	if len(ips) == 0 {
+		log.Errorf("No IP found for %s during create of %s", constants.ManagementHostName, config.Metadata.ID)
+		return nil, fmt.Errorf("No IP found on %s", constants.ManagementHostName)
+	}
+
+	if len(ips) > 1 {
+		log.Errorf("Multiple IPs found for %s during create of %s: %v", constants.ManagementHostName, config.Metadata.ID, ips)
+		return nil, fmt.Errorf("Multiple IPs found on %s: %#v", constants.ManagementHostName, ips)
+	}
+
+	uuid, err := instanceUUID(config.Metadata.ID)
+	if err != nil {
+		detail := fmt.Sprintf("unable to get instance UUID: %s", err)
+		log.Error(detail)
+		return nil, errors.New(detail)
+	}
+
+	specconfig := &spec.VirtualMachineConfigSpecConfig{
+		NumCPUs:  int32(config.Resources.NumCPUs),
+		MemoryMB: config.Resources.MemoryMB,
+
+		ID:       config.Metadata.ID,
+		Name:     config.Metadata.Name,
+		BiosUUID: uuid,
+
+		// TODO: make this toggle for pod or single based on number of images joined
+		BootMediaPath: Config.BootstrapImagePath,
+		VMPathName:    fmt.Sprintf("[%s]", vmomiSession.Datastore.Name()),
+
+		Metadata: config.Metadata,
+	}
+
+	// if not vsan, set the datastore folder name to containerID
+	if !vmomiSession.IsVSAN(op) {
+		specconfig.VMPathName = fmt.Sprintf("[%s] %s/%s.vmx", vmomiSession.Datastore.Name(), specconfig.ID, specconfig.ID)
+	}
+
+	specconfig.VMFullName = util.DisplayName(op, specconfig, Config.ContainerNameConvention)
+
+	// log only core portions
+	s := specconfig
+	log.Debugf("id: %s, name: %s, cpu: %d, mem: %d, parent: %s, os: %s, path: %s", s.ID, s.Name, s.NumCPUs, s.MemoryMB, s.ParentImageID, s.BootMediaPath, s.VMPathName)
+	m := s.Metadata
+	log.Debugf("annotations: %#v, reponame: %s", m.Annotations, m.RepoName)
+	for name, sess := range m.Sessions {
+		log.Debugf("session: %s, path: %s, dir: %s, runblock: %t, tty: %t, restart: %t, stdin: %t, stopsig: %s",
+			name, sess.Cmd.Path, sess.Cmd.Dir, sess.RunBlock, sess.Tty, sess.Restart, sess.OpenStdin, sess.StopSignal)
+	}
+
+	// If the debug level is high, dump everything
+	// we still do the logging above for consistency so searching the logs for common strings works.
+	// TODO: move this into a debug level aware structure renderer
+	if Config.DebugLevel > 2 {
+		log.Debugf("Config: %#v", specconfig)
+		log.Debugf("Executor spec: %#v", *specconfig.Metadata)
+		for _, sess := range m.Sessions {
+			log.Debugf("Session spec: %#v", *sess)
+		}
+	}
+
+	// Create a linux guest
+	linux, err := guest.NewLinuxGuest(op, vmomiSession, specconfig)
+	if err != nil {
+		log.Errorf("Failed during linux specific spec generation during create of %s: %s", config.Metadata.ID, err)
+		return nil, err
+	}
+
+	h.Guest = linux
+	h.Spec = linux.Spec()
+
+	handlesLock.Lock()
+	defer handlesLock.Unlock()
+
+	handles.Add(h.key, h)
+
+	return h, nil
+}

vendor/github.com/vmware/vic/lib/portlayer/exec/vchinfo.go

@@ -0,0 +1,80 @@
+// Copyright 2016 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package exec
+
+import (
+	"context"
+
+	log "github.com/Sirupsen/logrus"
+
+	"github.com/vmware/govmomi/vim25/mo"
+	"github.com/vmware/govmomi/vim25/types"
+)
+
+type VCHStats struct {
+	CPULimit    int64 // resource pool CPU limit
+	CPUUsage    int64 // resource pool CPU usage in MhZ
+	MemoryLimit int64 // resource pool Memory limit
+	MemoryUsage int64 // resource pool Memory Usage
+}
+
+func GetVCHstats(ctx context.Context, moref ...types.ManagedObjectReference) VCHStats {
+	var p mo.ResourcePool
+	var vch VCHStats
+
+	if Config.ResourcePool == nil {
+		log.Errorf("Unable to retrieve VCHstats: Config.ResourcePool is nil")
+		return vch
+	}
+
+	r := Config.ResourcePool.Reference()
+	if len(moref) > 0 {
+		r = moref[0]
+	}
+
+	ps := []string{"config.cpuAllocation", "config.memoryAllocation", "runtime.cpu", "runtime.memory", "parent"}
+
+	if err := Config.ResourcePool.Properties(ctx, r, ps, &p); err != nil {
+		log.Errorf("VCH stats error: %s", err)
+		return vch
+	}
+
+	vch.CPUUsage = p.Runtime.Cpu.OverallUsage
+	vch.MemoryUsage = p.Runtime.Memory.OverallUsage
+
+	if p.Config.CpuAllocation.Limit != nil {
+		vch.CPULimit = *p.Config.CpuAllocation.Limit
+	}
+
+	if p.Config.MemoryAllocation.Limit != nil {
+		vch.MemoryLimit = *p.Config.MemoryAllocation.Limit
+	}
+
+	stats := []int64{vch.CPULimit,
+		vch.MemoryLimit,
+		vch.CPUUsage,
+		vch.MemoryUsage}
+
+	log.Debugf("The VCH stats are: %+v", stats)
+
+	// If any of the stats is -1, we need to get the vch stats from the parent resource pool
+	for _, v := range stats {
+		if v == -1 {
+			return GetVCHstats(ctx, *p.Parent)
+		}
+	}
+
+	return vch
+}