VMware vSphere Integrated Containers provider (#206)

* Add Virtual Kubelet provider for VIC

Initial virtual kubelet provider for VMware VIC.  This provider currently
handles creating and starting of a pod VM via the VIC portlayer and persona
server.  Image store handling via the VIC persona server.  This provider
currently requires the feature/wolfpack branch of VIC.

* Added pod stop and delete.  Also added node capacity.

Added the ability to stop and delete pod VMs via VIC.  Also retrieve
node capacity information from the VCH.

* Cleanup and readme file

Some file clean up and added a Readme.md markdown file for the VIC
provider.

* Cleaned up errors, added function comments, moved operation code

1. Cleaned up error handling.  Set standard for creating errors.
2. Added method prototype comments for all interface functions.
3. Moved PodCreator, PodStarter, PodStopper, and PodDeleter to a new folder.

* Add mocking code and unit tests for podcache, podcreator, and podstarter

Used the unit test framework used in VIC to handle assertions in the provider's
unit test.  Mocking code generated using OSS project mockery, which is compatible
with the testify assertion framework.

* Vendored packages for the VIC provider

Requires feature/wolfpack branch of VIC and a few specific commit sha of
projects used within VIC.

* Implementation of POD Stopper and Deleter unit tests (#4)

* Updated files for initial PR

vendor/github.com/vmware/vic/lib/portlayer/event/collector/collector.go

@@ -0,0 +1,40 @@
+// Copyright 2016 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package collector
+
+import (
+	"github.com/vmware/vic/lib/portlayer/event/events"
+)
+
+type Collector interface {
+
+	// AddMonitoredObject will add the object for event listening
+	AddMonitoredObject(ref string) error
+
+	// RemoveMonitoredObject will remove the object from event listening
+	RemoveMonitoredObject(ref string)
+
+	// Start listening for events and publish to function
+	Start() error
+
+	// Stop listening for events
+	Stop()
+
+	// Register a callback function
+	Register(func(events.Event))
+
+	// Name returns the collector name
+	Name() string
+}

vendor/github.com/vmware/vic/lib/portlayer/event/collector/vsphere/collector.go

@@ -0,0 +1,218 @@
+// Copyright 2016-2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package vsphere
+
+import (
+	"context"
+	"fmt"
+	"reflect"
+	"sync"
+
+	"github.com/vmware/vic/lib/portlayer/event/events"
+
+	vmwEvents "github.com/vmware/govmomi/event"
+	"github.com/vmware/govmomi/vim25"
+	"github.com/vmware/govmomi/vim25/types"
+
+	log "github.com/Sirupsen/logrus"
+)
+
+const (
+	name = "vSphere Event Collector"
+)
+
+type EventCollector struct {
+	vmwManager *vmwEvents.Manager
+	mos        monitoredCache
+	callback   func(events.Event)
+
+	lastProcessedID int32
+}
+
+type monitoredCache struct {
+	mu sync.RWMutex
+
+	mos map[string]types.ManagedObjectReference
+}
+
+func NewCollector(client *vim25.Client, objects ...string) *EventCollector {
+	ec := &EventCollector{
+		vmwManager: vmwEvents.NewManager(client),
+		mos:        monitoredCache{mos: make(map[string]types.ManagedObjectReference)},
+		// initialize to an index that will not be present in a page
+		lastProcessedID: -1,
+	}
+
+	for i := range objects {
+		// #nosec: Errors unhandled.
+		ec.AddMonitoredObject(objects[i])
+	}
+
+	return ec
+}
+
+func (ec *EventCollector) Name() string {
+	return name
+}
+
+// Register an event manager callback with the collector
+func (ec *EventCollector) Register(callback func(events.Event)) {
+	ec.callback = callback
+}
+
+func (ec *EventCollector) AddMonitoredObject(ref string) error {
+	ec.mos.mu.Lock()
+	defer ec.mos.mu.Unlock()
+
+	moRef := types.ManagedObjectReference{}
+	if !moRef.FromString(ref) {
+		return fmt.Errorf("%s received an invalid Object to monitor(%s)", name, ref)
+	}
+	ec.mos.mos[ref] = moRef
+	return nil
+}
+
+func (ec *EventCollector) RemoveMonitoredObject(ref string) {
+	ec.mos.mu.Lock()
+	defer ec.mos.mu.Unlock()
+
+	delete(ec.mos.mos, ref)
+}
+
+func (ec *EventCollector) monitoredObjects() []types.ManagedObjectReference {
+	ec.mos.mu.RLock()
+	defer ec.mos.mu.RUnlock()
+
+	refs := make([]types.ManagedObjectReference, 0, len(ec.mos.mos))
+	for k := range ec.mos.mos {
+		refs = append(refs, ec.mos.mos[k])
+	}
+	return refs
+}
+func (ec *EventCollector) Stop() {
+	_, err := ec.vmwManager.Destroy(context.Background())
+	if err != nil {
+		log.Warnf("%s failed to destroy the govmomi manager: %s", name, err.Error())
+	}
+}
+
+// eventTypes is used to filter the event collector so we only receive these event types.
+var eventTypes []string
+
+func init() {
+	events := []types.BaseEvent{
+		(*types.VmGuestShutdownEvent)(nil),
+		(*types.VmPoweredOnEvent)(nil),
+		(*types.DrsVmPoweredOnEvent)(nil),
+		(*types.VmPoweredOffEvent)(nil),
+		(*types.VmRemovedEvent)(nil),
+		(*types.VmSuspendedEvent)(nil),
+		(*types.VmMigratedEvent)(nil),
+		(*types.DrsVmMigratedEvent)(nil),
+		(*types.VmRelocatedEvent)(nil),
+	}
+
+	for _, event := range events {
+		eventTypes = append(eventTypes, reflect.TypeOf(event).Elem().Name())
+	}
+}
+
+// Start the event collector
+func (ec *EventCollector) Start() error {
+	// array of managed objects
+	refs := ec.monitoredObjects()
+
+	// only continue if we have object to monitor
+	if len(refs) == 0 {
+		return fmt.Errorf("%s requires at least one Monitored Object: objects[%d]", name, 0)
+	}
+
+	log.Debugf("%s starting collection for %d managed objects", name, len(refs))
+
+	// we don't want the event listener to timeout
+	ctx := context.Background()
+
+	// pageSize is the number of events on the last page of the eventCollector
+	// as new events are added the oldest are removed.  Originally this value
+	// was 1 and we encountered missed events due to them being evicted
+	// before being processed. We bumped to 25 but we still miss events during
+	// storms such as a host HA event.
+	// Setting pageSize to 1000 overwhelmed hostd via the task history and caused
+	// memory exhaustion. Setting pagesize to 200 while filtering for the specific
+	// types we require showed directly comparable memory overhead vs the 25 page
+	// size setting when running full ci. We may still have significantly higher
+	// memory usage in the scenario where we legitimately have events of interest
+	// at a rate of greater than 25 per page.
+	// This should eventually be replaced with a smaller maximum page size, a page
+	// cursor, and maybe a sliding window for the actual page size.
+	pageSize := int32(200)
+	// bool to follow the stream
+	followStream := true
+	// don't exceed the govmomi object limit
+	force := false
+
+	//TODO: need a proper way to handle failures / status
+	go func(pageSize int32, follow bool, ff bool, refs []types.ManagedObjectReference, ec *EventCollector) error {
+		// the govmomi event listener can only be configured once per session -- so if it's already listening it
+		// will be replaced
+		//
+		// the manager will be closed with the session
+
+		for {
+			err := ec.vmwManager.Events(ctx, refs, pageSize, followStream, force, func(_ types.ManagedObjectReference, page []types.BaseEvent) error {
+				evented(ec, page)
+				return nil
+			}, eventTypes...)
+			// TODO: this will disappear in the ether
+			if err != nil {
+				log.Debugf("Error configuring %s: %s", name, err.Error())
+			}
+		}
+	}(pageSize, followStream, force, refs, ec)
+
+	return nil
+}
+
+// evented will process the event and execute the registered callback
+//
+// Initial implmentation will only act on certain events -- future implementations
+// may provide more flexibility
+func evented(ec *EventCollector, page []types.BaseEvent) {
+	if ec.callback == nil {
+		log.Warn("No callback defined for EventManager")
+		return
+	}
+
+	if len(page) == 0 {
+		return
+	}
+
+	// skip events already seen
+	oldIndex := len(page)
+	for i := range page {
+		if page[i].GetEvent().Key == ec.lastProcessedID {
+			oldIndex = i
+		}
+	}
+
+	// events appear in page with most recent first - need to reverse for sane ordering
+	// we start from the first new event after the last one processed
+	for i := oldIndex - 1; i >= 0; i-- {
+		ec.callback(NewVMEvent(page[i]))
+
+		ec.lastProcessedID = page[i].GetEvent().Key
+	}
+
+}

vendor/github.com/vmware/vic/lib/portlayer/event/collector/vsphere/collector_test.go

@@ -0,0 +1,143 @@
+// Copyright 2016 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package vsphere
+
+import (
+	"strconv"
+	"testing"
+
+	"github.com/vmware/vic/lib/portlayer/event/events"
+
+	"github.com/vmware/govmomi/vim25/types"
+
+	"github.com/stretchr/testify/assert"
+)
+
+const (
+	LifeCycle = iota
+	Reconfigure
+	Mixed
+)
+
+// used to test callbacks
+var callcount int
+
+func newVMMO() *types.ManagedObjectReference {
+	return &types.ManagedObjectReference{Value: "101", Type: "vm"}
+}
+
+func TestMonitoredObject(t *testing.T) {
+
+	mgr := newCollector()
+	mo := newVMMO()
+
+	mgr.AddMonitoredObject(mo.String())
+	mos := mgr.monitoredObjects()
+	assert.Equal(t, 1, len(mos))
+	mgr.RemoveMonitoredObject(mo.String())
+	mos = mgr.monitoredObjects()
+	assert.Equal(t, 0, len(mos))
+}
+
+func TestRegistration(t *testing.T) {
+	mgr := newCollector()
+
+	mgr.Register(callMe)
+	assert.NotNil(t, mgr.callback)
+
+}
+
+func TestEvented(t *testing.T) {
+	mgr := newCollector()
+	callcount = 0
+
+	// register local callback
+	mgr.Register(callMe)
+
+	// test lifecycle events
+	page := eventPage(3, LifeCycle)
+	evented(mgr, page)
+	assert.Equal(t, 3, callcount)
+}
+
+func TestName(t *testing.T) {
+	mgr := newCollector()
+	assert.NotNil(t, mgr.Name())
+	assert.Equal(t, name, mgr.Name())
+}
+
+func TestStart(t *testing.T) {
+	mgr := newCollector()
+	// start should fail as no objects registered
+	assert.Error(t, mgr.Start())
+}
+
+func TestEventTypes(t *testing.T) {
+	if len(eventTypes) != 9 {
+		t.Fatalf("eventTypes=%d", len(eventTypes))
+	}
+
+	f := types.TypeFunc()
+
+	for _, name := range eventTypes {
+		_, ok := f(name)
+		if !ok {
+			t.Errorf("unknown event type: %q", name)
+		}
+	}
+}
+
+func newCollector() *EventCollector {
+	return &EventCollector{mos: monitoredCache{mos: make(map[string]types.ManagedObjectReference)}, lastProcessedID: -1}
+}
+
+// simple callback counter
+func callMe(vm events.Event) {
+	callcount++
+}
+
+// utility function to mock a vsphere event
+//
+// size is the number of events to create
+// lifeCycle is true when we want to generate state events
+// lifeCycle events == poweredOn, poweredOff, etc..
+
+func eventPage(size int, eventType int) []types.BaseEvent {
+	page := make([]types.BaseEvent, 0, size)
+	moid := 100
+	for i := 0; i < size; i++ {
+		var eve types.BaseEvent
+		var eType int
+		moid++
+		vm := types.ManagedObjectReference{Value: strconv.Itoa(moid), Type: "vm"}
+		eType = eventType
+		if eType == Mixed {
+			if i%2 == 0 {
+				eType = LifeCycle
+			} else {
+				eType = Reconfigure
+			}
+		}
+		if eType == LifeCycle {
+			eve = types.BaseEvent(&types.VmPoweredOnEvent{VmEvent: types.VmEvent{Event: types.Event{Vm: &types.VmEventArgument{Vm: vm}}}})
+		} else {
+			eve = types.BaseEvent(&types.VmReconfiguredEvent{VmEvent: types.VmEvent{Event: types.Event{Vm: &types.VmEventArgument{Vm: vm}}}})
+		}
+
+		page = append(page, eve)
+	}
+
+	return page
+}

vendor/github.com/vmware/vic/lib/portlayer/event/collector/vsphere/vm_event.go

@@ -0,0 +1,71 @@
+// Copyright 2016-2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package vsphere
+
+import (
+	"strconv"
+
+	"github.com/vmware/govmomi/vim25/types"
+
+	"github.com/vmware/vic/lib/portlayer/event/events"
+)
+
+type VMEvent struct {
+	*events.BaseEvent
+}
+
+func NewVMEvent(be types.BaseEvent) *VMEvent {
+	var ee string
+	// vm events that we care about
+	switch be.(type) {
+	case *types.VmPoweredOnEvent,
+		*types.DrsVmPoweredOnEvent:
+		ee = events.ContainerPoweredOn
+	case *types.VmPoweredOffEvent:
+		ee = events.ContainerPoweredOff
+	case *types.VmSuspendedEvent:
+		ee = events.ContainerSuspended
+	case *types.VmRemovedEvent:
+		ee = events.ContainerRemoved
+	case *types.VmGuestShutdownEvent:
+		ee = events.ContainerShutdown
+	case *types.VmMigratedEvent:
+		ee = events.ContainerMigrated
+	case *types.DrsVmMigratedEvent:
+		ee = events.ContainerMigratedByDrs
+	case *types.VmRelocatedEvent:
+		ee = events.ContainerRelocated
+	default:
+		panic("Unknown event")
+	}
+	e := be.GetEvent()
+	return &VMEvent{
+		&events.BaseEvent{
+			Event:       ee,
+			ID:          strconv.Itoa(int(e.Key)),
+			Detail:      e.FullFormattedMessage,
+			Ref:         e.Vm.Vm.String(),
+			CreatedTime: e.CreatedTime,
+		},
+	}
+
+}
+
+func (vme *VMEvent) Topic() string {
+	if vme.Type == "" {
+		vme.Type = events.NewEventType(vme)
+	}
+	return vme.Type.Topic()
+}

vendor/github.com/vmware/vic/lib/portlayer/event/collector/vsphere/vm_event_test.go

@@ -0,0 +1,43 @@
+// Copyright 2016 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package vsphere
+
+import (
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/vmware/govmomi/vim25/types"
+	"github.com/vmware/vic/lib/portlayer/event/events"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNewEvent(t *testing.T) {
+	vm := newVMMO()
+	k := 1
+	msg := "jojo the idiot circus boy"
+	tt := time.Now().UTC()
+	vmwEve := &types.VmPoweredOnEvent{VmEvent: types.VmEvent{Event: types.Event{CreatedTime: tt, FullFormattedMessage: msg, Key: int32(k), Vm: &types.VmEventArgument{Vm: *vm}}}}
+	vme := NewVMEvent(vmwEve)
+	assert.NotNil(t, vme)
+	assert.Equal(t, events.ContainerPoweredOn, vme.String())
+	assert.Equal(t, vm.String(), vme.Reference())
+	assert.Equal(t, strconv.Itoa(k), vme.EventID())
+	assert.Equal(t, msg, vme.Message())
+	assert.Equal(t, "vsphere.VMEvent", vme.Topic())
+	assert.Equal(t, tt, vme.Created())
+
+}

vendor/github.com/vmware/vic/lib/portlayer/event/doc.go

@@ -0,0 +1,39 @@
+// Copyright 2016 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/*
+Package event manages events via a simple pub / sub mechanism.  Events could be created
+by vic components or registered Collectors.
+
+Basic Overview
+
+The Event Manager provides basic pub / sub functionality.  A subscription consists of a
+topic (any defined Event), a subscription name (string) and a callback function.  When an event is
+published the event manager will determine the event type and check to see if any components
+have registered a callback for that event type.  For all subscriptions the event manager will
+facilitate the callback.  Publication of events can be accomplished by any component that has
+a pointer to the event manager or via the registered collectors.
+
+Collectors are responsible for collecting events or data from external systems and then publishing
+relevant vic events to the event manager.  In theory the collector could monitor anything and when
+certain criteria are meet publish vic events to the manager.  Collectors are registered with the
+event manager which instructs the collector where to publish.  Multiple collectors are allowed per
+event manager, but each collector has a single publish target.
+
+An example of a collector is the vSphere Event Collector which uses the vSphere EventHistoryCollector
+to monitor the vSphere event stream and publish relevant events to vic.  In the initial implementation
+the vSphere Event Collector is focused on a subset of VM Events that are then transformed to vic Events
+and published to the event manager.
+*/
+package event

vendor/github.com/vmware/vic/lib/portlayer/event/event_manager.go

@@ -0,0 +1,45 @@
+// Copyright 2016 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package event
+
+import (
+	"github.com/vmware/vic/lib/portlayer/event/collector"
+	"github.com/vmware/vic/lib/portlayer/event/events"
+)
+
+// EventManager will provide a basic event pub/sub implementation
+type EventManager interface {
+
+	// RegisterCollector a collector with the manager
+	RegisterCollector(collector.Collector)
+
+	// Collectors returns registered collectors
+	Collectors() map[string]collector.Collector
+
+	// Subscribe for event callbacks
+	Subscribe(eventTopic string, caller string, callback func(events.Event)) Subscriber
+
+	// Unsubscribe from event callbacks
+	Unsubscribe(eventTopic string, caller string)
+
+	// Subscribers will return the subscriber map
+	Subscribers() map[string]map[string]Subscriber
+
+	// Subscribed returns subscriber count
+	Subscribed() int
+
+	// Publish the event to the subscribers
+	Publish(e events.Event)
+}

vendor/github.com/vmware/vic/lib/portlayer/event/events/base_event.go

@@ -0,0 +1,68 @@
+// Copyright 2016 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package events
+
+import (
+	"fmt"
+	"path"
+	"reflect"
+	"time"
+)
+
+type EventType string
+
+type BaseEvent struct {
+	Type        EventType
+	Event       string
+	ID          string
+	Detail      string
+	Ref         string
+	CreatedTime time.Time
+}
+
+func (be *BaseEvent) EventID() string {
+	return be.ID
+}
+
+// return event type / description
+func (be *BaseEvent) String() string {
+	return be.Event
+}
+
+func (be *BaseEvent) Message() string {
+	return be.Detail
+}
+
+func (be *BaseEvent) Reference() string {
+	return be.Ref
+}
+
+func (be *BaseEvent) Created() time.Time {
+	return be.CreatedTime
+}
+
+// NewEventType utility function that uses reflection to return
+// the event type
+func NewEventType(kind interface{}) EventType {
+	t := reflect.TypeOf(kind)
+	if t.Kind() == reflect.Ptr {
+		t = t.Elem()
+	}
+	return EventType(fmt.Sprintf("%s.%s", path.Base(t.PkgPath()), t.Name()))
+}
+
+func (t EventType) Topic() string {
+	return string(t)
+}

vendor/github.com/vmware/vic/lib/portlayer/event/events/base_event_test.go

@@ -0,0 +1,43 @@
+// Copyright 2016 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package events
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNewEventType(t *testing.T) {
+	topic := NewEventType(BaseEvent{})
+	assert.Contains(t, topic.Topic(), "events", ".", "BaseEvent")
+	topic = NewEventType(&BaseEvent{})
+	assert.Contains(t, topic.Topic(), "events", ".", "BaseEvent")
+}
+
+func TestBaseEvent(t *testing.T) {
+	be := &BaseEvent{
+		Event:  "PoweredOn",
+		ID:     "12",
+		Detail: "VM 123 PoweredOn",
+		Ref:    "vm:12",
+	}
+
+	assert.Equal(t, "PoweredOn", be.String())
+	assert.Equal(t, "12", be.EventID())
+	assert.Equal(t, "VM 123 PoweredOn", be.Message())
+	assert.Equal(t, "vm:12", be.Reference())
+
+}

vendor/github.com/vmware/vic/lib/portlayer/event/events/container_event.go

@@ -0,0 +1,43 @@
+// Copyright 2016-2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package events
+
+const (
+	ContainerCreated       = "Created"
+	ContainerFailed        = "Failed"
+	ContainerMigrated      = "Migrated"
+	ContainerMigratedByDrs = "MigratedByDrs"
+	ContainerPoweredOff    = "PoweredOff"
+	ContainerPoweredOn     = "PoweredOn"
+	ContainerReconfigured  = "Reconfigured"
+	ContainerRelocated     = "Relocated"
+	ContainerRemoved       = "Removed"
+	ContainerResumed       = "Resumed"
+	ContainerShutdown      = "Shutdown"
+	ContainerStarted       = "Started"
+	ContainerStopped       = "Stopped"
+	ContainerSuspended     = "Suspended"
+)
+
+type ContainerEvent struct {
+	*BaseEvent
+}
+
+func (ce *ContainerEvent) Topic() string {
+	if ce.Type == "" {
+		ce.Type = NewEventType(ce)
+	}
+	return ce.Type.Topic()
+}

vendor/github.com/vmware/vic/lib/portlayer/event/events/events.go

@@ -0,0 +1,37 @@
+// Copyright 2016 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package events
+
+import (
+	"time"
+)
+
+type Event interface {
+	EventTopic
+	// id of event
+	EventID() string
+	// event (PowerOn, PowerOff, etc)
+	String() string
+	// reference evented object
+	Reference() string
+	// event message
+	Message() string
+
+	Created() time.Time
+}
+
+type EventTopic interface {
+	Topic() string
+}

vendor/github.com/vmware/vic/lib/portlayer/event/events/vsphere/state_event.go

@@ -0,0 +1,61 @@
+// Copyright 2018 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package vsphere
+
+import (
+	"time"
+
+	"github.com/vmware/govmomi/vim25/types"
+
+	"github.com/vmware/vic/lib/portlayer/event/events"
+	"github.com/vmware/vic/pkg/trace"
+)
+
+type StateEvent struct {
+	*events.BaseEvent
+}
+
+func NewStateEvent(op trace.Operation, state types.VirtualMachinePowerState, ref types.ManagedObjectReference) *StateEvent {
+	var ee string
+	// vm power states that we care about
+	switch state {
+	case types.VirtualMachinePowerStatePoweredOn:
+		ee = events.ContainerPoweredOn
+	case types.VirtualMachinePowerStatePoweredOff:
+		ee = events.ContainerPoweredOff
+	case types.VirtualMachinePowerStateSuspended:
+		ee = events.ContainerSuspended
+	default:
+		panic("Unknown event")
+	}
+
+	return &StateEvent{
+		&events.BaseEvent{
+			Event:       ee,
+			ID:          op.ID(),
+			Detail:      "Created from power state " + string(state),
+			Ref:         ref.String(),
+			CreatedTime: time.Now(),
+		},
+	}
+
+}
+
+func (se *StateEvent) Topic() string {
+	if se.Type == "" {
+		se.Type = events.NewEventType(se)
+	}
+	return se.Type.Topic()
+}

vendor/github.com/vmware/vic/lib/portlayer/event/manager.go

@@ -0,0 +1,163 @@
+// Copyright 2016 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package event
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/vmware/vic/lib/portlayer/event/collector"
+	"github.com/vmware/vic/lib/portlayer/event/events"
+	"github.com/vmware/vic/pkg/trace"
+
+	log "github.com/Sirupsen/logrus"
+)
+
+type Manager struct {
+	cos    collectorCache
+	subs   subscriberCache
+	eventQ chan events.Event
+}
+
+const eventQSize = 1000
+
+type collectorCache struct {
+	mu sync.RWMutex
+
+	collectors map[string]collector.Collector
+}
+
+type subscriberCache struct {
+	mu sync.RWMutex
+
+	subscribers map[string]map[string]Subscriber
+}
+
+func NewEventManager(collectors ...collector.Collector) *Manager {
+	mgr := &Manager{
+		cos: collectorCache{
+			collectors: make(map[string]collector.Collector),
+		},
+		subs: subscriberCache{
+			subscribers: make(map[string]map[string]Subscriber),
+		},
+		eventQ: make(chan events.Event, eventQSize),
+	}
+
+	// register any collectors provided
+	for i := range collectors {
+		mgr.RegisterCollector(collectors[i])
+	}
+
+	// event processor routine
+	go func() {
+		for e := range mgr.eventQ {
+			// subscribers for this event
+			mgr.subs.mu.RLock()
+			subs := mgr.subs.subscribers[e.Topic()]
+			mgr.subs.mu.RUnlock()
+
+			log.Debugf("Found %d subscribers to %s: %s - %s", len(subs), e.EventID(), e.Topic(), e.Message())
+
+			for sub, s := range subs {
+				log.Debugf("Event manager calling back to %s for Event(%s): %s", sub, e.EventID(), e.Topic())
+				s.onEvent(e)
+			}
+		}
+	}()
+
+	return mgr
+}
+
+func (mgr *Manager) RegisterCollector(collector collector.Collector) {
+	if collector == nil {
+		return
+	}
+
+	mgr.cos.mu.Lock()
+	defer mgr.cos.mu.Unlock()
+
+	collector.Register(mgr.Publish)
+
+	mgr.cos.collectors[collector.Name()] = collector
+}
+
+func (mgr *Manager) Collectors() map[string]collector.Collector {
+	mgr.cos.mu.RLock()
+	defer mgr.cos.mu.RUnlock()
+
+	c := make(map[string]collector.Collector)
+	for name, collector := range mgr.cos.collectors {
+		c[name] = collector
+	}
+	return c
+}
+
+// Subscribe to the event manager for callback
+func (mgr *Manager) Subscribe(eventTopic string, caller string, callback func(events.Event)) Subscriber {
+	defer trace.End(trace.Begin(fmt.Sprintf("%s:%s", eventTopic, caller)))
+	mgr.subs.mu.Lock()
+	defer mgr.subs.mu.Unlock()
+
+	if _, ok := mgr.subs.subscribers[eventTopic]; !ok {
+		mgr.subs.subscribers[eventTopic] = make(map[string]Subscriber)
+	}
+	s := newSubscriber(eventTopic, caller, callback)
+	mgr.subs.subscribers[eventTopic][caller] = s
+	return s
+}
+
+// Unsubscribe from callbacks
+func (mgr *Manager) Unsubscribe(eventTopic string, caller string) {
+	defer trace.End(trace.Begin(fmt.Sprintf("%s:%s", eventTopic, caller)))
+	mgr.subs.mu.Lock()
+	defer mgr.subs.mu.Unlock()
+	if _, ok := mgr.subs.subscribers[eventTopic]; ok {
+		delete(mgr.subs.subscribers[eventTopic], caller)
+	}
+}
+
+func (mgr *Manager) Subscribers() map[string]map[string]Subscriber {
+	mgr.subs.mu.RLock()
+	defer mgr.subs.mu.RUnlock()
+	s := make(map[string]map[string]Subscriber)
+	for i, m := range mgr.subs.subscribers {
+
+		if _, ok := s[i]; !ok {
+			s[i] = make(map[string]Subscriber)
+		}
+
+		for k, v := range m {
+			s[i][k] = v
+		}
+	}
+	return s
+}
+
+// RegistryCount returns the callback count
+func (mgr *Manager) Subscribed() int {
+	mgr.subs.mu.RLock()
+	defer mgr.subs.mu.RUnlock()
+	count := 0
+	for _, m := range mgr.subs.subscribers {
+		count += len(m)
+	}
+	return count
+}
+
+// Publish events to subscribers
+func (mgr *Manager) Publish(e events.Event) {
+	mgr.eventQ <- e
+}

vendor/github.com/vmware/vic/lib/portlayer/event/manager_test.go

@@ -0,0 +1,98 @@
+// Copyright 2016 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package event
+
+import (
+	"testing"
+
+	"github.com/vmware/vic/lib/portlayer/event/collector/vsphere"
+	"github.com/vmware/vic/lib/portlayer/event/events"
+
+	"github.com/vmware/govmomi/vim25/types"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNewManager(t *testing.T) {
+	mgr := NewEventManager()
+	assert.NotNil(t, mgr)
+}
+
+func TestTopic(t *testing.T) {
+	vmEvent := newVMEvent()
+	assert.Equal(t, vmEvent.Topic(), "vsphere.VMEvent")
+}
+
+func TestSubscribe(t *testing.T) {
+	mgr := NewEventManager()
+	topic := events.NewEventType(vsphere.VMEvent{}).Topic()
+	mgr.Subscribe(topic, "tester", callback)
+	subs := mgr.Subscribers()
+	assert.Equal(t, 1, len(subs))
+	assert.Equal(t, 1, mgr.Subscribed())
+
+	mgr.Subscribe(topic, "tester2", callback)
+	subs = mgr.Subscribers()
+
+	// should still have 1 topic
+	assert.Equal(t, 1, len(subs))
+	// now two subscribers for that topic
+	assert.Equal(t, 2, mgr.Subscribed())
+
+	mgr.Subscribe(events.NewEventType(&vsphere.VMEvent{}).Topic(), "tester3", callback)
+	subs = mgr.Subscribers()
+	// should still have 1 topic
+	assert.Equal(t, 1, len(subs))
+	// now two subscribers for that topic
+	assert.Equal(t, 3, mgr.Subscribed())
+
+	mgr.Unsubscribe(topic, "tester2")
+	subs = mgr.Subscribers()
+	// should still have 1 topic
+	assert.Equal(t, 1, len(subs))
+	// now two subscribers for that topic
+	assert.Equal(t, 2, mgr.Subscribed())
+
+	mgr.Unsubscribe(events.NewEventType(&vsphere.VMEvent{}).Topic(), "tester3")
+	subs = mgr.Subscribers()
+	// should still have 1 topic
+	assert.Equal(t, 1, len(subs))
+	// now one subscribers for that topic
+	assert.Equal(t, 1, mgr.Subscribed())
+
+}
+
+func TestRegisterCollector(t *testing.T) {
+	mgr := NewEventManager()
+	// register nil
+	mgr.RegisterCollector(nil)
+	assert.Equal(t, 0, len(mgr.Collectors()))
+}
+
+// utility methods
+func newVMMO() *types.ManagedObjectReference {
+	return &types.ManagedObjectReference{Value: "101", Type: "vm"}
+}
+
+func newBaseEvent() types.BaseEvent {
+	vm := newVMMO()
+	return types.BaseEvent(&types.VmPoweredOnEvent{VmEvent: types.VmEvent{Event: types.Event{Vm: &types.VmEventArgument{Vm: *vm}}}})
+}
+
+func newVMEvent() *vsphere.VMEvent {
+	return vsphere.NewVMEvent(newBaseEvent())
+}
+
+func callback(e events.Event) {}

vendor/github.com/vmware/vic/lib/portlayer/event/subscriber.go

@@ -0,0 +1,209 @@
+// Copyright 2016-2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package event
+
+import (
+	"sync/atomic"
+
+	log "github.com/Sirupsen/logrus"
+
+	"github.com/vmware/vic/lib/portlayer/event/events"
+)
+
+const (
+	suspendDisabled int32 = iota
+	suspendDiscard
+	suspendQueue
+)
+
+type Subscriber interface {
+	// Topic returns the topic this subscriber is subscribed to
+	Topic() string
+	// Name returns the name of the subscriber
+	Name() string
+
+	// Suspend suspends processing events by the subscriber. If
+	// queueEvents is true, the events are queued until Resume()
+	// is called. If queueEvents is false, events passed into
+	// onEvent() after this call are discarded.
+	Suspend(queueEvents bool)
+	// Resume resumes processing of events by the subscriber.
+	// If Suspend() was called with queueEvents as true, any events
+	// that were passed to onEvent() after Suspend() returned are
+	// processed first.
+	Resume()
+	// IsSuspended returns true if the subscriber is suspended.
+	IsSuspended() bool
+
+	// Discarded returns the number of packets that were discarded by
+	// the subscriber as a result of Pause() being called with
+	// queueEvents as false.
+	Discarded() uint64
+	// Dropped returns the number of packets that were dropped when
+	// the event queue overflows. This only happens when Pause()
+	// is called with queueEvents as true.
+	Dropped() uint64
+
+	// onEvent is called by event.Manager to send an event to
+	// a subscriber
+	onEvent(events.Event)
+}
+
+type subscriber struct {
+	topic              string
+	name               string
+	callback           func(e events.Event)
+	eventQ             chan events.Event
+	suspendState       int32
+	discarded, dropped uint64
+	suspend            chan suspendCmd
+}
+
+type suspendCmd struct {
+	suspend bool
+	done    chan struct{}
+}
+
+const maxEventQueueSize = 1000
+
+// newSubscriber creates a new subscriber to topic
+func newSubscriber(topic, name string, callback func(e events.Event)) Subscriber {
+	s := &subscriber{
+		topic:    topic,
+		name:     name,
+		callback: callback,
+		eventQ:   make(chan events.Event, maxEventQueueSize),
+		suspend:  make(chan suspendCmd),
+	}
+
+	go func() {
+		suspended := false
+		var done chan struct{}
+		for {
+			if done != nil {
+				done <- struct{}{}
+				done = nil
+			}
+
+			if suspended {
+				select {
+				case c := <-s.suspend:
+					suspended = c.suspend
+					done = c.done
+				}
+
+				continue
+			}
+
+			// not suspended
+			select {
+			case e := <-s.eventQ:
+				s.callback(e)
+			case c := <-s.suspend:
+				suspended = c.suspend
+				done = c.done
+			}
+		}
+	}()
+
+	return s
+}
+
+// Topic returns the topic this subscriber is subscribed to
+func (s *subscriber) Topic() string {
+	return s.topic
+}
+
+// Name returns the name of the subscriber
+func (s *subscriber) Name() string {
+	return s.name
+}
+
+// onEvent is called by event.Manager to send an event to
+// a subscriber
+func (s *subscriber) onEvent(e events.Event) {
+	switch atomic.LoadInt32(&s.suspendState) {
+	case suspendDisabled:
+		s.eventQ <- e
+	case suspendDiscard:
+		log.Warnf("discarding event %q", e)
+		atomic.AddUint64(&s.discarded, 1)
+	case suspendQueue:
+		done := false
+		for !done {
+			select {
+			case s.eventQ <- e:
+				done = true
+			default:
+				// make room; discard oldest
+				log.Warnf("dropping event %q", <-s.eventQ)
+				atomic.AddUint64(&s.dropped, 1)
+			}
+		}
+	}
+}
+
+// Suspend suspends processing events by the subscriber. If
+// queueEvents is true, the events are queued until Resume()
+// is called. If queueEvents is false, events passed into
+// onEvent() after this call are discarded.
+func (s *subscriber) Suspend(queueEvents bool) {
+	defer func() {
+		done := make(chan struct{})
+		s.suspend <- suspendCmd{suspend: true, done: done}
+		<-done
+		close(done)
+	}()
+
+	if queueEvents {
+		atomic.StoreInt32(&s.suspendState, suspendQueue)
+		return
+	}
+
+	atomic.StoreInt32(&s.suspendState, suspendDiscard)
+}
+
+// Resume resumes processing of events by the subscriber.
+// If Suspend() was called with queueEvents as true, any events
+// that were passed to onEvent() after Suspend() returned are
+// processed first.
+func (s *subscriber) Resume() {
+	defer func() {
+		done := make(chan struct{})
+		s.suspend <- suspendCmd{suspend: false, done: done}
+		<-done
+		close(done)
+	}()
+	atomic.StoreInt32(&s.suspendState, suspendDisabled)
+}
+
+// IsSuspended returns true if the subscriber is suspended.
+func (s *subscriber) IsSuspended() bool {
+	return atomic.LoadInt32(&s.suspendState) != suspendDisabled
+}
+
+// Discarded returns the number of packets that were discarded by
+// the subscriber as a result of Pause() being called with
+// queueEvents as false.
+func (s *subscriber) Discarded() uint64 {
+	return atomic.LoadUint64(&s.discarded)
+}
+
+// Dropped returns the number of packets that were dropped when
+// the event queue overflows. This only happens when Pause()
+// is called with queueEvents as true.
+func (s *subscriber) Dropped() uint64 {
+	return atomic.LoadUint64(&s.dropped)
+}

vendor/github.com/vmware/vic/lib/portlayer/event/subscriber_test.go

@@ -0,0 +1,218 @@
+// Copyright 2016-2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package event
+
+import (
+	"strconv"
+	"testing"
+	"time"
+
+	log "github.com/Sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
+
+	"github.com/vmware/vic/lib/portlayer/event/events"
+)
+
+func init() {
+	log.SetLevel(log.DebugLevel)
+}
+
+type mockCollector struct {
+	c func(events.Event)
+}
+
+// AddMonitoredObject will add the object for event listening
+func (m *mockCollector) AddMonitoredObject(_ string) error {
+	return nil
+}
+
+// RemoveMonitoredObject will remove the object from event listening
+func (m *mockCollector) RemoveMonitoredObject(_ string) {
+}
+
+// Start listening for events and publish to function
+func (m *mockCollector) Start() error {
+	return nil
+}
+
+// Stop listening for events
+func (m *mockCollector) Stop() {}
+
+// Register a callback function
+func (m *mockCollector) Register(c func(events.Event)) {
+	m.c = c
+}
+
+// Name returns the collector name
+func (m *mockCollector) Name() string {
+	return "mock"
+}
+
+type mockEvent struct {
+	id string
+}
+
+// id of event
+func (e *mockEvent) EventID() string {
+	return e.id
+}
+
+// event (PowerOn, PowerOff, etc)
+func (e *mockEvent) String() string {
+	return e.id
+}
+
+// reference evented object
+func (e *mockEvent) Reference() string {
+	return ""
+}
+
+// event message
+func (e *mockEvent) Message() string {
+	return ""
+}
+
+func (e *mockEvent) Created() time.Time {
+	return time.Now()
+}
+
+func (e *mockEvent) Topic() string {
+	return "test"
+}
+
+func TestSuspendQueue(t *testing.T) {
+	c := &mockCollector{}
+	m := NewEventManager(c)
+	var evs []events.Event
+	done := make(chan struct{})
+	s := m.Subscribe("test", "test", func(e events.Event) {
+		evs = append(evs, e)
+		if len(evs) == 100 {
+			close(done)
+		}
+	})
+
+	suspended := false
+	for i := 0; i < 100; i++ {
+		if !suspended && i >= 50 {
+			s.Suspend(true)
+			assert.True(t, s.IsSuspended())
+			suspended = true
+		}
+		c.c(&mockEvent{id: strconv.Itoa(i)})
+	}
+
+	select {
+	case <-done:
+		assert.Fail(t, "unexpectedly got all events despite suspend")
+	case <-time.After(2 * time.Second):
+		assert.Condition(t, func() bool { return len(evs) <= 50 })
+	}
+
+	s.Resume()
+	assert.False(t, s.IsSuspended())
+
+	select {
+	case <-done:
+	case <-time.After(2 * time.Second):
+		assert.Fail(t, "timed out waiting for suspended events")
+	}
+
+	// check for dups
+	for i := range evs {
+		for j := range evs {
+			if j == i {
+				continue
+			}
+			if evs[j].EventID() == evs[i].EventID() {
+				assert.Fail(t, "dup event found for id %d", evs[j].EventID())
+			}
+		}
+	}
+}
+
+func TestSuspendDiscard(t *testing.T) {
+	c := &mockCollector{}
+	m := NewEventManager(c)
+	var evs []events.Event
+	s := m.Subscribe("test", "test", func(e events.Event) {
+		assert.Fail(t, "got an event %q when expecting none", e)
+	})
+
+	// discard events
+	s.Suspend(false)
+	assert.True(t, s.IsSuspended())
+	for i := 0; i < 50; i++ {
+		c.c(&mockEvent{id: strconv.Itoa(i)})
+	}
+
+	<-time.After(5 * time.Second)
+
+	assert.Empty(t, evs)
+	assert.Equal(t, uint64(50), s.Discarded())
+	assert.Equal(t, uint64(0), s.Dropped())
+}
+
+func TestSuspendOverflow(t *testing.T) {
+	c := &mockCollector{}
+	m := NewEventManager(c)
+	var evs []events.Event
+	done := make(chan struct{})
+	s := m.Subscribe("test", "test", func(e events.Event) {
+		evs = append(evs, e)
+		if len(evs) == maxEventQueueSize {
+			close(done)
+		}
+	})
+
+	s.Suspend(true)
+	assert.True(t, s.IsSuspended())
+	for i := 0; i < maxEventQueueSize+1; i++ {
+		c.c(&mockEvent{id: strconv.Itoa(i)})
+	}
+
+	select {
+	case <-done:
+		assert.Fail(t, "unexpectedly got all events despite suspend")
+	case <-time.After(5 * time.Second):
+	}
+
+	s.Resume()
+	assert.False(t, s.IsSuspended())
+
+	select {
+	case <-done:
+	case <-time.After(5 * time.Second):
+		assert.Fail(t, "timed out waiting for sentinel event")
+	}
+
+	assert.Len(t, evs, maxEventQueueSize)
+	assert.Equal(t, uint64(1), s.Dropped())
+	assert.Equal(t, uint64(0), s.Discarded())
+
+	// check for dups
+	for i := range evs {
+		// should not have an event with id 0
+		assert.NotEqual(t, 0, evs[i].EventID(), "got event with event id 0")
+		for j := range evs {
+			if j == i {
+				continue
+			}
+			if evs[j].EventID() == evs[i].EventID() {
+				assert.Fail(t, "dup event found for id %d", evs[j].EventID())
+			}
+		}
+	}
+}