VMware vSphere Integrated Containers provider (#206)

* Add Virtual Kubelet provider for VIC

Initial virtual kubelet provider for VMware VIC.  This provider currently
handles creating and starting of a pod VM via the VIC portlayer and persona
server.  Image store handling via the VIC persona server.  This provider
currently requires the feature/wolfpack branch of VIC.

* Added pod stop and delete.  Also added node capacity.

Added the ability to stop and delete pod VMs via VIC.  Also retrieve
node capacity information from the VCH.

* Cleanup and readme file

Some file clean up and added a Readme.md markdown file for the VIC
provider.

* Cleaned up errors, added function comments, moved operation code

1. Cleaned up error handling.  Set standard for creating errors.
2. Added method prototype comments for all interface functions.
3. Moved PodCreator, PodStarter, PodStopper, and PodDeleter to a new folder.

* Add mocking code and unit tests for podcache, podcreator, and podstarter

Used the unit test framework used in VIC to handle assertions in the provider's
unit test.  Mocking code generated using OSS project mockery, which is compatible
with the testify assertion framework.

* Vendored packages for the VIC provider

Requires feature/wolfpack branch of VIC and a few specific commit sha of
projects used within VIC.

* Implementation of POD Stopper and Deleter unit tests (#4)

* Updated files for initial PR

vendor/github.com/docker/libnetwork/datastore/cache.go

@@ -0,0 +1,153 @@
+package datastore
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/docker/libkv/store"
+	"github.com/docker/libkv/store/boltdb"
+)
+
+type kvMap map[string]KVObject
+
+type cache struct {
+	sync.Mutex
+	kmm map[string]kvMap
+	ds  *datastore
+}
+
+func newCache(ds *datastore) *cache {
+	return &cache{kmm: make(map[string]kvMap), ds: ds}
+}
+
+func (c *cache) kmap(kvObject KVObject) (kvMap, error) {
+	var err error
+
+	c.Lock()
+	keyPrefix := Key(kvObject.KeyPrefix()...)
+	kmap, ok := c.kmm[keyPrefix]
+	c.Unlock()
+
+	if ok {
+		return kmap, nil
+	}
+
+	kmap = kvMap{}
+
+	// Bail out right away if the kvObject does not implement KVConstructor
+	ctor, ok := kvObject.(KVConstructor)
+	if !ok {
+		return nil, fmt.Errorf("error while populating kmap, object does not implement KVConstructor interface")
+	}
+
+	kvList, err := c.ds.store.List(keyPrefix)
+	if err != nil {
+		// In case of BoltDB it may return ErrBoltBucketNotFound when no writes
+		// have ever happened on the db bucket. So check for both err codes
+		if err == store.ErrKeyNotFound || err == boltdb.ErrBoltBucketNotFound {
+			// If the store doesn't have anything then there is nothing to
+			// populate in the cache. Just bail out.
+			goto out
+		}
+
+		return nil, fmt.Errorf("error while populating kmap: %v", err)
+	}
+
+	for _, kvPair := range kvList {
+		// Ignore empty kvPair values
+		if len(kvPair.Value) == 0 {
+			continue
+		}
+
+		dstO := ctor.New()
+		err = dstO.SetValue(kvPair.Value)
+		if err != nil {
+			return nil, err
+		}
+
+		// Make sure the object has a correct view of the DB index in
+		// case we need to modify it and update the DB.
+		dstO.SetIndex(kvPair.LastIndex)
+
+		kmap[Key(dstO.Key()...)] = dstO
+	}
+
+out:
+	// There may multiple go routines racing to fill the
+	// cache. The one which places the kmap in c.kmm first
+	// wins. The others should just use what the first populated.
+	c.Lock()
+	kmapNew, ok := c.kmm[keyPrefix]
+	if ok {
+		c.Unlock()
+		return kmapNew, nil
+	}
+
+	c.kmm[keyPrefix] = kmap
+	c.Unlock()
+
+	return kmap, nil
+}
+
+func (c *cache) add(kvObject KVObject) error {
+	kmap, err := c.kmap(kvObject)
+	if err != nil {
+		return err
+	}
+
+	c.Lock()
+	kmap[Key(kvObject.Key()...)] = kvObject
+	c.Unlock()
+	return nil
+}
+
+func (c *cache) del(kvObject KVObject) error {
+	kmap, err := c.kmap(kvObject)
+	if err != nil {
+		return err
+	}
+
+	c.Lock()
+	delete(kmap, Key(kvObject.Key()...))
+	c.Unlock()
+	return nil
+}
+
+func (c *cache) get(key string, kvObject KVObject) error {
+	kmap, err := c.kmap(kvObject)
+	if err != nil {
+		return err
+	}
+
+	c.Lock()
+	defer c.Unlock()
+
+	o, ok := kmap[Key(kvObject.Key()...)]
+	if !ok {
+		return ErrKeyNotFound
+	}
+
+	ctor, ok := o.(KVConstructor)
+	if !ok {
+		return fmt.Errorf("kvobject does not implement KVConstructor interface. could not get object")
+	}
+
+	return ctor.CopyTo(kvObject)
+}
+
+func (c *cache) list(kvObject KVObject) ([]KVObject, error) {
+	kmap, err := c.kmap(kvObject)
+	if err != nil {
+		return nil, err
+	}
+
+	c.Lock()
+	defer c.Unlock()
+
+	var kvol []KVObject
+	for _, v := range kmap {
+		kvol = append(kvol, v)
+	}
+
+	return kvol, nil
+}

vendor/github.com/docker/libnetwork/datastore/datastore.go

@@ -0,0 +1,572 @@
+package datastore
+
+import (
+	"fmt"
+	"log"
+	"reflect"
+	"strings"
+	"sync"
+
+	"github.com/docker/libkv"
+	"github.com/docker/libkv/store"
+	"github.com/docker/libkv/store/boltdb"
+	"github.com/docker/libkv/store/consul"
+	"github.com/docker/libkv/store/etcd"
+	"github.com/docker/libkv/store/zookeeper"
+	"github.com/docker/libnetwork/types"
+)
+
+//DataStore exported
+type DataStore interface {
+	// GetObject gets data from datastore and unmarshals to the specified object
+	GetObject(key string, o KVObject) error
+	// PutObject adds a new Record based on an object into the datastore
+	PutObject(kvObject KVObject) error
+	// PutObjectAtomic provides an atomic add and update operation for a Record
+	PutObjectAtomic(kvObject KVObject) error
+	// DeleteObject deletes a record
+	DeleteObject(kvObject KVObject) error
+	// DeleteObjectAtomic performs an atomic delete operation
+	DeleteObjectAtomic(kvObject KVObject) error
+	// DeleteTree deletes a record
+	DeleteTree(kvObject KVObject) error
+	// Watchable returns whether the store is watchable are not
+	Watchable() bool
+	// Watch for changes on a KVObject
+	Watch(kvObject KVObject, stopCh <-chan struct{}) (<-chan KVObject, error)
+	// RestartWatch retriggers stopped Watches
+	RestartWatch()
+	// Active returns if the store is active
+	Active() bool
+	// List returns of a list of KVObjects belonging to the parent
+	// key. The caller must pass a KVObject of the same type as
+	// the objects that need to be listed
+	List(string, KVObject) ([]KVObject, error)
+	// Scope returns the scope of the store
+	Scope() string
+	// KVStore returns access to the KV Store
+	KVStore() store.Store
+	// Close closes the data store
+	Close()
+}
+
+// ErrKeyModified is raised for an atomic update when the update is working on a stale state
+var (
+	ErrKeyModified = store.ErrKeyModified
+	ErrKeyNotFound = store.ErrKeyNotFound
+)
+
+type datastore struct {
+	scope   string
+	store   store.Store
+	cache   *cache
+	watchCh chan struct{}
+	active  bool
+	sync.Mutex
+}
+
+// KVObject is  Key/Value interface used by objects to be part of the DataStore
+type KVObject interface {
+	// Key method lets an object to provide the Key to be used in KV Store
+	Key() []string
+	// KeyPrefix method lets an object to return immediate parent key that can be used for tree walk
+	KeyPrefix() []string
+	// Value method lets an object to marshal its content to be stored in the KV store
+	Value() []byte
+	// SetValue is used by the datastore to set the object's value when loaded from the data store.
+	SetValue([]byte) error
+	// Index method returns the latest DB Index as seen by the object
+	Index() uint64
+	// SetIndex method allows the datastore to store the latest DB Index into the object
+	SetIndex(uint64)
+	// True if the object exists in the datastore, false if it hasn't been stored yet.
+	// When SetIndex() is called, the object has been stored.
+	Exists() bool
+	// DataScope indicates the storage scope of the KV object
+	DataScope() string
+	// Skip provides a way for a KV Object to avoid persisting it in the KV Store
+	Skip() bool
+}
+
+// KVConstructor interface defines methods which can construct a KVObject from another.
+type KVConstructor interface {
+	// New returns a new object which is created based on the
+	// source object
+	New() KVObject
+	// CopyTo deep copies the contents of the implementing object
+	// to the passed destination object
+	CopyTo(KVObject) error
+}
+
+// ScopeCfg represents Datastore configuration.
+type ScopeCfg struct {
+	Client ScopeClientCfg
+}
+
+// ScopeClientCfg represents Datastore Client-only mode configuration
+type ScopeClientCfg struct {
+	Provider string
+	Address  string
+	Config   *store.Config
+}
+
+const (
+	// LocalScope indicates to store the KV object in local datastore such as boltdb
+	LocalScope = "local"
+	// GlobalScope indicates to store the KV object in global datastore such as consul/etcd/zookeeper
+	GlobalScope   = "global"
+	defaultPrefix = "/var/lib/docker/network/files"
+)
+
+const (
+	// NetworkKeyPrefix is the prefix for network key in the kv store
+	NetworkKeyPrefix = "network"
+	// EndpointKeyPrefix is the prefix for endpoint key in the kv store
+	EndpointKeyPrefix = "endpoint"
+)
+
+var (
+	defaultScopes = makeDefaultScopes()
+)
+
+func makeDefaultScopes() map[string]*ScopeCfg {
+	def := make(map[string]*ScopeCfg)
+	def[LocalScope] = &ScopeCfg{
+		Client: ScopeClientCfg{
+			Provider: string(store.BOLTDB),
+			Address:  defaultPrefix + "/local-kv.db",
+			Config: &store.Config{
+				Bucket: "libnetwork",
+			},
+		},
+	}
+
+	return def
+}
+
+var defaultRootChain = []string{"docker", "network", "v1.0"}
+var rootChain = defaultRootChain
+
+func init() {
+	consul.Register()
+	zookeeper.Register()
+	etcd.Register()
+	boltdb.Register()
+}
+
+// DefaultScopes returns a map of default scopes and it's config for clients to use.
+func DefaultScopes(dataDir string) map[string]*ScopeCfg {
+	if dataDir != "" {
+		defaultScopes[LocalScope].Client.Address = dataDir + "/network/files/local-kv.db"
+		return defaultScopes
+	}
+
+	defaultScopes[LocalScope].Client.Address = defaultPrefix + "/local-kv.db"
+	return defaultScopes
+}
+
+// IsValid checks if the scope config has valid configuration.
+func (cfg *ScopeCfg) IsValid() bool {
+	if cfg == nil ||
+		strings.TrimSpace(cfg.Client.Provider) == "" ||
+		strings.TrimSpace(cfg.Client.Address) == "" {
+		return false
+	}
+
+	return true
+}
+
+//Key provides convenient method to create a Key
+func Key(key ...string) string {
+	keychain := append(rootChain, key...)
+	str := strings.Join(keychain, "/")
+	return str + "/"
+}
+
+//ParseKey provides convenient method to unpack the key to complement the Key function
+func ParseKey(key string) ([]string, error) {
+	chain := strings.Split(strings.Trim(key, "/"), "/")
+
+	// The key must atleast be equal to the rootChain in order to be considered as valid
+	if len(chain) <= len(rootChain) || !reflect.DeepEqual(chain[0:len(rootChain)], rootChain) {
+		return nil, types.BadRequestErrorf("invalid Key : %s", key)
+	}
+	return chain[len(rootChain):], nil
+}
+
+// newClient used to connect to KV Store
+func newClient(scope string, kv string, addr string, config *store.Config, cached bool) (DataStore, error) {
+
+	if cached && scope != LocalScope {
+		return nil, fmt.Errorf("caching supported only for scope %s", LocalScope)
+	}
+
+	if config == nil {
+		config = &store.Config{}
+	}
+
+	var addrs []string
+
+	if kv == string(store.BOLTDB) {
+		// Parse file path
+		addrs = strings.Split(addr, ",")
+	} else {
+		// Parse URI
+		parts := strings.SplitN(addr, "/", 2)
+		addrs = strings.Split(parts[0], ",")
+
+		// Add the custom prefix to the root chain
+		if len(parts) == 2 {
+			rootChain = append([]string{parts[1]}, defaultRootChain...)
+		}
+	}
+
+	store, err := libkv.NewStore(store.Backend(kv), addrs, config)
+	if err != nil {
+		return nil, err
+	}
+
+	ds := &datastore{scope: scope, store: store, active: true, watchCh: make(chan struct{})}
+	if cached {
+		ds.cache = newCache(ds)
+	}
+
+	return ds, nil
+}
+
+// NewDataStore creates a new instance of LibKV data store
+func NewDataStore(scope string, cfg *ScopeCfg) (DataStore, error) {
+	if cfg == nil || cfg.Client.Provider == "" || cfg.Client.Address == "" {
+		c, ok := defaultScopes[scope]
+		if !ok || c.Client.Provider == "" || c.Client.Address == "" {
+			return nil, fmt.Errorf("unexpected scope %s without configuration passed", scope)
+		}
+
+		cfg = c
+	}
+
+	var cached bool
+	if scope == LocalScope {
+		cached = true
+	}
+
+	return newClient(scope, cfg.Client.Provider, cfg.Client.Address, cfg.Client.Config, cached)
+}
+
+func (ds *datastore) Close() {
+	ds.store.Close()
+}
+
+func (ds *datastore) Scope() string {
+	return ds.scope
+}
+
+func (ds *datastore) Active() bool {
+	return ds.active
+}
+
+func (ds *datastore) Watchable() bool {
+	return ds.scope != LocalScope
+}
+
+func (ds *datastore) Watch(kvObject KVObject, stopCh <-chan struct{}) (<-chan KVObject, error) {
+	sCh := make(chan struct{})
+
+	ctor, ok := kvObject.(KVConstructor)
+	if !ok {
+		return nil, fmt.Errorf("error watching object type %T, object does not implement KVConstructor interface", kvObject)
+	}
+
+	kvpCh, err := ds.store.Watch(Key(kvObject.Key()...), sCh)
+	if err != nil {
+		return nil, err
+	}
+
+	kvoCh := make(chan KVObject)
+
+	go func() {
+	retry_watch:
+		var err error
+
+		// Make sure to get a new instance of watch channel
+		ds.Lock()
+		watchCh := ds.watchCh
+		ds.Unlock()
+
+	loop:
+		for {
+			select {
+			case <-stopCh:
+				close(sCh)
+				return
+			case kvPair := <-kvpCh:
+				// If the backend KV store gets reset libkv's go routine
+				// for the watch can exit resulting in a nil value in
+				// channel.
+				if kvPair == nil {
+					ds.Lock()
+					ds.active = false
+					ds.Unlock()
+					break loop
+				}
+
+				dstO := ctor.New()
+
+				if err = dstO.SetValue(kvPair.Value); err != nil {
+					log.Printf("Could not unmarshal kvpair value = %s", string(kvPair.Value))
+					break
+				}
+
+				dstO.SetIndex(kvPair.LastIndex)
+				kvoCh <- dstO
+			}
+		}
+
+		// Wait on watch channel for a re-trigger when datastore becomes active
+		<-watchCh
+
+		kvpCh, err = ds.store.Watch(Key(kvObject.Key()...), sCh)
+		if err != nil {
+			log.Printf("Could not watch the key %s in store: %v", Key(kvObject.Key()...), err)
+		}
+
+		goto retry_watch
+	}()
+
+	return kvoCh, nil
+}
+
+func (ds *datastore) RestartWatch() {
+	ds.Lock()
+	defer ds.Unlock()
+
+	ds.active = true
+	watchCh := ds.watchCh
+	ds.watchCh = make(chan struct{})
+	close(watchCh)
+}
+
+func (ds *datastore) KVStore() store.Store {
+	return ds.store
+}
+
+// PutObjectAtomic adds a new Record based on an object into the datastore
+func (ds *datastore) PutObjectAtomic(kvObject KVObject) error {
+	var (
+		previous *store.KVPair
+		pair     *store.KVPair
+		err      error
+	)
+	ds.Lock()
+	defer ds.Unlock()
+
+	if kvObject == nil {
+		return types.BadRequestErrorf("invalid KV Object : nil")
+	}
+
+	kvObjValue := kvObject.Value()
+
+	if kvObjValue == nil {
+		return types.BadRequestErrorf("invalid KV Object with a nil Value for key %s", Key(kvObject.Key()...))
+	}
+
+	if kvObject.Skip() {
+		goto add_cache
+	}
+
+	if kvObject.Exists() {
+		previous = &store.KVPair{Key: Key(kvObject.Key()...), LastIndex: kvObject.Index()}
+	} else {
+		previous = nil
+	}
+
+	_, pair, err = ds.store.AtomicPut(Key(kvObject.Key()...), kvObjValue, previous, nil)
+	if err != nil {
+		return err
+	}
+
+	kvObject.SetIndex(pair.LastIndex)
+
+add_cache:
+	if ds.cache != nil {
+		return ds.cache.add(kvObject)
+	}
+
+	return nil
+}
+
+// PutObject adds a new Record based on an object into the datastore
+func (ds *datastore) PutObject(kvObject KVObject) error {
+	ds.Lock()
+	defer ds.Unlock()
+
+	if kvObject == nil {
+		return types.BadRequestErrorf("invalid KV Object : nil")
+	}
+
+	if kvObject.Skip() {
+		goto add_cache
+	}
+
+	if err := ds.putObjectWithKey(kvObject, kvObject.Key()...); err != nil {
+		return err
+	}
+
+add_cache:
+	if ds.cache != nil {
+		return ds.cache.add(kvObject)
+	}
+
+	return nil
+}
+
+func (ds *datastore) putObjectWithKey(kvObject KVObject, key ...string) error {
+	kvObjValue := kvObject.Value()
+
+	if kvObjValue == nil {
+		return types.BadRequestErrorf("invalid KV Object with a nil Value for key %s", Key(kvObject.Key()...))
+	}
+	return ds.store.Put(Key(key...), kvObjValue, nil)
+}
+
+// GetObject returns a record matching the key
+func (ds *datastore) GetObject(key string, o KVObject) error {
+	ds.Lock()
+	defer ds.Unlock()
+
+	if ds.cache != nil {
+		return ds.cache.get(key, o)
+	}
+
+	kvPair, err := ds.store.Get(key)
+	if err != nil {
+		return err
+	}
+
+	if err := o.SetValue(kvPair.Value); err != nil {
+		return err
+	}
+
+	// Make sure the object has a correct view of the DB index in
+	// case we need to modify it and update the DB.
+	o.SetIndex(kvPair.LastIndex)
+	return nil
+}
+
+func (ds *datastore) ensureParent(parent string) error {
+	exists, err := ds.store.Exists(parent)
+	if err != nil {
+		return err
+	}
+	if exists {
+		return nil
+	}
+	return ds.store.Put(parent, []byte{}, &store.WriteOptions{IsDir: true})
+}
+
+func (ds *datastore) List(key string, kvObject KVObject) ([]KVObject, error) {
+	ds.Lock()
+	defer ds.Unlock()
+
+	if ds.cache != nil {
+		return ds.cache.list(kvObject)
+	}
+
+	// Bail out right away if the kvObject does not implement KVConstructor
+	ctor, ok := kvObject.(KVConstructor)
+	if !ok {
+		return nil, fmt.Errorf("error listing objects, object does not implement KVConstructor interface")
+	}
+
+	// Make sure the parent key exists
+	if err := ds.ensureParent(key); err != nil {
+		return nil, err
+	}
+
+	kvList, err := ds.store.List(key)
+	if err != nil {
+		return nil, err
+	}
+
+	var kvol []KVObject
+	for _, kvPair := range kvList {
+		if len(kvPair.Value) == 0 {
+			continue
+		}
+
+		dstO := ctor.New()
+		if err := dstO.SetValue(kvPair.Value); err != nil {
+			return nil, err
+		}
+
+		// Make sure the object has a correct view of the DB index in
+		// case we need to modify it and update the DB.
+		dstO.SetIndex(kvPair.LastIndex)
+
+		kvol = append(kvol, dstO)
+	}
+
+	return kvol, nil
+}
+
+// DeleteObject unconditionally deletes a record from the store
+func (ds *datastore) DeleteObject(kvObject KVObject) error {
+	ds.Lock()
+	defer ds.Unlock()
+
+	// cleaup the cache first
+	if ds.cache != nil {
+		ds.cache.del(kvObject)
+	}
+
+	if kvObject.Skip() {
+		return nil
+	}
+
+	return ds.store.Delete(Key(kvObject.Key()...))
+}
+
+// DeleteObjectAtomic performs atomic delete on a record
+func (ds *datastore) DeleteObjectAtomic(kvObject KVObject) error {
+	ds.Lock()
+	defer ds.Unlock()
+
+	if kvObject == nil {
+		return types.BadRequestErrorf("invalid KV Object : nil")
+	}
+
+	previous := &store.KVPair{Key: Key(kvObject.Key()...), LastIndex: kvObject.Index()}
+
+	if kvObject.Skip() {
+		goto del_cache
+	}
+
+	if _, err := ds.store.AtomicDelete(Key(kvObject.Key()...), previous); err != nil {
+		return err
+	}
+
+del_cache:
+	// cleanup the cache only if AtomicDelete went through successfully
+	if ds.cache != nil {
+		return ds.cache.del(kvObject)
+	}
+
+	return nil
+}
+
+// DeleteTree unconditionally deletes a record from the store
+func (ds *datastore) DeleteTree(kvObject KVObject) error {
+	ds.Lock()
+	defer ds.Unlock()
+
+	// cleaup the cache first
+	if ds.cache != nil {
+		ds.cache.del(kvObject)
+	}
+
+	if kvObject.Skip() {
+		return nil
+	}
+
+	return ds.store.DeleteTree(Key(kvObject.KeyPrefix()...))
+}

vendor/github.com/docker/libnetwork/datastore/datastore_test.go

@@ -0,0 +1,254 @@
+package datastore
+
+import (
+	"encoding/json"
+	"reflect"
+	"testing"
+
+	"github.com/docker/libnetwork/options"
+	_ "github.com/docker/libnetwork/testutils"
+	"github.com/stretchr/testify/assert"
+)
+
+var dummyKey = "dummy"
+
+// NewCustomDataStore can be used by other Tests in order to use custom datastore
+func NewTestDataStore() DataStore {
+	return &datastore{scope: LocalScope, store: NewMockStore()}
+}
+
+func TestKey(t *testing.T) {
+	eKey := []string{"hello", "world"}
+	sKey := Key(eKey...)
+	if sKey != "docker/network/v1.0/hello/world/" {
+		t.Fatalf("unexpected key : %s", sKey)
+	}
+}
+
+func TestParseKey(t *testing.T) {
+	keySlice, err := ParseKey("/docker/network/v1.0/hello/world/")
+	if err != nil {
+		t.Fatal(err)
+	}
+	eKey := []string{"hello", "world"}
+	if len(keySlice) < 2 || !reflect.DeepEqual(eKey, keySlice) {
+		t.Fatalf("unexpected unkey : %s", keySlice)
+	}
+}
+
+func TestInvalidDataStore(t *testing.T) {
+	config := &ScopeCfg{}
+	config.Client.Provider = "invalid"
+	config.Client.Address = "localhost:8500"
+	_, err := NewDataStore(GlobalScope, config)
+	if err == nil {
+		t.Fatal("Invalid Datastore connection configuration must result in a failure")
+	}
+}
+
+func TestKVObjectFlatKey(t *testing.T) {
+	store := NewTestDataStore()
+	expected := dummyKVObject("1000", true)
+	err := store.PutObject(expected)
+	if err != nil {
+		t.Fatal(err)
+	}
+	keychain := []string{dummyKey, "1000"}
+	data, err := store.KVStore().Get(Key(keychain...))
+	if err != nil {
+		t.Fatal(err)
+	}
+	var n dummyObject
+	json.Unmarshal(data.Value, &n)
+	if n.Name != expected.Name {
+		t.Fatalf("Dummy object doesn't match the expected object")
+	}
+}
+
+func TestAtomicKVObjectFlatKey(t *testing.T) {
+	store := NewTestDataStore()
+	expected := dummyKVObject("1111", true)
+	assert.False(t, expected.Exists())
+	err := store.PutObjectAtomic(expected)
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.True(t, expected.Exists())
+
+	// PutObjectAtomic automatically sets the Index again. Hence the following must pass.
+
+	err = store.PutObjectAtomic(expected)
+	if err != nil {
+		t.Fatal("Atomic update should succeed.")
+	}
+
+	// Get the latest index and try PutObjectAtomic again for the same Key
+	// This must succeed as well
+	data, err := store.KVStore().Get(Key(expected.Key()...))
+	if err != nil {
+		t.Fatal(err)
+	}
+	n := dummyObject{}
+	json.Unmarshal(data.Value, &n)
+	n.ID = "1111"
+	n.SetIndex(data.LastIndex)
+	n.ReturnValue = true
+	err = store.PutObjectAtomic(&n)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Get the Object using GetObject, then set again.
+	newObj := dummyObject{}
+	err = store.GetObject(Key(expected.Key()...), &newObj)
+	assert.True(t, newObj.Exists())
+	err = store.PutObjectAtomic(&n)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+}
+
+// dummy data used to test the datastore
+type dummyObject struct {
+	Name        string                `kv:"leaf"`
+	NetworkType string                `kv:"leaf"`
+	EnableIPv6  bool                  `kv:"leaf"`
+	Rec         *recStruct            `kv:"recursive"`
+	Dict        map[string]*recStruct `kv:"iterative"`
+	Generic     options.Generic       `kv:"iterative"`
+	ID          string
+	DBIndex     uint64
+	DBExists    bool
+	SkipSave    bool
+	ReturnValue bool
+}
+
+func (n *dummyObject) Key() []string {
+	return []string{dummyKey, n.ID}
+}
+
+func (n *dummyObject) KeyPrefix() []string {
+	return []string{dummyKey}
+}
+
+func (n *dummyObject) Value() []byte {
+	if !n.ReturnValue {
+		return nil
+	}
+
+	b, err := json.Marshal(n)
+	if err != nil {
+		return nil
+	}
+	return b
+}
+
+func (n *dummyObject) SetValue(value []byte) error {
+	return json.Unmarshal(value, n)
+}
+
+func (n *dummyObject) Index() uint64 {
+	return n.DBIndex
+}
+
+func (n *dummyObject) SetIndex(index uint64) {
+	n.DBIndex = index
+	n.DBExists = true
+}
+
+func (n *dummyObject) Exists() bool {
+	return n.DBExists
+}
+
+func (n *dummyObject) Skip() bool {
+	return n.SkipSave
+}
+
+func (n *dummyObject) DataScope() string {
+	return LocalScope
+}
+
+func (n *dummyObject) MarshalJSON() ([]byte, error) {
+	netMap := make(map[string]interface{})
+	netMap["name"] = n.Name
+	netMap["networkType"] = n.NetworkType
+	netMap["enableIPv6"] = n.EnableIPv6
+	netMap["generic"] = n.Generic
+	return json.Marshal(netMap)
+}
+
+func (n *dummyObject) UnmarshalJSON(b []byte) (err error) {
+	var netMap map[string]interface{}
+	if err := json.Unmarshal(b, &netMap); err != nil {
+		return err
+	}
+	n.Name = netMap["name"].(string)
+	n.NetworkType = netMap["networkType"].(string)
+	n.EnableIPv6 = netMap["enableIPv6"].(bool)
+	n.Generic = netMap["generic"].(map[string]interface{})
+	return nil
+}
+
+// dummy structure to test "recursive" cases
+type recStruct struct {
+	Name     string            `kv:"leaf"`
+	Field1   int               `kv:"leaf"`
+	Dict     map[string]string `kv:"iterative"`
+	DBIndex  uint64
+	DBExists bool
+	SkipSave bool
+}
+
+func (r *recStruct) Key() []string {
+	return []string{"recStruct"}
+}
+func (r *recStruct) Value() []byte {
+	b, err := json.Marshal(r)
+	if err != nil {
+		return nil
+	}
+	return b
+}
+
+func (r *recStruct) SetValue(value []byte) error {
+	return json.Unmarshal(value, r)
+}
+
+func (r *recStruct) Index() uint64 {
+	return r.DBIndex
+}
+
+func (r *recStruct) SetIndex(index uint64) {
+	r.DBIndex = index
+	r.DBExists = true
+}
+
+func (r *recStruct) Exists() bool {
+	return r.DBExists
+}
+
+func (r *recStruct) Skip() bool {
+	return r.SkipSave
+}
+
+func dummyKVObject(id string, retValue bool) *dummyObject {
+	cDict := make(map[string]string)
+	cDict["foo"] = "bar"
+	cDict["hello"] = "world"
+	n := dummyObject{
+		Name:        "testNw",
+		NetworkType: "bridge",
+		EnableIPv6:  true,
+		Rec:         &recStruct{"gen", 5, cDict, 0, false, false},
+		ID:          id,
+		DBIndex:     0,
+		ReturnValue: retValue,
+		DBExists:    false,
+		SkipSave:    false}
+	generic := make(map[string]interface{})
+	generic["label1"] = &recStruct{"value1", 1, cDict, 0, false, false}
+	generic["label2"] = "subnet=10.1.1.0/16"
+	n.Generic = generic
+	return &n
+}

vendor/github.com/docker/libnetwork/datastore/mock_store.go

@@ -0,0 +1,129 @@
+package datastore
+
+import (
+	"errors"
+
+	"github.com/docker/libkv/store"
+	"github.com/docker/libnetwork/types"
+)
+
+var (
+	// ErrNotImplmented exported
+	ErrNotImplmented = errors.New("Functionality not implemented")
+)
+
+// MockData exported
+type MockData struct {
+	Data  []byte
+	Index uint64
+}
+
+// MockStore exported
+type MockStore struct {
+	db map[string]*MockData
+}
+
+// NewMockStore creates a Map backed Datastore that is useful for mocking
+func NewMockStore() *MockStore {
+	db := make(map[string]*MockData)
+	return &MockStore{db}
+}
+
+// Get the value at "key", returns the last modified index
+// to use in conjunction to CAS calls
+func (s *MockStore) Get(key string) (*store.KVPair, error) {
+	mData := s.db[key]
+	if mData == nil {
+		return nil, nil
+	}
+	return &store.KVPair{Value: mData.Data, LastIndex: mData.Index}, nil
+
+}
+
+// Put a value at "key"
+func (s *MockStore) Put(key string, value []byte, options *store.WriteOptions) error {
+	mData := s.db[key]
+	if mData == nil {
+		mData = &MockData{value, 0}
+	}
+	mData.Index = mData.Index + 1
+	s.db[key] = mData
+	return nil
+}
+
+// Delete a value at "key"
+func (s *MockStore) Delete(key string) error {
+	delete(s.db, key)
+	return nil
+}
+
+// Exists checks that the key exists inside the store
+func (s *MockStore) Exists(key string) (bool, error) {
+	_, ok := s.db[key]
+	return ok, nil
+}
+
+// List gets a range of values at "directory"
+func (s *MockStore) List(prefix string) ([]*store.KVPair, error) {
+	return nil, ErrNotImplmented
+}
+
+// DeleteTree deletes a range of values at "directory"
+func (s *MockStore) DeleteTree(prefix string) error {
+	delete(s.db, prefix)
+	return nil
+}
+
+// Watch a single key for modifications
+func (s *MockStore) Watch(key string, stopCh <-chan struct{}) (<-chan *store.KVPair, error) {
+	return nil, ErrNotImplmented
+}
+
+// WatchTree triggers a watch on a range of values at "directory"
+func (s *MockStore) WatchTree(prefix string, stopCh <-chan struct{}) (<-chan []*store.KVPair, error) {
+	return nil, ErrNotImplmented
+}
+
+// NewLock exposed
+func (s *MockStore) NewLock(key string, options *store.LockOptions) (store.Locker, error) {
+	return nil, ErrNotImplmented
+}
+
+// AtomicPut put a value at "key" if the key has not been
+// modified in the meantime, throws an error if this is the case
+func (s *MockStore) AtomicPut(key string, newValue []byte, previous *store.KVPair, options *store.WriteOptions) (bool, *store.KVPair, error) {
+	mData := s.db[key]
+
+	if previous == nil {
+		if mData != nil {
+			return false, nil, types.BadRequestErrorf("atomic put failed because key exists")
+		} // Else OK.
+	} else {
+		if mData == nil {
+			return false, nil, types.BadRequestErrorf("atomic put failed because key exists")
+		}
+		if mData != nil && mData.Index != previous.LastIndex {
+			return false, nil, types.BadRequestErrorf("atomic put failed due to mismatched Index")
+		} // Else OK.
+	}
+	err := s.Put(key, newValue, nil)
+	if err != nil {
+		return false, nil, err
+	}
+	return true, &store.KVPair{Key: key, Value: newValue, LastIndex: s.db[key].Index}, nil
+}
+
+// AtomicDelete deletes a value at "key" if the key has not
+// been modified in the meantime, throws an error if this is the case
+func (s *MockStore) AtomicDelete(key string, previous *store.KVPair) (bool, error) {
+	mData := s.db[key]
+	if mData != nil && mData.Index != previous.LastIndex {
+		return false, types.BadRequestErrorf("atomic delete failed due to mismatched Index")
+	}
+	return true, s.Delete(key)
+}
+
+// Close closes the client connection
+func (s *MockStore) Close() {
+	return
+}