VMware vSphere Integrated Containers provider (#206)

* Add Virtual Kubelet provider for VIC

Initial virtual kubelet provider for VMware VIC.  This provider currently
handles creating and starting of a pod VM via the VIC portlayer and persona
server.  Image store handling via the VIC persona server.  This provider
currently requires the feature/wolfpack branch of VIC.

* Added pod stop and delete.  Also added node capacity.

Added the ability to stop and delete pod VMs via VIC.  Also retrieve
node capacity information from the VCH.

* Cleanup and readme file

Some file clean up and added a Readme.md markdown file for the VIC
provider.

* Cleaned up errors, added function comments, moved operation code

1. Cleaned up error handling.  Set standard for creating errors.
2. Added method prototype comments for all interface functions.
3. Moved PodCreator, PodStarter, PodStopper, and PodDeleter to a new folder.

* Add mocking code and unit tests for podcache, podcreator, and podstarter

Used the unit test framework used in VIC to handle assertions in the provider's
unit test.  Mocking code generated using OSS project mockery, which is compatible
with the testify assertion framework.

* Vendored packages for the VIC provider

Requires feature/wolfpack branch of VIC and a few specific commit sha of
projects used within VIC.

* Implementation of POD Stopper and Deleter unit tests (#4)

* Updated files for initial PR

vendor/github.com/docker/libnetwork/drivers/remote/api/api.go

@@ -0,0 +1,164 @@
+/*
+Package api represents all requests and responses suitable for conversation
+with a remote driver.
+*/
+package api
+
+import (
+	"net"
+
+	"github.com/docker/libnetwork/driverapi"
+)
+
+// Response is the basic response structure used in all responses.
+type Response struct {
+	Err string
+}
+
+// GetError returns the error from the response, if any.
+func (r *Response) GetError() string {
+	return r.Err
+}
+
+// GetCapabilityResponse is the response of GetCapability request
+type GetCapabilityResponse struct {
+	Response
+	Scope string
+}
+
+// CreateNetworkRequest requests a new network.
+type CreateNetworkRequest struct {
+	// A network ID that remote plugins are expected to store for future
+	// reference.
+	NetworkID string
+
+	// A free form map->object interface for communication of options.
+	Options map[string]interface{}
+
+	// IPAMData contains the address pool information for this network
+	IPv4Data, IPv6Data []driverapi.IPAMData
+}
+
+// CreateNetworkResponse is the response to the CreateNetworkRequest.
+type CreateNetworkResponse struct {
+	Response
+}
+
+// DeleteNetworkRequest is the request to delete an existing network.
+type DeleteNetworkRequest struct {
+	// The ID of the network to delete.
+	NetworkID string
+}
+
+// DeleteNetworkResponse is the response to a request for deleting a network.
+type DeleteNetworkResponse struct {
+	Response
+}
+
+// CreateEndpointRequest is the request to create an endpoint within a network.
+type CreateEndpointRequest struct {
+	// Provided at create time, this will be the network id referenced.
+	NetworkID string
+	// The ID of the endpoint for later reference.
+	EndpointID string
+	Interface  *EndpointInterface
+	Options    map[string]interface{}
+}
+
+// EndpointInterface represents an interface endpoint.
+type EndpointInterface struct {
+	Address     string
+	AddressIPv6 string
+	MacAddress  string
+}
+
+// CreateEndpointResponse is the response to the CreateEndpoint action.
+type CreateEndpointResponse struct {
+	Response
+	Interface *EndpointInterface
+}
+
+// Interface is the representation of a linux interface.
+type Interface struct {
+	Address     *net.IPNet
+	AddressIPv6 *net.IPNet
+	MacAddress  net.HardwareAddr
+}
+
+// DeleteEndpointRequest describes the API for deleting an endpoint.
+type DeleteEndpointRequest struct {
+	NetworkID  string
+	EndpointID string
+}
+
+// DeleteEndpointResponse is the response to the DeleteEndpoint action.
+type DeleteEndpointResponse struct {
+	Response
+}
+
+// EndpointInfoRequest retrieves information about the endpoint from the network driver.
+type EndpointInfoRequest struct {
+	NetworkID  string
+	EndpointID string
+}
+
+// EndpointInfoResponse is the response to an EndpointInfoRequest.
+type EndpointInfoResponse struct {
+	Response
+	Value map[string]interface{}
+}
+
+// JoinRequest describes the API for joining an endpoint to a sandbox.
+type JoinRequest struct {
+	NetworkID  string
+	EndpointID string
+	SandboxKey string
+	Options    map[string]interface{}
+}
+
+// InterfaceName is the struct represetation of a pair of devices with source
+// and destination, for the purposes of putting an endpoint into a container.
+type InterfaceName struct {
+	SrcName   string
+	DstName   string
+	DstPrefix string
+}
+
+// StaticRoute is the plain JSON representation of a static route.
+type StaticRoute struct {
+	Destination string
+	RouteType   int
+	NextHop     string
+}
+
+// JoinResponse is the response to a JoinRequest.
+type JoinResponse struct {
+	Response
+	InterfaceName         *InterfaceName
+	Gateway               string
+	GatewayIPv6           string
+	StaticRoutes          []StaticRoute
+	DisableGatewayService bool
+}
+
+// LeaveRequest describes the API for detaching an endpoint from a sandbox.
+type LeaveRequest struct {
+	NetworkID  string
+	EndpointID string
+}
+
+// LeaveResponse is the answer to LeaveRequest.
+type LeaveResponse struct {
+	Response
+}
+
+// DiscoveryNotification represents a discovery notification
+type DiscoveryNotification struct {
+	DiscoveryType driverapi.DiscoveryType
+	DiscoveryData interface{}
+}
+
+// DiscoveryResponse is used by libnetwork to log any plugin error processing the discovery notifications
+type DiscoveryResponse struct {
+	Response
+}

vendor/github.com/docker/libnetwork/drivers/remote/driver.go

@@ -0,0 +1,327 @@
+package remote
+
+import (
+	"fmt"
+	"net"
+
+	log "github.com/Sirupsen/logrus"
+	"github.com/docker/docker/pkg/plugins"
+	"github.com/docker/libnetwork/datastore"
+	"github.com/docker/libnetwork/driverapi"
+	"github.com/docker/libnetwork/drivers/remote/api"
+	"github.com/docker/libnetwork/types"
+)
+
+type driver struct {
+	endpoint    *plugins.Client
+	networkType string
+}
+
+type maybeError interface {
+	GetError() string
+}
+
+func newDriver(name string, client *plugins.Client) driverapi.Driver {
+	return &driver{networkType: name, endpoint: client}
+}
+
+// Init makes sure a remote driver is registered when a network driver
+// plugin is activated.
+func Init(dc driverapi.DriverCallback, config map[string]interface{}) error {
+	plugins.Handle(driverapi.NetworkPluginEndpointType, func(name string, client *plugins.Client) {
+		// negotiate driver capability with client
+		d := newDriver(name, client)
+		c, err := d.(*driver).getCapabilities()
+		if err != nil {
+			log.Errorf("error getting capability for %s due to %v", name, err)
+			return
+		}
+		if err = dc.RegisterDriver(name, d, *c); err != nil {
+			log.Errorf("error registering driver for %s due to %v", name, err)
+		}
+	})
+	return nil
+}
+
+// Get capability from client
+func (d *driver) getCapabilities() (*driverapi.Capability, error) {
+	var capResp api.GetCapabilityResponse
+	if err := d.call("GetCapabilities", nil, &capResp); err != nil {
+		return nil, err
+	}
+
+	c := &driverapi.Capability{}
+	switch capResp.Scope {
+	case "global":
+		c.DataScope = datastore.GlobalScope
+	case "local":
+		c.DataScope = datastore.LocalScope
+	default:
+		return nil, fmt.Errorf("invalid capability: expecting 'local' or 'global', got %s", capResp.Scope)
+	}
+
+	return c, nil
+}
+
+// Config is not implemented for remote drivers, since it is assumed
+// to be supplied to the remote process out-of-band (e.g., as command
+// line arguments).
+func (d *driver) Config(option map[string]interface{}) error {
+	return &driverapi.ErrNotImplemented{}
+}
+
+func (d *driver) call(methodName string, arg interface{}, retVal maybeError) error {
+	method := driverapi.NetworkPluginEndpointType + "." + methodName
+	err := d.endpoint.Call(method, arg, retVal)
+	if err != nil {
+		return err
+	}
+	if e := retVal.GetError(); e != "" {
+		return fmt.Errorf("remote: %s", e)
+	}
+	return nil
+}
+
+func (d *driver) CreateNetwork(id string, options map[string]interface{}, ipV4Data, ipV6Data []driverapi.IPAMData) error {
+	create := &api.CreateNetworkRequest{
+		NetworkID: id,
+		Options:   options,
+		IPv4Data:  ipV4Data,
+		IPv6Data:  ipV6Data,
+	}
+	return d.call("CreateNetwork", create, &api.CreateNetworkResponse{})
+}
+
+func (d *driver) DeleteNetwork(nid string) error {
+	delete := &api.DeleteNetworkRequest{NetworkID: nid}
+	return d.call("DeleteNetwork", delete, &api.DeleteNetworkResponse{})
+}
+
+func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo, epOptions map[string]interface{}) error {
+	if ifInfo == nil {
+		return fmt.Errorf("must not be called with nil InterfaceInfo")
+	}
+
+	reqIface := &api.EndpointInterface{}
+	if ifInfo.Address() != nil {
+		reqIface.Address = ifInfo.Address().String()
+	}
+	if ifInfo.AddressIPv6() != nil {
+		reqIface.AddressIPv6 = ifInfo.AddressIPv6().String()
+	}
+	if ifInfo.MacAddress() != nil {
+		reqIface.MacAddress = ifInfo.MacAddress().String()
+	}
+
+	create := &api.CreateEndpointRequest{
+		NetworkID:  nid,
+		EndpointID: eid,
+		Interface:  reqIface,
+		Options:    epOptions,
+	}
+	var res api.CreateEndpointResponse
+	if err := d.call("CreateEndpoint", create, &res); err != nil {
+		return err
+	}
+
+	inIface, err := parseInterface(res)
+	if err != nil {
+		return err
+	}
+	if inIface == nil {
+		// Remote driver did not set any field
+		return nil
+	}
+
+	if inIface.MacAddress != nil {
+		if err := ifInfo.SetMacAddress(inIface.MacAddress); err != nil {
+			return errorWithRollback(fmt.Sprintf("driver modified interface MAC address: %v", err), d.DeleteEndpoint(nid, eid))
+		}
+	}
+	if inIface.Address != nil {
+		if err := ifInfo.SetIPAddress(inIface.Address); err != nil {
+			return errorWithRollback(fmt.Sprintf("driver modified interface address: %v", err), d.DeleteEndpoint(nid, eid))
+		}
+	}
+	if inIface.AddressIPv6 != nil {
+		if err := ifInfo.SetIPAddress(inIface.AddressIPv6); err != nil {
+			return errorWithRollback(fmt.Sprintf("driver modified interface address: %v", err), d.DeleteEndpoint(nid, eid))
+		}
+	}
+
+	return nil
+}
+
+func errorWithRollback(msg string, err error) error {
+	rollback := "rolled back"
+	if err != nil {
+		rollback = "failed to roll back: " + err.Error()
+	}
+	return fmt.Errorf("%s; %s", msg, rollback)
+}
+
+func (d *driver) DeleteEndpoint(nid, eid string) error {
+	delete := &api.DeleteEndpointRequest{
+		NetworkID:  nid,
+		EndpointID: eid,
+	}
+	return d.call("DeleteEndpoint", delete, &api.DeleteEndpointResponse{})
+}
+
+func (d *driver) EndpointOperInfo(nid, eid string) (map[string]interface{}, error) {
+	info := &api.EndpointInfoRequest{
+		NetworkID:  nid,
+		EndpointID: eid,
+	}
+	var res api.EndpointInfoResponse
+	if err := d.call("EndpointOperInfo", info, &res); err != nil {
+		return nil, err
+	}
+	return res.Value, nil
+}
+
+// Join method is invoked when a Sandbox is attached to an endpoint.
+func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo, options map[string]interface{}) error {
+	join := &api.JoinRequest{
+		NetworkID:  nid,
+		EndpointID: eid,
+		SandboxKey: sboxKey,
+		Options:    options,
+	}
+	var (
+		res api.JoinResponse
+		err error
+	)
+	if err = d.call("Join", join, &res); err != nil {
+		return err
+	}
+
+	ifaceName := res.InterfaceName
+	if iface := jinfo.InterfaceName(); iface != nil && ifaceName != nil {
+		if err := iface.SetNames(ifaceName.SrcName, ifaceName.DstPrefix); err != nil {
+			return errorWithRollback(fmt.Sprintf("failed to set interface name: %s", err), d.Leave(nid, eid))
+		}
+	}
+
+	var addr net.IP
+	if res.Gateway != "" {
+		if addr = net.ParseIP(res.Gateway); addr == nil {
+			return fmt.Errorf(`unable to parse Gateway "%s"`, res.Gateway)
+		}
+		if jinfo.SetGateway(addr) != nil {
+			return errorWithRollback(fmt.Sprintf("failed to set gateway: %v", addr), d.Leave(nid, eid))
+		}
+	}
+	if res.GatewayIPv6 != "" {
+		if addr = net.ParseIP(res.GatewayIPv6); addr == nil {
+			return fmt.Errorf(`unable to parse GatewayIPv6 "%s"`, res.GatewayIPv6)
+		}
+		if jinfo.SetGatewayIPv6(addr) != nil {
+			return errorWithRollback(fmt.Sprintf("failed to set gateway IPv6: %v", addr), d.Leave(nid, eid))
+		}
+	}
+	if len(res.StaticRoutes) > 0 {
+		routes, err := parseStaticRoutes(res)
+		if err != nil {
+			return err
+		}
+		for _, route := range routes {
+			if jinfo.AddStaticRoute(route.Destination, route.RouteType, route.NextHop) != nil {
+				return errorWithRollback(fmt.Sprintf("failed to set static route: %v", route), d.Leave(nid, eid))
+			}
+		}
+	}
+	if res.DisableGatewayService {
+		jinfo.DisableGatewayService()
+	}
+	return nil
+}
+
+// Leave method is invoked when a Sandbox detaches from an endpoint.
+func (d *driver) Leave(nid, eid string) error {
+	leave := &api.LeaveRequest{
+		NetworkID:  nid,
+		EndpointID: eid,
+	}
+	return d.call("Leave", leave, &api.LeaveResponse{})
+}
+
+func (d *driver) Type() string {
+	return d.networkType
+}
+
+// DiscoverNew is a notification for a new discovery event, such as a new node joining a cluster
+func (d *driver) DiscoverNew(dType driverapi.DiscoveryType, data interface{}) error {
+	if dType != driverapi.NodeDiscovery {
+		return fmt.Errorf("Unknown discovery type : %v", dType)
+	}
+	notif := &api.DiscoveryNotification{
+		DiscoveryType: dType,
+		DiscoveryData: data,
+	}
+	return d.call("DiscoverNew", notif, &api.DiscoveryResponse{})
+}
+
+// DiscoverDelete is a notification for a discovery delete event, such as a node leaving a cluster
+func (d *driver) DiscoverDelete(dType driverapi.DiscoveryType, data interface{}) error {
+	if dType != driverapi.NodeDiscovery {
+		return fmt.Errorf("Unknown discovery type : %v", dType)
+	}
+	notif := &api.DiscoveryNotification{
+		DiscoveryType: dType,
+		DiscoveryData: data,
+	}
+	return d.call("DiscoverDelete", notif, &api.DiscoveryResponse{})
+}
+
+func parseStaticRoutes(r api.JoinResponse) ([]*types.StaticRoute, error) {
+	var routes = make([]*types.StaticRoute, len(r.StaticRoutes))
+	for i, inRoute := range r.StaticRoutes {
+		var err error
+		outRoute := &types.StaticRoute{RouteType: inRoute.RouteType}
+
+		if inRoute.Destination != "" {
+			if outRoute.Destination, err = types.ParseCIDR(inRoute.Destination); err != nil {
+				return nil, err
+			}
+		}
+
+		if inRoute.NextHop != "" {
+			outRoute.NextHop = net.ParseIP(inRoute.NextHop)
+			if outRoute.NextHop == nil {
+				return nil, fmt.Errorf("failed to parse nexthop IP %s", inRoute.NextHop)
+			}
+		}
+
+		routes[i] = outRoute
+	}
+	return routes, nil
+}
+
+// parseInterfaces validates all the parameters of an Interface and returns them.
+func parseInterface(r api.CreateEndpointResponse) (*api.Interface, error) {
+	var outIf *api.Interface
+
+	inIf := r.Interface
+	if inIf != nil {
+		var err error
+		outIf = &api.Interface{}
+		if inIf.Address != "" {
+			if outIf.Address, err = types.ParseCIDR(inIf.Address); err != nil {
+				return nil, err
+			}
+		}
+		if inIf.AddressIPv6 != "" {
+			if outIf.AddressIPv6, err = types.ParseCIDR(inIf.AddressIPv6); err != nil {
+				return nil, err
+			}
+		}
+		if inIf.MacAddress != "" {
+			if outIf.MacAddress, err = net.ParseMAC(inIf.MacAddress); err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	return outIf, nil
+}

vendor/github.com/docker/libnetwork/drivers/remote/driver_test.go

@@ -0,0 +1,559 @@
+package remote
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"testing"
+
+	"github.com/docker/docker/pkg/plugins"
+	"github.com/docker/libnetwork/datastore"
+	"github.com/docker/libnetwork/driverapi"
+	_ "github.com/docker/libnetwork/testutils"
+	"github.com/docker/libnetwork/types"
+)
+
+func decodeToMap(r *http.Request) (res map[string]interface{}, err error) {
+	err = json.NewDecoder(r.Body).Decode(&res)
+	return
+}
+
+func handle(t *testing.T, mux *http.ServeMux, method string, h func(map[string]interface{}) interface{}) {
+	mux.HandleFunc(fmt.Sprintf("/%s.%s", driverapi.NetworkPluginEndpointType, method), func(w http.ResponseWriter, r *http.Request) {
+		ask, err := decodeToMap(r)
+		if err != nil && err != io.EOF {
+			t.Fatal(err)
+		}
+		answer := h(ask)
+		err = json.NewEncoder(w).Encode(&answer)
+		if err != nil {
+			t.Fatal(err)
+		}
+	})
+}
+
+func setupPlugin(t *testing.T, name string, mux *http.ServeMux) func() {
+	if err := os.MkdirAll("/etc/docker/plugins", 0755); err != nil {
+		t.Fatal(err)
+	}
+
+	server := httptest.NewServer(mux)
+	if server == nil {
+		t.Fatal("Failed to start a HTTP Server")
+	}
+
+	if err := ioutil.WriteFile(fmt.Sprintf("/etc/docker/plugins/%s.spec", name), []byte(server.URL), 0644); err != nil {
+		t.Fatal(err)
+	}
+
+	mux.HandleFunc("/Plugin.Activate", func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/vnd.docker.plugins.v1+json")
+		fmt.Fprintf(w, `{"Implements": ["%s"]}`, driverapi.NetworkPluginEndpointType)
+	})
+
+	return func() {
+		if err := os.RemoveAll("/etc/docker/plugins"); err != nil {
+			t.Fatal(err)
+		}
+		server.Close()
+	}
+}
+
+type testEndpoint struct {
+	t                     *testing.T
+	src                   string
+	dst                   string
+	address               string
+	addressIPv6           string
+	macAddress            string
+	gateway               string
+	gatewayIPv6           string
+	resolvConfPath        string
+	hostsPath             string
+	nextHop               string
+	destination           string
+	routeType             int
+	disableGatewayService bool
+}
+
+func (test *testEndpoint) Interface() driverapi.InterfaceInfo {
+	return test
+}
+
+func (test *testEndpoint) Address() *net.IPNet {
+	if test.address == "" {
+		return nil
+	}
+	nw, _ := types.ParseCIDR(test.address)
+	return nw
+}
+
+func (test *testEndpoint) AddressIPv6() *net.IPNet {
+	if test.addressIPv6 == "" {
+		return nil
+	}
+	nw, _ := types.ParseCIDR(test.addressIPv6)
+	return nw
+}
+
+func (test *testEndpoint) MacAddress() net.HardwareAddr {
+	if test.macAddress == "" {
+		return nil
+	}
+	mac, _ := net.ParseMAC(test.macAddress)
+	return mac
+}
+
+func (test *testEndpoint) SetMacAddress(mac net.HardwareAddr) error {
+	if test.macAddress != "" {
+		return types.ForbiddenErrorf("endpoint interface MAC address present (%s). Cannot be modified with %s.", test.macAddress, mac)
+	}
+	if mac == nil {
+		return types.BadRequestErrorf("tried to set nil MAC address to endpoint interface")
+	}
+	test.macAddress = mac.String()
+	return nil
+}
+
+func (test *testEndpoint) SetIPAddress(address *net.IPNet) error {
+	if address.IP == nil {
+		return types.BadRequestErrorf("tried to set nil IP address to endpoint interface")
+	}
+	if address.IP.To4() == nil {
+		return setAddress(&test.addressIPv6, address)
+	}
+	return setAddress(&test.address, address)
+}
+
+func setAddress(ifaceAddr *string, address *net.IPNet) error {
+	if *ifaceAddr != "" {
+		return types.ForbiddenErrorf("endpoint interface IP present (%s). Cannot be modified with (%s).", *ifaceAddr, address)
+	}
+	*ifaceAddr = address.String()
+	return nil
+}
+
+func (test *testEndpoint) InterfaceName() driverapi.InterfaceNameInfo {
+	return test
+}
+
+func compareIPs(t *testing.T, kind string, shouldBe string, supplied net.IP) {
+	ip := net.ParseIP(shouldBe)
+	if ip == nil {
+		t.Fatalf(`Invalid IP to test against: "%s"`, shouldBe)
+	}
+	if !ip.Equal(supplied) {
+		t.Fatalf(`%s IPs are not equal: expected "%s", got %v`, kind, shouldBe, supplied)
+	}
+}
+
+func compareIPNets(t *testing.T, kind string, shouldBe string, supplied net.IPNet) {
+	_, net, _ := net.ParseCIDR(shouldBe)
+	if net == nil {
+		t.Fatalf(`Invalid IP network to test against: "%s"`, shouldBe)
+	}
+	if !types.CompareIPNet(net, &supplied) {
+		t.Fatalf(`%s IP networks are not equal: expected "%s", got %v`, kind, shouldBe, supplied)
+	}
+}
+
+func (test *testEndpoint) SetGateway(ipv4 net.IP) error {
+	compareIPs(test.t, "Gateway", test.gateway, ipv4)
+	return nil
+}
+
+func (test *testEndpoint) SetGatewayIPv6(ipv6 net.IP) error {
+	compareIPs(test.t, "GatewayIPv6", test.gatewayIPv6, ipv6)
+	return nil
+}
+
+func (test *testEndpoint) SetNames(src string, dst string) error {
+	if test.src != src {
+		test.t.Fatalf(`Wrong SrcName; expected "%s", got "%s"`, test.src, src)
+	}
+	if test.dst != dst {
+		test.t.Fatalf(`Wrong DstPrefix; expected "%s", got "%s"`, test.dst, dst)
+	}
+	return nil
+}
+
+func (test *testEndpoint) AddStaticRoute(destination *net.IPNet, routeType int, nextHop net.IP) error {
+	compareIPNets(test.t, "Destination", test.destination, *destination)
+	compareIPs(test.t, "NextHop", test.nextHop, nextHop)
+
+	if test.routeType != routeType {
+		test.t.Fatalf(`Wrong RouteType; expected "%d", got "%d"`, test.routeType, routeType)
+	}
+
+	return nil
+}
+
+func (test *testEndpoint) DisableGatewayService() {
+	test.disableGatewayService = true
+}
+
+func TestGetEmptyCapabilities(t *testing.T) {
+	var plugin = "test-net-driver-empty-cap"
+
+	mux := http.NewServeMux()
+	defer setupPlugin(t, plugin, mux)()
+
+	handle(t, mux, "GetCapabilities", func(msg map[string]interface{}) interface{} {
+		return map[string]interface{}{}
+	})
+
+	p, err := plugins.Get(plugin, driverapi.NetworkPluginEndpointType)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	d := newDriver(plugin, p.Client)
+	if d.Type() != plugin {
+		t.Fatal("Driver type does not match that given")
+	}
+
+	_, err = d.(*driver).getCapabilities()
+	if err == nil {
+		t.Fatal("There should be error reported when get empty capability")
+	}
+}
+
+func TestGetExtraCapabilities(t *testing.T) {
+	var plugin = "test-net-driver-extra-cap"
+
+	mux := http.NewServeMux()
+	defer setupPlugin(t, plugin, mux)()
+
+	handle(t, mux, "GetCapabilities", func(msg map[string]interface{}) interface{} {
+		return map[string]interface{}{
+			"Scope": "local",
+			"foo":   "bar",
+		}
+	})
+
+	p, err := plugins.Get(plugin, driverapi.NetworkPluginEndpointType)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	d := newDriver(plugin, p.Client)
+	if d.Type() != plugin {
+		t.Fatal("Driver type does not match that given")
+	}
+
+	c, err := d.(*driver).getCapabilities()
+	if err != nil {
+		t.Fatal(err)
+	} else if c.DataScope != datastore.LocalScope {
+		t.Fatalf("get capability '%s', expecting 'local'", c.DataScope)
+	}
+}
+
+func TestGetInvalidCapabilities(t *testing.T) {
+	var plugin = "test-net-driver-invalid-cap"
+
+	mux := http.NewServeMux()
+	defer setupPlugin(t, plugin, mux)()
+
+	handle(t, mux, "GetCapabilities", func(msg map[string]interface{}) interface{} {
+		return map[string]interface{}{
+			"Scope": "fake",
+		}
+	})
+
+	p, err := plugins.Get(plugin, driverapi.NetworkPluginEndpointType)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	d := newDriver(plugin, p.Client)
+	if d.Type() != plugin {
+		t.Fatal("Driver type does not match that given")
+	}
+
+	_, err = d.(*driver).getCapabilities()
+	if err == nil {
+		t.Fatal("There should be error reported when get invalid capability")
+	}
+}
+
+func TestRemoteDriver(t *testing.T) {
+	var plugin = "test-net-driver"
+
+	ep := &testEndpoint{
+		t:              t,
+		src:            "vethsrc",
+		dst:            "vethdst",
+		address:        "192.168.5.7/16",
+		addressIPv6:    "2001:DB8::5:7/48",
+		macAddress:     "",
+		gateway:        "192.168.0.1",
+		gatewayIPv6:    "2001:DB8::1",
+		hostsPath:      "/here/comes/the/host/path",
+		resolvConfPath: "/there/goes/the/resolv/conf",
+		destination:    "10.0.0.0/8",
+		nextHop:        "10.0.0.1",
+		routeType:      1,
+	}
+
+	mux := http.NewServeMux()
+	defer setupPlugin(t, plugin, mux)()
+
+	var networkID string
+
+	handle(t, mux, "GetCapabilities", func(msg map[string]interface{}) interface{} {
+		return map[string]interface{}{
+			"Scope": "global",
+		}
+	})
+	handle(t, mux, "CreateNetwork", func(msg map[string]interface{}) interface{} {
+		nid := msg["NetworkID"]
+		var ok bool
+		if networkID, ok = nid.(string); !ok {
+			t.Fatal("RPC did not include network ID string")
+		}
+		return map[string]interface{}{}
+	})
+	handle(t, mux, "DeleteNetwork", func(msg map[string]interface{}) interface{} {
+		if nid, ok := msg["NetworkID"]; !ok || nid != networkID {
+			t.Fatal("Network ID missing or does not match that created")
+		}
+		return map[string]interface{}{}
+	})
+	handle(t, mux, "CreateEndpoint", func(msg map[string]interface{}) interface{} {
+		iface := map[string]interface{}{
+			"MacAddress": ep.macAddress,
+		}
+		return map[string]interface{}{
+			"Interface": iface,
+		}
+	})
+	handle(t, mux, "Join", func(msg map[string]interface{}) interface{} {
+		options := msg["Options"].(map[string]interface{})
+		foo, ok := options["foo"].(string)
+		if !ok || foo != "fooValue" {
+			t.Fatalf("Did not receive expected foo string in request options: %+v", msg)
+		}
+		return map[string]interface{}{
+			"Gateway":        ep.gateway,
+			"GatewayIPv6":    ep.gatewayIPv6,
+			"HostsPath":      ep.hostsPath,
+			"ResolvConfPath": ep.resolvConfPath,
+			"InterfaceName": map[string]interface{}{
+				"SrcName":   ep.src,
+				"DstPrefix": ep.dst,
+			},
+			"StaticRoutes": []map[string]interface{}{
+				{
+					"Destination": ep.destination,
+					"RouteType":   ep.routeType,
+					"NextHop":     ep.nextHop,
+				},
+			},
+		}
+	})
+	handle(t, mux, "Leave", func(msg map[string]interface{}) interface{} {
+		return map[string]string{}
+	})
+	handle(t, mux, "DeleteEndpoint", func(msg map[string]interface{}) interface{} {
+		return map[string]interface{}{}
+	})
+	handle(t, mux, "EndpointOperInfo", func(msg map[string]interface{}) interface{} {
+		return map[string]interface{}{
+			"Value": map[string]string{
+				"Arbitrary": "key",
+				"Value":     "pairs?",
+			},
+		}
+	})
+	handle(t, mux, "DiscoverNew", func(msg map[string]interface{}) interface{} {
+		return map[string]string{}
+	})
+	handle(t, mux, "DiscoverDelete", func(msg map[string]interface{}) interface{} {
+		return map[string]interface{}{}
+	})
+
+	p, err := plugins.Get(plugin, driverapi.NetworkPluginEndpointType)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	d := newDriver(plugin, p.Client)
+	if d.Type() != plugin {
+		t.Fatal("Driver type does not match that given")
+	}
+
+	c, err := d.(*driver).getCapabilities()
+	if err != nil {
+		t.Fatal(err)
+	} else if c.DataScope != datastore.GlobalScope {
+		t.Fatalf("get capability '%s', expecting 'global'", c.DataScope)
+	}
+
+	netID := "dummy-network"
+	err = d.CreateNetwork(netID, map[string]interface{}{}, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	endID := "dummy-endpoint"
+	err = d.CreateEndpoint(netID, endID, ep, map[string]interface{}{})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	joinOpts := map[string]interface{}{"foo": "fooValue"}
+	err = d.Join(netID, endID, "sandbox-key", ep, joinOpts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err = d.EndpointOperInfo(netID, endID); err != nil {
+		t.Fatal(err)
+	}
+	if err = d.Leave(netID, endID); err != nil {
+		t.Fatal(err)
+	}
+	if err = d.DeleteEndpoint(netID, endID); err != nil {
+		t.Fatal(err)
+	}
+	if err = d.DeleteNetwork(netID); err != nil {
+		t.Fatal(err)
+	}
+
+	data := driverapi.NodeDiscoveryData{
+		Address: "192.168.1.1",
+	}
+	if err = d.DiscoverNew(driverapi.NodeDiscovery, data); err != nil {
+		t.Fatal(err)
+	}
+	if err = d.DiscoverDelete(driverapi.NodeDiscovery, data); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestDriverError(t *testing.T) {
+	var plugin = "test-net-driver-error"
+
+	mux := http.NewServeMux()
+	defer setupPlugin(t, plugin, mux)()
+
+	handle(t, mux, "CreateEndpoint", func(msg map[string]interface{}) interface{} {
+		return map[string]interface{}{
+			"Err": "this should get raised as an error",
+		}
+	})
+
+	p, err := plugins.Get(plugin, driverapi.NetworkPluginEndpointType)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	driver := newDriver(plugin, p.Client)
+
+	if err := driver.CreateEndpoint("dummy", "dummy", &testEndpoint{t: t}, map[string]interface{}{}); err == nil {
+		t.Fatalf("Expected error from driver")
+	}
+}
+
+func TestMissingValues(t *testing.T) {
+	var plugin = "test-net-driver-missing"
+
+	mux := http.NewServeMux()
+	defer setupPlugin(t, plugin, mux)()
+
+	ep := &testEndpoint{
+		t: t,
+	}
+
+	handle(t, mux, "CreateEndpoint", func(msg map[string]interface{}) interface{} {
+		iface := map[string]interface{}{
+			"Address":     ep.address,
+			"AddressIPv6": ep.addressIPv6,
+			"MacAddress":  ep.macAddress,
+		}
+		return map[string]interface{}{
+			"Interface": iface,
+		}
+	})
+
+	p, err := plugins.Get(plugin, driverapi.NetworkPluginEndpointType)
+	if err != nil {
+		t.Fatal(err)
+	}
+	driver := newDriver(plugin, p.Client)
+
+	if err := driver.CreateEndpoint("dummy", "dummy", ep, map[string]interface{}{}); err != nil {
+		t.Fatal(err)
+	}
+}
+
+type rollbackEndpoint struct {
+}
+
+func (r *rollbackEndpoint) Interface() driverapi.InterfaceInfo {
+	return r
+}
+
+func (r *rollbackEndpoint) MacAddress() net.HardwareAddr {
+	return nil
+}
+
+func (r *rollbackEndpoint) Address() *net.IPNet {
+	return nil
+}
+
+func (r *rollbackEndpoint) AddressIPv6() *net.IPNet {
+	return nil
+}
+
+func (r *rollbackEndpoint) SetMacAddress(mac net.HardwareAddr) error {
+	return fmt.Errorf("invalid mac")
+}
+
+func (r *rollbackEndpoint) SetIPAddress(ip *net.IPNet) error {
+	return fmt.Errorf("invalid ip")
+}
+
+func TestRollback(t *testing.T) {
+	var plugin = "test-net-driver-rollback"
+
+	mux := http.NewServeMux()
+	defer setupPlugin(t, plugin, mux)()
+
+	rolledback := false
+
+	handle(t, mux, "CreateEndpoint", func(msg map[string]interface{}) interface{} {
+		iface := map[string]interface{}{
+			"Address":     "192.168.4.5/16",
+			"AddressIPv6": "",
+			"MacAddress":  "7a:12:34:56:78:90",
+		}
+		return map[string]interface{}{
+			"Interface": interface{}(iface),
+		}
+	})
+	handle(t, mux, "DeleteEndpoint", func(msg map[string]interface{}) interface{} {
+		rolledback = true
+		return map[string]interface{}{}
+	})
+
+	p, err := plugins.Get(plugin, driverapi.NetworkPluginEndpointType)
+	if err != nil {
+		t.Fatal(err)
+	}
+	driver := newDriver(plugin, p.Client)
+
+	ep := &rollbackEndpoint{}
+
+	if err := driver.CreateEndpoint("dummy", "dummy", ep.Interface(), map[string]interface{}{}); err == nil {
+		t.Fatalf("Expected error from driver")
+	}
+	if !rolledback {
+		t.Fatalf("Expected to have had DeleteEndpoint called")
+	}
+}