VMware vSphere Integrated Containers provider (#206)

* Add Virtual Kubelet provider for VIC

Initial virtual kubelet provider for VMware VIC.  This provider currently
handles creating and starting of a pod VM via the VIC portlayer and persona
server.  Image store handling via the VIC persona server.  This provider
currently requires the feature/wolfpack branch of VIC.

* Added pod stop and delete.  Also added node capacity.

Added the ability to stop and delete pod VMs via VIC.  Also retrieve
node capacity information from the VCH.

* Cleanup and readme file

Some file clean up and added a Readme.md markdown file for the VIC
provider.

* Cleaned up errors, added function comments, moved operation code

1. Cleaned up error handling.  Set standard for creating errors.
2. Added method prototype comments for all interface functions.
3. Moved PodCreator, PodStarter, PodStopper, and PodDeleter to a new folder.

* Add mocking code and unit tests for podcache, podcreator, and podstarter

Used the unit test framework used in VIC to handle assertions in the provider's
unit test.  Mocking code generated using OSS project mockery, which is compatible
with the testify assertion framework.

* Vendored packages for the VIC provider

Requires feature/wolfpack branch of VIC and a few specific commit sha of
projects used within VIC.

* Implementation of POD Stopper and Deleter unit tests (#4)

* Updated files for initial PR
This commit is contained in:
Loc Nguyen
2018-06-04 15:41:32 -07:00
committed by Ria Bhatia
parent 98a111e8b7
commit 513cebe7b7
6296 changed files with 1123685 additions and 8 deletions

View File

@@ -0,0 +1,4 @@
package osl
// IfaceOption is a function option type to set interface options
type IfaceOption func()

View File

@@ -0,0 +1,386 @@
package osl
import (
"fmt"
"net"
"os/exec"
"regexp"
"sync"
"syscall"
"github.com/docker/libnetwork/types"
"github.com/vishvananda/netlink"
)
// IfaceOption is a function option type to set interface options
type IfaceOption func(i *nwIface)
type nwIface struct {
srcName string
dstName string
master string
dstMaster string
address *net.IPNet
addressIPv6 *net.IPNet
routes []*net.IPNet
bridge bool
ns *networkNamespace
sync.Mutex
}
func (i *nwIface) SrcName() string {
i.Lock()
defer i.Unlock()
return i.srcName
}
func (i *nwIface) DstName() string {
i.Lock()
defer i.Unlock()
return i.dstName
}
func (i *nwIface) DstMaster() string {
i.Lock()
defer i.Unlock()
return i.dstMaster
}
func (i *nwIface) Bridge() bool {
i.Lock()
defer i.Unlock()
return i.bridge
}
func (i *nwIface) Master() string {
i.Lock()
defer i.Unlock()
return i.master
}
func (i *nwIface) Address() *net.IPNet {
i.Lock()
defer i.Unlock()
return types.GetIPNetCopy(i.address)
}
func (i *nwIface) AddressIPv6() *net.IPNet {
i.Lock()
defer i.Unlock()
return types.GetIPNetCopy(i.addressIPv6)
}
func (i *nwIface) Routes() []*net.IPNet {
i.Lock()
defer i.Unlock()
routes := make([]*net.IPNet, len(i.routes))
for index, route := range i.routes {
r := types.GetIPNetCopy(route)
routes[index] = r
}
return routes
}
func (n *networkNamespace) Interfaces() []Interface {
n.Lock()
defer n.Unlock()
ifaces := make([]Interface, len(n.iFaces))
for i, iface := range n.iFaces {
ifaces[i] = iface
}
return ifaces
}
func (i *nwIface) Remove() error {
i.Lock()
n := i.ns
i.Unlock()
n.Lock()
path := n.path
isDefault := n.isDefault
n.Unlock()
return nsInvoke(path, func(nsFD int) error { return nil }, func(callerFD int) error {
// Find the network inteerface identified by the DstName attribute.
iface, err := netlink.LinkByName(i.DstName())
if err != nil {
return err
}
// Down the interface before configuring
if err := netlink.LinkSetDown(iface); err != nil {
return err
}
err = netlink.LinkSetName(iface, i.SrcName())
if err != nil {
fmt.Println("LinkSetName failed: ", err)
return err
}
// if it is a bridge just delete it.
if i.Bridge() {
if err := netlink.LinkDel(iface); err != nil {
return fmt.Errorf("failed deleting bridge %q: %v", i.SrcName(), err)
}
} else if !isDefault {
// Move the network interface to caller namespace.
if err := netlink.LinkSetNsFd(iface, callerFD); err != nil {
fmt.Println("LinkSetNsPid failed: ", err)
return err
}
}
n.Lock()
for index, intf := range n.iFaces {
if intf == i {
n.iFaces = append(n.iFaces[:index], n.iFaces[index+1:]...)
break
}
}
n.Unlock()
return nil
})
}
// Returns the sandbox's side veth interface statistics
func (i *nwIface) Statistics() (*types.InterfaceStatistics, error) {
i.Lock()
n := i.ns
i.Unlock()
n.Lock()
path := n.path
n.Unlock()
s := &types.InterfaceStatistics{}
err := nsInvoke(path, func(nsFD int) error { return nil }, func(callerFD int) error {
// For some reason ioutil.ReadFile(netStatsFile) reads the file in
// the default netns when this code is invoked from docker.
// Executing "cat <netStatsFile>" works as expected.
data, err := exec.Command("cat", netStatsFile).Output()
if err != nil {
return fmt.Errorf("failure opening %s: %v", netStatsFile, err)
}
return scanInterfaceStats(string(data), i.DstName(), s)
})
if err != nil {
err = fmt.Errorf("failed to retrieve the statistics for %s in netns %s: %v", i.DstName(), path, err)
}
return s, err
}
func (n *networkNamespace) findDst(srcName string, isBridge bool) string {
n.Lock()
defer n.Unlock()
for _, i := range n.iFaces {
// The master should match the srcname of the interface and the
// master interface should be of type bridge, if searching for a bridge type
if i.SrcName() == srcName && (!isBridge || i.Bridge()) {
return i.DstName()
}
}
return ""
}
func (n *networkNamespace) AddInterface(srcName, dstPrefix string, options ...IfaceOption) error {
i := &nwIface{srcName: srcName, dstName: dstPrefix, ns: n}
i.processInterfaceOptions(options...)
if i.master != "" {
i.dstMaster = n.findDst(i.master, true)
if i.dstMaster == "" {
return fmt.Errorf("could not find an appropriate master %q for %q",
i.master, i.srcName)
}
}
n.Lock()
if n.isDefault {
i.dstName = i.srcName
} else {
i.dstName = fmt.Sprintf("%s%d", i.dstName, n.nextIfIndex)
n.nextIfIndex++
}
path := n.path
isDefault := n.isDefault
n.Unlock()
return nsInvoke(path, func(nsFD int) error {
// If it is a bridge interface we have to create the bridge inside
// the namespace so don't try to lookup the interface using srcName
if i.bridge {
return nil
}
// Find the network interface identified by the SrcName attribute.
iface, err := netlink.LinkByName(i.srcName)
if err != nil {
return fmt.Errorf("failed to get link by name %q: %v", i.srcName, err)
}
// Move the network interface to the destination
// namespace only if the namespace is not a default
// type
if !isDefault {
if err := netlink.LinkSetNsFd(iface, nsFD); err != nil {
return fmt.Errorf("failed to set namespace on link %q: %v", i.srcName, err)
}
}
return nil
}, func(callerFD int) error {
if i.bridge {
link := &netlink.Bridge{
LinkAttrs: netlink.LinkAttrs{
Name: i.srcName,
},
}
if err := netlink.LinkAdd(link); err != nil {
return fmt.Errorf("failed to create bridge %q: %v", i.srcName, err)
}
}
// Find the network interface identified by the SrcName attribute.
iface, err := netlink.LinkByName(i.srcName)
if err != nil {
return fmt.Errorf("failed to get link by name %q: %v", i.srcName, err)
}
// Down the interface before configuring
if err := netlink.LinkSetDown(iface); err != nil {
return fmt.Errorf("failed to set link down: %v", err)
}
// Configure the interface now this is moved in the proper namespace.
if err := configureInterface(iface, i); err != nil {
return err
}
// Up the interface.
if err := netlink.LinkSetUp(iface); err != nil {
return fmt.Errorf("failed to set link up: %v", err)
}
// Set the routes on the interface. This can only be done when the interface is up.
if err := setInterfaceRoutes(iface, i); err != nil {
return fmt.Errorf("error setting interface %q routes to %q: %v", iface.Attrs().Name, i.Routes(), err)
}
n.Lock()
n.iFaces = append(n.iFaces, i)
n.Unlock()
return nil
})
}
func configureInterface(iface netlink.Link, i *nwIface) error {
ifaceName := iface.Attrs().Name
ifaceConfigurators := []struct {
Fn func(netlink.Link, *nwIface) error
ErrMessage string
}{
{setInterfaceName, fmt.Sprintf("error renaming interface %q to %q", ifaceName, i.DstName())},
{setInterfaceIP, fmt.Sprintf("error setting interface %q IP to %q", ifaceName, i.Address())},
{setInterfaceIPv6, fmt.Sprintf("error setting interface %q IPv6 to %q", ifaceName, i.AddressIPv6())},
{setInterfaceMaster, fmt.Sprintf("error setting interface %q master to %q", ifaceName, i.DstMaster())},
}
for _, config := range ifaceConfigurators {
if err := config.Fn(iface, i); err != nil {
return fmt.Errorf("%s: %v", config.ErrMessage, err)
}
}
return nil
}
func setInterfaceMaster(iface netlink.Link, i *nwIface) error {
if i.DstMaster() == "" {
return nil
}
return netlink.LinkSetMaster(iface, &netlink.Bridge{
LinkAttrs: netlink.LinkAttrs{Name: i.DstMaster()}})
}
func setInterfaceIP(iface netlink.Link, i *nwIface) error {
if i.Address() == nil {
return nil
}
ipAddr := &netlink.Addr{IPNet: i.Address(), Label: ""}
return netlink.AddrAdd(iface, ipAddr)
}
func setInterfaceIPv6(iface netlink.Link, i *nwIface) error {
if i.AddressIPv6() == nil {
return nil
}
ipAddr := &netlink.Addr{IPNet: i.AddressIPv6(), Label: "", Flags: syscall.IFA_F_NODAD}
return netlink.AddrAdd(iface, ipAddr)
}
func setInterfaceName(iface netlink.Link, i *nwIface) error {
return netlink.LinkSetName(iface, i.DstName())
}
func setInterfaceRoutes(iface netlink.Link, i *nwIface) error {
for _, route := range i.Routes() {
err := netlink.RouteAdd(&netlink.Route{
Scope: netlink.SCOPE_LINK,
LinkIndex: iface.Attrs().Index,
Dst: route,
})
if err != nil {
return err
}
}
return nil
}
// In older kernels (like the one in Centos 6.6 distro) sysctl does not have netns support. Therefore
// we cannot gather the statistics from /sys/class/net/<dev>/statistics/<counter> files. Per-netns stats
// are naturally found in /proc/net/dev in kernels which support netns (ifconfig relies on that).
const (
netStatsFile = "/proc/net/dev"
base = "[ ]*%s:([ ]+[0-9]+){16}"
)
func scanInterfaceStats(data, ifName string, i *types.InterfaceStatistics) error {
var (
bktStr string
bkt uint64
)
regex := fmt.Sprintf(base, ifName)
re := regexp.MustCompile(regex)
line := re.FindString(data)
_, err := fmt.Sscanf(line, "%s %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d",
&bktStr, &i.RxBytes, &i.RxPackets, &i.RxErrors, &i.RxDropped, &bkt, &bkt, &bkt,
&bkt, &i.TxBytes, &i.TxPackets, &i.TxErrors, &i.TxDropped, &bkt, &bkt, &bkt, &bkt)
return err
}

View File

@@ -0,0 +1,4 @@
package osl
// IfaceOption is a function option type to set interface options
type IfaceOption func()

View File

@@ -0,0 +1,323 @@
package osl
import (
"fmt"
"net"
"os"
"os/exec"
"runtime"
"sync"
"syscall"
"time"
log "github.com/Sirupsen/logrus"
"github.com/docker/docker/pkg/reexec"
"github.com/docker/libnetwork/ns"
"github.com/docker/libnetwork/types"
"github.com/vishvananda/netlink"
"github.com/vishvananda/netns"
)
const prefix = "/var/run/docker/netns"
var (
once sync.Once
garbagePathMap = make(map[string]bool)
gpmLock sync.Mutex
gpmWg sync.WaitGroup
gpmCleanupPeriod = 60 * time.Second
gpmChan = make(chan chan struct{})
nsOnce sync.Once
)
// The networkNamespace type is the linux implementation of the Sandbox
// interface. It represents a linux network namespace, and moves an interface
// into it when called on method AddInterface or sets the gateway etc.
type networkNamespace struct {
path string
iFaces []*nwIface
gw net.IP
gwv6 net.IP
staticRoutes []*types.StaticRoute
neighbors []*neigh
nextIfIndex int
isDefault bool
sync.Mutex
}
func init() {
reexec.Register("netns-create", reexecCreateNamespace)
}
func createBasePath() {
err := os.MkdirAll(prefix, 0755)
if err != nil {
panic("Could not create net namespace path directory")
}
// Start the garbage collection go routine
go removeUnusedPaths()
}
func removeUnusedPaths() {
gpmLock.Lock()
period := gpmCleanupPeriod
gpmLock.Unlock()
ticker := time.NewTicker(period)
for {
var (
gc chan struct{}
gcOk bool
)
select {
case <-ticker.C:
case gc, gcOk = <-gpmChan:
}
gpmLock.Lock()
pathList := make([]string, 0, len(garbagePathMap))
for path := range garbagePathMap {
pathList = append(pathList, path)
}
garbagePathMap = make(map[string]bool)
gpmWg.Add(1)
gpmLock.Unlock()
for _, path := range pathList {
os.Remove(path)
}
gpmWg.Done()
if gcOk {
close(gc)
}
}
}
func addToGarbagePaths(path string) {
gpmLock.Lock()
garbagePathMap[path] = true
gpmLock.Unlock()
}
func removeFromGarbagePaths(path string) {
gpmLock.Lock()
delete(garbagePathMap, path)
gpmLock.Unlock()
}
// GC triggers garbage collection of namespace path right away
// and waits for it.
func GC() {
gpmLock.Lock()
if len(garbagePathMap) == 0 {
// No need for GC if map is empty
gpmLock.Unlock()
return
}
gpmLock.Unlock()
// if content exists in the garbage paths
// we can trigger GC to run, providing a
// channel to be notified on completion
waitGC := make(chan struct{})
gpmChan <- waitGC
// wait for GC completion
<-waitGC
}
// GenerateKey generates a sandbox key based on the passed
// container id.
func GenerateKey(containerID string) string {
maxLen := 12
if len(containerID) < maxLen {
maxLen = len(containerID)
}
return prefix + "/" + containerID[:maxLen]
}
// NewSandbox provides a new sandbox instance created in an os specific way
// provided a key which uniquely identifies the sandbox
func NewSandbox(key string, osCreate bool) (Sandbox, error) {
err := createNetworkNamespace(key, osCreate)
if err != nil {
return nil, err
}
return &networkNamespace{path: key, isDefault: !osCreate}, nil
}
func (n *networkNamespace) InterfaceOptions() IfaceOptionSetter {
return n
}
func (n *networkNamespace) NeighborOptions() NeighborOptionSetter {
return n
}
func mountNetworkNamespace(basePath string, lnPath string) error {
if err := syscall.Mount(basePath, lnPath, "bind", syscall.MS_BIND, ""); err != nil {
return err
}
if err := loopbackUp(); err != nil {
return err
}
return nil
}
// GetSandboxForExternalKey returns sandbox object for the supplied path
func GetSandboxForExternalKey(basePath string, key string) (Sandbox, error) {
var err error
if err = createNamespaceFile(key); err != nil {
return nil, err
}
n := &networkNamespace{path: basePath}
n.InvokeFunc(func() {
err = mountNetworkNamespace(basePath, key)
})
if err != nil {
return nil, err
}
return &networkNamespace{path: key}, nil
}
func reexecCreateNamespace() {
if len(os.Args) < 2 {
log.Fatal("no namespace path provided")
}
if err := mountNetworkNamespace("/proc/self/ns/net", os.Args[1]); err != nil {
log.Fatal(err)
}
}
func createNetworkNamespace(path string, osCreate bool) error {
if err := createNamespaceFile(path); err != nil {
return err
}
cmd := &exec.Cmd{
Path: reexec.Self(),
Args: append([]string{"netns-create"}, path),
Stdout: os.Stdout,
Stderr: os.Stderr,
}
if osCreate {
cmd.SysProcAttr = &syscall.SysProcAttr{}
cmd.SysProcAttr.Cloneflags = syscall.CLONE_NEWNET
}
if err := cmd.Run(); err != nil {
return fmt.Errorf("namespace creation reexec command failed: %v", err)
}
return nil
}
func unmountNamespaceFile(path string) {
if _, err := os.Stat(path); err == nil {
syscall.Unmount(path, syscall.MNT_DETACH)
}
}
func createNamespaceFile(path string) (err error) {
var f *os.File
once.Do(createBasePath)
// Remove it from garbage collection list if present
removeFromGarbagePaths(path)
// If the path is there unmount it first
unmountNamespaceFile(path)
// wait for garbage collection to complete if it is in progress
// before trying to create the file.
gpmWg.Wait()
if f, err = os.Create(path); err == nil {
f.Close()
}
return err
}
func loopbackUp() error {
iface, err := netlink.LinkByName("lo")
if err != nil {
return err
}
return netlink.LinkSetUp(iface)
}
func (n *networkNamespace) InvokeFunc(f func()) error {
return nsInvoke(n.nsPath(), func(nsFD int) error { return nil }, func(callerFD int) error {
f()
return nil
})
}
// InitOSContext initializes OS context while configuring network resources
func InitOSContext() func() {
runtime.LockOSThread()
nsOnce.Do(ns.Init)
if err := ns.SetNamespace(); err != nil {
log.Error(err)
}
return runtime.UnlockOSThread
}
func nsInvoke(path string, prefunc func(nsFD int) error, postfunc func(callerFD int) error) error {
defer InitOSContext()()
f, err := os.OpenFile(path, os.O_RDONLY, 0)
if err != nil {
return fmt.Errorf("failed get network namespace %q: %v", path, err)
}
defer f.Close()
nsFD := f.Fd()
// Invoked before the namespace switch happens but after the namespace file
// handle is obtained.
if err := prefunc(int(nsFD)); err != nil {
return fmt.Errorf("failed in prefunc: %v", err)
}
if err = netns.Set(netns.NsHandle(nsFD)); err != nil {
return err
}
defer ns.SetNamespace()
// Invoked after the namespace switch.
return postfunc(ns.ParseHandlerInt())
}
func (n *networkNamespace) nsPath() string {
n.Lock()
defer n.Unlock()
return n.path
}
func (n *networkNamespace) Info() Info {
return n
}
func (n *networkNamespace) Key() string {
return n.path
}
func (n *networkNamespace) Destroy() error {
// Assuming no running process is executing in this network namespace,
// unmounting is sufficient to destroy it.
if err := syscall.Unmount(n.path, syscall.MNT_DETACH); err != nil {
return err
}
// Stash it into the garbage collection list
addToGarbagePaths(n.path)
return nil
}

View File

@@ -0,0 +1,12 @@
// +build !linux,!windows,!freebsd
package osl
// GC triggers garbage collection of namespace path right away
// and waits for it.
func GC() {
}
func GetSandboxForExternalKey(path string, key string) (Sandbox, error) {
return nil, nil
}

View File

@@ -0,0 +1,39 @@
package osl
import "testing"
// GenerateKey generates a sandbox key based on the passed
// container id.
func GenerateKey(containerID string) string {
maxLen := 12
if len(containerID) < maxLen {
maxLen = len(containerID)
}
return containerID[:maxLen]
}
// NewSandbox provides a new sandbox instance created in an os specific way
// provided a key which uniquely identifies the sandbox
func NewSandbox(key string, osCreate bool) (Sandbox, error) {
return nil, nil
}
func GetSandboxForExternalKey(path string, key string) (Sandbox, error) {
return nil, nil
}
// GC triggers garbage collection of namespace path right away
// and waits for it.
func GC() {
}
// InitOSContext initializes OS context while configuring network resources
func InitOSContext() func() {
return func() {}
}
// SetupTestOSContext sets up a separate test OS context in which tests will be executed.
func SetupTestOSContext(t *testing.T) func() {
return func() {}
}

View File

@@ -0,0 +1,4 @@
package osl
// NeighOption is a function option type to set neighbor options
type NeighOption func()

138
vendor/github.com/docker/libnetwork/osl/neigh_linux.go generated vendored Normal file
View File

@@ -0,0 +1,138 @@
package osl
import (
"bytes"
"fmt"
"net"
"github.com/vishvananda/netlink"
)
// NeighOption is a function option type to set interface options
type NeighOption func(nh *neigh)
type neigh struct {
dstIP net.IP
dstMac net.HardwareAddr
linkName string
linkDst string
family int
}
func (n *networkNamespace) findNeighbor(dstIP net.IP, dstMac net.HardwareAddr) *neigh {
n.Lock()
defer n.Unlock()
for _, nh := range n.neighbors {
if nh.dstIP.Equal(dstIP) && bytes.Equal(nh.dstMac, dstMac) {
return nh
}
}
return nil
}
func (n *networkNamespace) DeleteNeighbor(dstIP net.IP, dstMac net.HardwareAddr) error {
nh := n.findNeighbor(dstIP, dstMac)
if nh == nil {
return fmt.Errorf("could not find the neighbor entry to delete")
}
return nsInvoke(n.nsPath(), func(nsFD int) error { return nil }, func(callerFD int) error {
var iface netlink.Link
if nh.linkDst != "" {
var err error
iface, err = netlink.LinkByName(nh.linkDst)
if err != nil {
return fmt.Errorf("could not find interface with destination name %s: %v",
nh.linkDst, err)
}
}
nlnh := &netlink.Neigh{
IP: dstIP,
State: netlink.NUD_PERMANENT,
Family: nh.family,
}
if nlnh.Family > 0 {
nlnh.HardwareAddr = dstMac
nlnh.Flags = netlink.NTF_SELF
}
if nh.linkDst != "" {
nlnh.LinkIndex = iface.Attrs().Index
}
if err := netlink.NeighDel(nlnh); err != nil {
return fmt.Errorf("could not delete neighbor entry: %v", err)
}
for i, nh := range n.neighbors {
if nh.dstIP.Equal(dstIP) && bytes.Equal(nh.dstMac, dstMac) {
n.neighbors = append(n.neighbors[:i], n.neighbors[i+1:]...)
}
}
return nil
})
}
func (n *networkNamespace) AddNeighbor(dstIP net.IP, dstMac net.HardwareAddr, options ...NeighOption) error {
nh := n.findNeighbor(dstIP, dstMac)
if nh != nil {
// If it exists silently return
return nil
}
nh = &neigh{
dstIP: dstIP,
dstMac: dstMac,
}
nh.processNeighOptions(options...)
if nh.linkName != "" {
nh.linkDst = n.findDst(nh.linkName, false)
if nh.linkDst == "" {
return fmt.Errorf("could not find the interface with name %s", nh.linkName)
}
}
return nsInvoke(n.nsPath(), func(nsFD int) error { return nil }, func(callerFD int) error {
var iface netlink.Link
if nh.linkDst != "" {
var err error
iface, err = netlink.LinkByName(nh.linkDst)
if err != nil {
return fmt.Errorf("could not find interface with destination name %s: %v",
nh.linkDst, err)
}
}
nlnh := &netlink.Neigh{
IP: dstIP,
HardwareAddr: dstMac,
State: netlink.NUD_PERMANENT,
Family: nh.family,
}
if nlnh.Family > 0 {
nlnh.Flags = netlink.NTF_SELF
}
if nh.linkDst != "" {
nlnh.LinkIndex = iface.Attrs().Index
}
if err := netlink.NeighSet(nlnh); err != nil {
return fmt.Errorf("could not add neighbor entry: %v", err)
}
n.neighbors = append(n.neighbors, nh)
return nil
})
}

View File

@@ -0,0 +1,4 @@
package osl
// NeighOption is a function option type to set neighbor options
type NeighOption func()

View File

@@ -0,0 +1,61 @@
package osl
import "net"
func (nh *neigh) processNeighOptions(options ...NeighOption) {
for _, opt := range options {
if opt != nil {
opt(nh)
}
}
}
func (n *networkNamespace) LinkName(name string) NeighOption {
return func(nh *neigh) {
nh.linkName = name
}
}
func (n *networkNamespace) Family(family int) NeighOption {
return func(nh *neigh) {
nh.family = family
}
}
func (i *nwIface) processInterfaceOptions(options ...IfaceOption) {
for _, opt := range options {
if opt != nil {
opt(i)
}
}
}
func (n *networkNamespace) Bridge(isBridge bool) IfaceOption {
return func(i *nwIface) {
i.bridge = isBridge
}
}
func (n *networkNamespace) Master(name string) IfaceOption {
return func(i *nwIface) {
i.master = name
}
}
func (n *networkNamespace) Address(addr *net.IPNet) IfaceOption {
return func(i *nwIface) {
i.address = addr
}
}
func (n *networkNamespace) AddressIPv6(addr *net.IPNet) IfaceOption {
return func(i *nwIface) {
i.addressIPv6 = addr
}
}
func (n *networkNamespace) Routes(routes []*net.IPNet) IfaceOption {
return func(i *nwIface) {
i.routes = routes
}
}

197
vendor/github.com/docker/libnetwork/osl/route_linux.go generated vendored Normal file
View File

@@ -0,0 +1,197 @@
package osl
import (
"fmt"
"net"
"github.com/docker/libnetwork/types"
"github.com/vishvananda/netlink"
)
func (n *networkNamespace) Gateway() net.IP {
n.Lock()
defer n.Unlock()
return n.gw
}
func (n *networkNamespace) GatewayIPv6() net.IP {
n.Lock()
defer n.Unlock()
return n.gwv6
}
func (n *networkNamespace) StaticRoutes() []*types.StaticRoute {
n.Lock()
defer n.Unlock()
routes := make([]*types.StaticRoute, len(n.staticRoutes))
for i, route := range n.staticRoutes {
r := route.GetCopy()
routes[i] = r
}
return routes
}
func (n *networkNamespace) setGateway(gw net.IP) {
n.Lock()
n.gw = gw
n.Unlock()
}
func (n *networkNamespace) setGatewayIPv6(gwv6 net.IP) {
n.Lock()
n.gwv6 = gwv6
n.Unlock()
}
func (n *networkNamespace) SetGateway(gw net.IP) error {
// Silently return if the gateway is empty
if len(gw) == 0 {
return nil
}
err := programGateway(n.nsPath(), gw, true)
if err == nil {
n.setGateway(gw)
}
return err
}
func (n *networkNamespace) UnsetGateway() error {
gw := n.Gateway()
// Silently return if the gateway is empty
if len(gw) == 0 {
return nil
}
err := programGateway(n.nsPath(), gw, false)
if err == nil {
n.setGateway(net.IP{})
}
return err
}
func programGateway(path string, gw net.IP, isAdd bool) error {
return nsInvoke(path, func(nsFD int) error { return nil }, func(callerFD int) error {
gwRoutes, err := netlink.RouteGet(gw)
if err != nil {
return fmt.Errorf("route for the gateway %s could not be found: %v", gw, err)
}
if isAdd {
return netlink.RouteAdd(&netlink.Route{
Scope: netlink.SCOPE_UNIVERSE,
LinkIndex: gwRoutes[0].LinkIndex,
Gw: gw,
})
}
return netlink.RouteDel(&netlink.Route{
Scope: netlink.SCOPE_UNIVERSE,
LinkIndex: gwRoutes[0].LinkIndex,
Gw: gw,
})
})
}
// Program a route in to the namespace routing table.
func programRoute(path string, dest *net.IPNet, nh net.IP) error {
return nsInvoke(path, func(nsFD int) error { return nil }, func(callerFD int) error {
gwRoutes, err := netlink.RouteGet(nh)
if err != nil {
return fmt.Errorf("route for the next hop %s could not be found: %v", nh, err)
}
return netlink.RouteAdd(&netlink.Route{
Scope: netlink.SCOPE_UNIVERSE,
LinkIndex: gwRoutes[0].LinkIndex,
Gw: nh,
Dst: dest,
})
})
}
// Delete a route from the namespace routing table.
func removeRoute(path string, dest *net.IPNet, nh net.IP) error {
return nsInvoke(path, func(nsFD int) error { return nil }, func(callerFD int) error {
gwRoutes, err := netlink.RouteGet(nh)
if err != nil {
return fmt.Errorf("route for the next hop could not be found: %v", err)
}
return netlink.RouteDel(&netlink.Route{
Scope: netlink.SCOPE_UNIVERSE,
LinkIndex: gwRoutes[0].LinkIndex,
Gw: nh,
Dst: dest,
})
})
}
func (n *networkNamespace) SetGatewayIPv6(gwv6 net.IP) error {
// Silently return if the gateway is empty
if len(gwv6) == 0 {
return nil
}
err := programGateway(n.nsPath(), gwv6, true)
if err == nil {
n.SetGatewayIPv6(gwv6)
}
return err
}
func (n *networkNamespace) UnsetGatewayIPv6() error {
gwv6 := n.GatewayIPv6()
// Silently return if the gateway is empty
if len(gwv6) == 0 {
return nil
}
err := programGateway(n.nsPath(), gwv6, false)
if err == nil {
n.Lock()
n.gwv6 = net.IP{}
n.Unlock()
}
return err
}
func (n *networkNamespace) AddStaticRoute(r *types.StaticRoute) error {
err := programRoute(n.nsPath(), r.Destination, r.NextHop)
if err == nil {
n.Lock()
n.staticRoutes = append(n.staticRoutes, r)
n.Unlock()
}
return err
}
func (n *networkNamespace) RemoveStaticRoute(r *types.StaticRoute) error {
err := removeRoute(n.nsPath(), r.Destination, r.NextHop)
if err == nil {
n.Lock()
lastIndex := len(n.staticRoutes) - 1
for i, v := range n.staticRoutes {
if v == r {
// Overwrite the route we're removing with the last element
n.staticRoutes[i] = n.staticRoutes[lastIndex]
// Shorten the slice to trim the extra element
n.staticRoutes = n.staticRoutes[:lastIndex]
break
}
}
n.Unlock()
}
return err
}

153
vendor/github.com/docker/libnetwork/osl/sandbox.go generated vendored Normal file
View File

@@ -0,0 +1,153 @@
// Package osl describes structures and interfaces which abstract os entities
package osl
import (
"net"
"github.com/docker/libnetwork/types"
)
// Sandbox represents a network sandbox, identified by a specific key. It
// holds a list of Interfaces, routes etc, and more can be added dynamically.
type Sandbox interface {
// The path where the network namespace is mounted.
Key() string
// Add an existing Interface to this sandbox. The operation will rename
// from the Interface SrcName to DstName as it moves, and reconfigure the
// interface according to the specified settings. The caller is expected
// to only provide a prefix for DstName. The AddInterface api will auto-generate
// an appropriate suffix for the DstName to disambiguate.
AddInterface(SrcName string, DstPrefix string, options ...IfaceOption) error
// Set default IPv4 gateway for the sandbox
SetGateway(gw net.IP) error
// Set default IPv6 gateway for the sandbox
SetGatewayIPv6(gw net.IP) error
// Unset the previously set default IPv4 gateway in the sandbox
UnsetGateway() error
// Unset the previously set default IPv6 gateway in the sandbox
UnsetGatewayIPv6() error
// Add a static route to the sandbox.
AddStaticRoute(*types.StaticRoute) error
// Remove a static route from the sandbox.
RemoveStaticRoute(*types.StaticRoute) error
// AddNeighbor adds a neighbor entry into the sandbox.
AddNeighbor(dstIP net.IP, dstMac net.HardwareAddr, option ...NeighOption) error
// DeleteNeighbor deletes neighbor entry from the sandbox.
DeleteNeighbor(dstIP net.IP, dstMac net.HardwareAddr) error
// Returns an interface with methods to set neighbor options.
NeighborOptions() NeighborOptionSetter
// Returns an interface with methods to set interface options.
InterfaceOptions() IfaceOptionSetter
//Invoke
InvokeFunc(func()) error
// Returns an interface with methods to get sandbox state.
Info() Info
// Destroy the sandbox
Destroy() error
}
// NeighborOptionSetter interfaces defines the option setter methods for interface options
type NeighborOptionSetter interface {
// LinkName returns an option setter to set the srcName of the link that should
// be used in the neighbor entry
LinkName(string) NeighOption
// Family returns an option setter to set the address family for the neighbor
// entry. eg. AF_BRIDGE
Family(int) NeighOption
}
// IfaceOptionSetter interface defines the option setter methods for interface options.
type IfaceOptionSetter interface {
// Bridge returns an option setter to set if the interface is a bridge.
Bridge(bool) IfaceOption
// Address returns an option setter to set IPv4 address.
Address(*net.IPNet) IfaceOption
// Address returns an option setter to set IPv6 address.
AddressIPv6(*net.IPNet) IfaceOption
// Master returns an option setter to set the master interface if any for this
// interface. The master interface name should refer to the srcname of a
// previously added interface of type bridge.
Master(string) IfaceOption
// Address returns an option setter to set interface routes.
Routes([]*net.IPNet) IfaceOption
}
// Info represents all possible information that
// the driver wants to place in the sandbox which includes
// interfaces, routes and gateway
type Info interface {
// The collection of Interface previously added with the AddInterface
// method. Note that this doesn't incude network interfaces added in any
// other way (such as the default loopback interface which are automatically
// created on creation of a sandbox).
Interfaces() []Interface
// IPv4 gateway for the sandbox.
Gateway() net.IP
// IPv6 gateway for the sandbox.
GatewayIPv6() net.IP
// Additional static routes for the sandbox. (Note that directly
// connected routes are stored on the particular interface they refer to.)
StaticRoutes() []*types.StaticRoute
// TODO: Add ip tables etc.
}
// Interface represents the settings and identity of a network device. It is
// used as a return type for Network.Link, and it is common practice for the
// caller to use this information when moving interface SrcName from host
// namespace to DstName in a different net namespace with the appropriate
// network settings.
type Interface interface {
// The name of the interface in the origin network namespace.
SrcName() string
// The name that will be assigned to the interface once moves inside a
// network namespace. When the caller passes in a DstName, it is only
// expected to pass a prefix. The name will modified with an appropriately
// auto-generated suffix.
DstName() string
// IPv4 address for the interface.
Address() *net.IPNet
// IPv6 address for the interface.
AddressIPv6() *net.IPNet
// IP routes for the interface.
Routes() []*net.IPNet
// Bridge returns true if the interface is a bridge
Bridge() bool
// Master returns the srcname of the master interface for this interface.
Master() string
// Remove an interface from the sandbox by renaming to original name
// and moving it out of the sandbox.
Remove() error
// Statistics returns the statistics for this interface
Statistics() (*types.InterfaceStatistics, error)
}

View File

@@ -0,0 +1,40 @@
package osl
import "testing"
// GenerateKey generates a sandbox key based on the passed
// container id.
func GenerateKey(containerID string) string {
maxLen := 12
if len(containerID) < maxLen {
maxLen = len(containerID)
}
return containerID[:maxLen]
}
// NewSandbox provides a new sandbox instance created in an os specific way
// provided a key which uniquely identifies the sandbox
func NewSandbox(key string, osCreate bool) (Sandbox, error) {
return nil, nil
}
// GetSandboxForExternalKey returns sandbox object for the supplied path
func GetSandboxForExternalKey(path string, key string) (Sandbox, error) {
return nil, nil
}
// GC triggers garbage collection of namespace path right away
// and waits for it.
func GC() {
}
// InitOSContext initializes OS context while configuring network resources
func InitOSContext() func() {
return func() {}
}
// SetupTestOSContext sets up a separate test OS context in which tests will be executed.
func SetupTestOSContext(t *testing.T) func() {
return func() {}
}

View File

@@ -0,0 +1,224 @@
package osl
import (
"net"
"os"
"path/filepath"
"runtime"
"syscall"
"testing"
"time"
"github.com/docker/libnetwork/netutils"
"github.com/docker/libnetwork/testutils"
"github.com/docker/libnetwork/types"
"github.com/vishvananda/netlink"
"github.com/vishvananda/netlink/nl"
"github.com/vishvananda/netns"
)
const (
vethName1 = "wierdlongname1"
vethName2 = "wierdlongname2"
vethName3 = "wierdlongname3"
vethName4 = "wierdlongname4"
sboxIfaceName = "containername"
)
func newKey(t *testing.T) (string, error) {
name, err := netutils.GenerateRandomName("netns", 12)
if err != nil {
return "", err
}
name = filepath.Join("/tmp", name)
if _, err := os.Create(name); err != nil {
return "", err
}
// Set the rpmCleanupPeriod to be low to make the test run quicker
gpmLock.Lock()
gpmCleanupPeriod = 2 * time.Second
gpmLock.Unlock()
return name, nil
}
func newInfo(t *testing.T) (Sandbox, error) {
veth := &netlink.Veth{
LinkAttrs: netlink.LinkAttrs{Name: vethName1, TxQLen: 0},
PeerName: vethName2}
if err := netlink.LinkAdd(veth); err != nil {
return nil, err
}
// Store the sandbox side pipe interface
// This is needed for cleanup on DeleteEndpoint()
intf1 := &nwIface{}
intf1.srcName = vethName2
intf1.dstName = sboxIfaceName
ip4, addr, err := net.ParseCIDR("192.168.1.100/24")
if err != nil {
return nil, err
}
intf1.address = addr
intf1.address.IP = ip4
// ip6, addrv6, err := net.ParseCIDR("2001:DB8::ABCD/48")
ip6, addrv6, err := net.ParseCIDR("fe80::2/64")
if err != nil {
return nil, err
}
intf1.addressIPv6 = addrv6
intf1.addressIPv6.IP = ip6
_, route, err := net.ParseCIDR("192.168.2.1/32")
if err != nil {
return nil, err
}
intf1.routes = []*net.IPNet{route}
intf2 := &nwIface{}
intf2.srcName = "testbridge"
intf2.dstName = sboxIfaceName
intf2.bridge = true
veth = &netlink.Veth{
LinkAttrs: netlink.LinkAttrs{Name: vethName3, TxQLen: 0},
PeerName: vethName4}
if err := netlink.LinkAdd(veth); err != nil {
return nil, err
}
intf3 := &nwIface{}
intf3.srcName = vethName4
intf3.dstName = sboxIfaceName
intf3.master = "testbridge"
info := &networkNamespace{iFaces: []*nwIface{intf1, intf2, intf3}}
info.gw = net.ParseIP("192.168.1.1")
// sinfo.GatewayIPv6 = net.ParseIP("2001:DB8::1")
info.gwv6 = net.ParseIP("fe80::1")
return info, nil
}
func verifySandbox(t *testing.T, s Sandbox, ifaceSuffixes []string) {
_, ok := s.(*networkNamespace)
if !ok {
t.Fatalf("The sandox interface returned is not of type networkNamespace")
}
origns, err := netns.Get()
if err != nil {
t.Fatalf("Could not get the current netns: %v", err)
}
defer origns.Close()
f, err := os.OpenFile(s.Key(), os.O_RDONLY, 0)
if err != nil {
t.Fatalf("Failed top open network namespace path %q: %v", s.Key(), err)
}
defer f.Close()
runtime.LockOSThread()
defer runtime.UnlockOSThread()
nsFD := f.Fd()
if err = netns.Set(netns.NsHandle(nsFD)); err != nil {
t.Fatalf("Setting to the namespace pointed to by the sandbox %s failed: %v", s.Key(), err)
}
defer netns.Set(origns)
for _, suffix := range ifaceSuffixes {
_, err = netlink.LinkByName(sboxIfaceName + suffix)
if err != nil {
t.Fatalf("Could not find the interface %s inside the sandbox: %v",
sboxIfaceName+suffix, err)
}
}
}
func verifyCleanup(t *testing.T, s Sandbox, wait bool) {
if wait {
time.Sleep(time.Duration(gpmCleanupPeriod * 2))
}
if _, err := os.Stat(s.Key()); err == nil {
if wait {
t.Fatalf("The sandbox path %s is not getting cleaned up even after twice the cleanup period", s.Key())
} else {
t.Fatalf("The sandbox path %s is not cleaned up after running gc", s.Key())
}
}
}
func TestScanStatistics(t *testing.T) {
data :=
"Inter-| Receive | Transmit\n" +
" face |bytes packets errs drop fifo frame compressed multicast|bytes packets errs drop fifo colls carrier compressed\n" +
" eth0: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n" +
" wlan0: 7787685 11141 0 0 0 0 0 0 1681390 7220 0 0 0 0 0 0\n" +
" lo: 783782 1853 0 0 0 0 0 0 783782 1853 0 0 0 0 0 0\n" +
"lxcbr0: 0 0 0 0 0 0 0 0 9006 61 0 0 0 0 0 0\n"
i := &types.InterfaceStatistics{}
if err := scanInterfaceStats(data, "wlan0", i); err != nil {
t.Fatal(err)
}
if i.TxBytes != 1681390 || i.TxPackets != 7220 || i.RxBytes != 7787685 || i.RxPackets != 11141 {
t.Fatalf("Error scanning the statistics")
}
if err := scanInterfaceStats(data, "lxcbr0", i); err != nil {
t.Fatal(err)
}
if i.TxBytes != 9006 || i.TxPackets != 61 || i.RxBytes != 0 || i.RxPackets != 0 {
t.Fatalf("Error scanning the statistics")
}
}
func TestDisableIPv6DAD(t *testing.T) {
if testutils.RunningOnCircleCI() {
t.Skipf("Skipping as not supported on CIRCLE CI kernel")
}
defer testutils.SetupTestOSContext(t)()
ipv6, _ := types.ParseCIDR("2001:db8::44/64")
iface := &nwIface{addressIPv6: ipv6}
veth := &netlink.Veth{
LinkAttrs: netlink.LinkAttrs{Name: "sideA"},
PeerName: "sideB",
}
err := netlink.LinkAdd(veth)
if err != nil {
t.Fatal(err)
}
link, err := netlink.LinkByName("sideA")
if err != nil {
t.Fatal(err)
}
err = setInterfaceIPv6(link, iface)
if err != nil {
t.Fatal(err)
}
addrList, err := netlink.AddrList(link, nl.FAMILY_V6)
if err != nil {
t.Fatal(err)
}
if addrList[0].Flags&syscall.IFA_F_NODAD == 0 {
t.Fatalf("Unexpected interface flags: 0x%x. Expected to contain 0x%x", addrList[0].Flags, syscall.IFA_F_NODAD)
}
}

190
vendor/github.com/docker/libnetwork/osl/sandbox_test.go generated vendored Normal file
View File

@@ -0,0 +1,190 @@
package osl
import (
"os"
"runtime"
"testing"
"github.com/docker/docker/pkg/reexec"
"github.com/docker/libnetwork/testutils"
)
func TestMain(m *testing.M) {
if reexec.Init() {
return
}
os.Exit(m.Run())
}
func TestSandboxCreate(t *testing.T) {
defer testutils.SetupTestOSContext(t)()
key, err := newKey(t)
if err != nil {
t.Fatalf("Failed to obtain a key: %v", err)
}
s, err := NewSandbox(key, true)
if err != nil {
t.Fatalf("Failed to create a new sandbox: %v", err)
}
runtime.LockOSThread()
if s.Key() != key {
t.Fatalf("s.Key() returned %s. Expected %s", s.Key(), key)
}
tbox, err := newInfo(t)
if err != nil {
t.Fatalf("Failed to generate new sandbox info: %v", err)
}
for _, i := range tbox.Info().Interfaces() {
err = s.AddInterface(i.SrcName(), i.DstName(),
tbox.InterfaceOptions().Bridge(i.Bridge()),
tbox.InterfaceOptions().Address(i.Address()),
tbox.InterfaceOptions().AddressIPv6(i.AddressIPv6()))
if err != nil {
t.Fatalf("Failed to add interfaces to sandbox: %v", err)
}
runtime.LockOSThread()
}
err = s.SetGateway(tbox.Info().Gateway())
if err != nil {
t.Fatalf("Failed to set gateway to sandbox: %v", err)
}
runtime.LockOSThread()
err = s.SetGatewayIPv6(tbox.Info().GatewayIPv6())
if err != nil {
t.Fatalf("Failed to set ipv6 gateway to sandbox: %v", err)
}
runtime.LockOSThread()
verifySandbox(t, s, []string{"0", "1", "2"})
runtime.LockOSThread()
err = s.Destroy()
if err != nil {
t.Fatal(err)
}
verifyCleanup(t, s, true)
}
func TestSandboxCreateTwice(t *testing.T) {
defer testutils.SetupTestOSContext(t)()
key, err := newKey(t)
if err != nil {
t.Fatalf("Failed to obtain a key: %v", err)
}
_, err = NewSandbox(key, true)
if err != nil {
t.Fatalf("Failed to create a new sandbox: %v", err)
}
runtime.LockOSThread()
// Create another sandbox with the same key to see if we handle it
// gracefully.
s, err := NewSandbox(key, true)
if err != nil {
t.Fatalf("Failed to create a new sandbox: %v", err)
}
runtime.LockOSThread()
err = s.Destroy()
if err != nil {
t.Fatal(err)
}
GC()
verifyCleanup(t, s, false)
}
func TestSandboxGC(t *testing.T) {
key, err := newKey(t)
if err != nil {
t.Fatalf("Failed to obtain a key: %v", err)
}
s, err := NewSandbox(key, true)
if err != nil {
t.Fatalf("Failed to create a new sandbox: %v", err)
}
err = s.Destroy()
if err != nil {
t.Fatal(err)
}
GC()
verifyCleanup(t, s, false)
}
func TestAddRemoveInterface(t *testing.T) {
defer testutils.SetupTestOSContext(t)()
key, err := newKey(t)
if err != nil {
t.Fatalf("Failed to obtain a key: %v", err)
}
s, err := NewSandbox(key, true)
if err != nil {
t.Fatalf("Failed to create a new sandbox: %v", err)
}
runtime.LockOSThread()
if s.Key() != key {
t.Fatalf("s.Key() returned %s. Expected %s", s.Key(), key)
}
tbox, err := newInfo(t)
if err != nil {
t.Fatalf("Failed to generate new sandbox info: %v", err)
}
for _, i := range tbox.Info().Interfaces() {
err = s.AddInterface(i.SrcName(), i.DstName(),
tbox.InterfaceOptions().Bridge(i.Bridge()),
tbox.InterfaceOptions().Address(i.Address()),
tbox.InterfaceOptions().AddressIPv6(i.AddressIPv6()))
if err != nil {
t.Fatalf("Failed to add interfaces to sandbox: %v", err)
}
runtime.LockOSThread()
}
verifySandbox(t, s, []string{"0", "1", "2"})
runtime.LockOSThread()
interfaces := s.Info().Interfaces()
if err := interfaces[0].Remove(); err != nil {
t.Fatalf("Failed to remove interfaces from sandbox: %v", err)
}
runtime.LockOSThread()
verifySandbox(t, s, []string{"1", "2"})
runtime.LockOSThread()
i := tbox.Info().Interfaces()[0]
if err := s.AddInterface(i.SrcName(), i.DstName(),
tbox.InterfaceOptions().Bridge(i.Bridge()),
tbox.InterfaceOptions().Address(i.Address()),
tbox.InterfaceOptions().AddressIPv6(i.AddressIPv6())); err != nil {
t.Fatalf("Failed to add interfaces to sandbox: %v", err)
}
runtime.LockOSThread()
verifySandbox(t, s, []string{"1", "2", "3"})
runtime.LockOSThread()
err = s.Destroy()
if err != nil {
t.Fatal(err)
}
GC()
verifyCleanup(t, s, false)
}

View File

@@ -0,0 +1,22 @@
// +build !linux,!windows,!freebsd
package osl
import "errors"
var (
// ErrNotImplemented is for platforms which don't implement sandbox
ErrNotImplemented = errors.New("not implemented")
)
// NewSandbox provides a new sandbox instance created in an os specific way
// provided a key which uniquely identifies the sandbox
func NewSandbox(key string, osCreate bool) (Sandbox, error) {
return nil, ErrNotImplemented
}
// GenerateKey generates a sandbox key based on the passed
// container id.
func GenerateKey(containerID string) string {
return ""
}

View File

@@ -0,0 +1,18 @@
// +build !linux
package osl
import (
"errors"
"testing"
)
var ErrNotImplemented = errors.New("not implemented")
func newKey(t *testing.T) (string, error) {
return nil, ErrNotImplemented
}
func verifySandbox(t *testing.T, s Sandbox) {
return
}