VMware vSphere Integrated Containers provider (#206)

* Add Virtual Kubelet provider for VIC

Initial virtual kubelet provider for VMware VIC.  This provider currently
handles creating and starting of a pod VM via the VIC portlayer and persona
server.  Image store handling via the VIC persona server.  This provider
currently requires the feature/wolfpack branch of VIC.

* Added pod stop and delete.  Also added node capacity.

Added the ability to stop and delete pod VMs via VIC.  Also retrieve
node capacity information from the VCH.

* Cleanup and readme file

Some file clean up and added a Readme.md markdown file for the VIC
provider.

* Cleaned up errors, added function comments, moved operation code

1. Cleaned up error handling.  Set standard for creating errors.
2. Added method prototype comments for all interface functions.
3. Moved PodCreator, PodStarter, PodStopper, and PodDeleter to a new folder.

* Add mocking code and unit tests for podcache, podcreator, and podstarter

Used the unit test framework used in VIC to handle assertions in the provider's
unit test.  Mocking code generated using OSS project mockery, which is compatible
with the testify assertion framework.

* Vendored packages for the VIC provider

Requires feature/wolfpack branch of VIC and a few specific commit sha of
projects used within VIC.

* Implementation of POD Stopper and Deleter unit tests (#4)

* Updated files for initial PR
This commit is contained in:
Loc Nguyen
2018-06-04 15:41:32 -07:00
committed by Ria Bhatia
parent 98a111e8b7
commit 513cebe7b7
6296 changed files with 1123685 additions and 8 deletions

View File

@@ -0,0 +1,228 @@
package portmapper
import (
"errors"
"fmt"
"net"
"sync"
"github.com/Sirupsen/logrus"
"github.com/docker/libnetwork/iptables"
"github.com/docker/libnetwork/portallocator"
)
type mapping struct {
proto string
userlandProxy userlandProxy
host net.Addr
container net.Addr
}
var newProxy = newProxyCommand
var (
// ErrUnknownBackendAddressType refers to an unknown container or unsupported address type
ErrUnknownBackendAddressType = errors.New("unknown container address type not supported")
// ErrPortMappedForIP refers to a port already mapped to an ip address
ErrPortMappedForIP = errors.New("port is already mapped to ip")
// ErrPortNotMapped refers to an unmapped port
ErrPortNotMapped = errors.New("port is not mapped")
)
// PortMapper manages the network address translation
type PortMapper struct {
chain *iptables.ChainInfo
bridgeName string
// udp:ip:port
currentMappings map[string]*mapping
lock sync.Mutex
Allocator *portallocator.PortAllocator
}
// New returns a new instance of PortMapper
func New() *PortMapper {
return NewWithPortAllocator(portallocator.Get())
}
// NewWithPortAllocator returns a new instance of PortMapper which will use the specified PortAllocator
func NewWithPortAllocator(allocator *portallocator.PortAllocator) *PortMapper {
return &PortMapper{
currentMappings: make(map[string]*mapping),
Allocator: allocator,
}
}
// SetIptablesChain sets the specified chain into portmapper
func (pm *PortMapper) SetIptablesChain(c *iptables.ChainInfo, bridgeName string) {
pm.chain = c
pm.bridgeName = bridgeName
}
// Map maps the specified container transport address to the host's network address and transport port
func (pm *PortMapper) Map(container net.Addr, hostIP net.IP, hostPort int, useProxy bool) (host net.Addr, err error) {
return pm.MapRange(container, hostIP, hostPort, hostPort, useProxy)
}
// MapRange maps the specified container transport address to the host's network address and transport port range
func (pm *PortMapper) MapRange(container net.Addr, hostIP net.IP, hostPortStart, hostPortEnd int, useProxy bool) (host net.Addr, err error) {
pm.lock.Lock()
defer pm.lock.Unlock()
var (
m *mapping
proto string
allocatedHostPort int
)
switch container.(type) {
case *net.TCPAddr:
proto = "tcp"
if allocatedHostPort, err = pm.Allocator.RequestPortInRange(hostIP, proto, hostPortStart, hostPortEnd); err != nil {
return nil, err
}
m = &mapping{
proto: proto,
host: &net.TCPAddr{IP: hostIP, Port: allocatedHostPort},
container: container,
}
if useProxy {
m.userlandProxy = newProxy(proto, hostIP, allocatedHostPort, container.(*net.TCPAddr).IP, container.(*net.TCPAddr).Port)
} else {
m.userlandProxy = newDummyProxy(proto, hostIP, allocatedHostPort)
}
case *net.UDPAddr:
proto = "udp"
if allocatedHostPort, err = pm.Allocator.RequestPortInRange(hostIP, proto, hostPortStart, hostPortEnd); err != nil {
return nil, err
}
m = &mapping{
proto: proto,
host: &net.UDPAddr{IP: hostIP, Port: allocatedHostPort},
container: container,
}
if useProxy {
m.userlandProxy = newProxy(proto, hostIP, allocatedHostPort, container.(*net.UDPAddr).IP, container.(*net.UDPAddr).Port)
} else {
m.userlandProxy = newDummyProxy(proto, hostIP, allocatedHostPort)
}
default:
return nil, ErrUnknownBackendAddressType
}
// release the allocated port on any further error during return.
defer func() {
if err != nil {
pm.Allocator.ReleasePort(hostIP, proto, allocatedHostPort)
}
}()
key := getKey(m.host)
if _, exists := pm.currentMappings[key]; exists {
return nil, ErrPortMappedForIP
}
containerIP, containerPort := getIPAndPort(m.container)
if err := pm.forward(iptables.Append, m.proto, hostIP, allocatedHostPort, containerIP.String(), containerPort); err != nil {
return nil, err
}
cleanup := func() error {
// need to undo the iptables rules before we return
m.userlandProxy.Stop()
pm.forward(iptables.Delete, m.proto, hostIP, allocatedHostPort, containerIP.String(), containerPort)
if err := pm.Allocator.ReleasePort(hostIP, m.proto, allocatedHostPort); err != nil {
return err
}
return nil
}
if err := m.userlandProxy.Start(); err != nil {
if err := cleanup(); err != nil {
return nil, fmt.Errorf("Error during port allocation cleanup: %v", err)
}
return nil, err
}
pm.currentMappings[key] = m
return m.host, nil
}
// Unmap removes stored mapping for the specified host transport address
func (pm *PortMapper) Unmap(host net.Addr) error {
pm.lock.Lock()
defer pm.lock.Unlock()
key := getKey(host)
data, exists := pm.currentMappings[key]
if !exists {
return ErrPortNotMapped
}
if data.userlandProxy != nil {
data.userlandProxy.Stop()
}
delete(pm.currentMappings, key)
containerIP, containerPort := getIPAndPort(data.container)
hostIP, hostPort := getIPAndPort(data.host)
if err := pm.forward(iptables.Delete, data.proto, hostIP, hostPort, containerIP.String(), containerPort); err != nil {
logrus.Errorf("Error on iptables delete: %s", err)
}
switch a := host.(type) {
case *net.TCPAddr:
return pm.Allocator.ReleasePort(a.IP, "tcp", a.Port)
case *net.UDPAddr:
return pm.Allocator.ReleasePort(a.IP, "udp", a.Port)
}
return nil
}
//ReMapAll will re-apply all port mappings
func (pm *PortMapper) ReMapAll() {
pm.lock.Lock()
defer pm.lock.Unlock()
logrus.Debugln("Re-applying all port mappings.")
for _, data := range pm.currentMappings {
containerIP, containerPort := getIPAndPort(data.container)
hostIP, hostPort := getIPAndPort(data.host)
if err := pm.forward(iptables.Append, data.proto, hostIP, hostPort, containerIP.String(), containerPort); err != nil {
logrus.Errorf("Error on iptables add: %s", err)
}
}
}
func getKey(a net.Addr) string {
switch t := a.(type) {
case *net.TCPAddr:
return fmt.Sprintf("%s:%d/%s", t.IP.String(), t.Port, "tcp")
case *net.UDPAddr:
return fmt.Sprintf("%s:%d/%s", t.IP.String(), t.Port, "udp")
}
return ""
}
func getIPAndPort(a net.Addr) (net.IP, int) {
switch t := a.(type) {
case *net.TCPAddr:
return t.IP, t.Port
case *net.UDPAddr:
return t.IP, t.Port
}
return nil, 0
}
func (pm *PortMapper) forward(action iptables.Action, proto string, sourceIP net.IP, sourcePort int, containerIP string, containerPort int) error {
if pm.chain == nil {
return nil
}
return pm.chain.Forward(action, sourceIP, sourcePort, proto, containerIP, containerPort, pm.bridgeName)
}

View File

@@ -0,0 +1,270 @@
package portmapper
import (
"net"
"strings"
"testing"
"github.com/docker/libnetwork/iptables"
_ "github.com/docker/libnetwork/testutils"
)
func init() {
// override this func to mock out the proxy server
newProxy = newMockProxyCommand
}
func TestSetIptablesChain(t *testing.T) {
pm := New()
c := &iptables.ChainInfo{
Name: "TEST",
}
if pm.chain != nil {
t.Fatal("chain should be nil at init")
}
pm.SetIptablesChain(c, "lo")
if pm.chain == nil {
t.Fatal("chain should not be nil after set")
}
}
func TestMapTCPPorts(t *testing.T) {
pm := New()
dstIP1 := net.ParseIP("192.168.0.1")
dstIP2 := net.ParseIP("192.168.0.2")
dstAddr1 := &net.TCPAddr{IP: dstIP1, Port: 80}
dstAddr2 := &net.TCPAddr{IP: dstIP2, Port: 80}
srcAddr1 := &net.TCPAddr{Port: 1080, IP: net.ParseIP("172.16.0.1")}
srcAddr2 := &net.TCPAddr{Port: 1080, IP: net.ParseIP("172.16.0.2")}
addrEqual := func(addr1, addr2 net.Addr) bool {
return (addr1.Network() == addr2.Network()) && (addr1.String() == addr2.String())
}
if host, err := pm.Map(srcAddr1, dstIP1, 80, true); err != nil {
t.Fatalf("Failed to allocate port: %s", err)
} else if !addrEqual(dstAddr1, host) {
t.Fatalf("Incorrect mapping result: expected %s:%s, got %s:%s",
dstAddr1.String(), dstAddr1.Network(), host.String(), host.Network())
}
if _, err := pm.Map(srcAddr1, dstIP1, 80, true); err == nil {
t.Fatalf("Port is in use - mapping should have failed")
}
if _, err := pm.Map(srcAddr2, dstIP1, 80, true); err == nil {
t.Fatalf("Port is in use - mapping should have failed")
}
if _, err := pm.Map(srcAddr2, dstIP2, 80, true); err != nil {
t.Fatalf("Failed to allocate port: %s", err)
}
if pm.Unmap(dstAddr1) != nil {
t.Fatalf("Failed to release port")
}
if pm.Unmap(dstAddr2) != nil {
t.Fatalf("Failed to release port")
}
if pm.Unmap(dstAddr2) == nil {
t.Fatalf("Port already released, but no error reported")
}
}
func TestGetUDPKey(t *testing.T) {
addr := &net.UDPAddr{IP: net.ParseIP("192.168.1.5"), Port: 53}
key := getKey(addr)
if expected := "192.168.1.5:53/udp"; key != expected {
t.Fatalf("expected key %s got %s", expected, key)
}
}
func TestGetTCPKey(t *testing.T) {
addr := &net.TCPAddr{IP: net.ParseIP("192.168.1.5"), Port: 80}
key := getKey(addr)
if expected := "192.168.1.5:80/tcp"; key != expected {
t.Fatalf("expected key %s got %s", expected, key)
}
}
func TestGetUDPIPAndPort(t *testing.T) {
addr := &net.UDPAddr{IP: net.ParseIP("192.168.1.5"), Port: 53}
ip, port := getIPAndPort(addr)
if expected := "192.168.1.5"; ip.String() != expected {
t.Fatalf("expected ip %s got %s", expected, ip)
}
if ep := 53; port != ep {
t.Fatalf("expected port %d got %d", ep, port)
}
}
func TestMapUDPPorts(t *testing.T) {
pm := New()
dstIP1 := net.ParseIP("192.168.0.1")
dstIP2 := net.ParseIP("192.168.0.2")
dstAddr1 := &net.UDPAddr{IP: dstIP1, Port: 80}
dstAddr2 := &net.UDPAddr{IP: dstIP2, Port: 80}
srcAddr1 := &net.UDPAddr{Port: 1080, IP: net.ParseIP("172.16.0.1")}
srcAddr2 := &net.UDPAddr{Port: 1080, IP: net.ParseIP("172.16.0.2")}
addrEqual := func(addr1, addr2 net.Addr) bool {
return (addr1.Network() == addr2.Network()) && (addr1.String() == addr2.String())
}
if host, err := pm.Map(srcAddr1, dstIP1, 80, true); err != nil {
t.Fatalf("Failed to allocate port: %s", err)
} else if !addrEqual(dstAddr1, host) {
t.Fatalf("Incorrect mapping result: expected %s:%s, got %s:%s",
dstAddr1.String(), dstAddr1.Network(), host.String(), host.Network())
}
if _, err := pm.Map(srcAddr1, dstIP1, 80, true); err == nil {
t.Fatalf("Port is in use - mapping should have failed")
}
if _, err := pm.Map(srcAddr2, dstIP1, 80, true); err == nil {
t.Fatalf("Port is in use - mapping should have failed")
}
if _, err := pm.Map(srcAddr2, dstIP2, 80, true); err != nil {
t.Fatalf("Failed to allocate port: %s", err)
}
if pm.Unmap(dstAddr1) != nil {
t.Fatalf("Failed to release port")
}
if pm.Unmap(dstAddr2) != nil {
t.Fatalf("Failed to release port")
}
if pm.Unmap(dstAddr2) == nil {
t.Fatalf("Port already released, but no error reported")
}
}
func TestMapAllPortsSingleInterface(t *testing.T) {
pm := New()
dstIP1 := net.ParseIP("0.0.0.0")
srcAddr1 := &net.TCPAddr{Port: 1080, IP: net.ParseIP("172.16.0.1")}
hosts := []net.Addr{}
var host net.Addr
var err error
defer func() {
for _, val := range hosts {
pm.Unmap(val)
}
}()
for i := 0; i < 10; i++ {
start, end := pm.Allocator.Begin, pm.Allocator.End
for i := start; i < end; i++ {
if host, err = pm.Map(srcAddr1, dstIP1, 0, true); err != nil {
t.Fatal(err)
}
hosts = append(hosts, host)
}
if _, err := pm.Map(srcAddr1, dstIP1, start, true); err == nil {
t.Fatalf("Port %d should be bound but is not", start)
}
for _, val := range hosts {
if err := pm.Unmap(val); err != nil {
t.Fatal(err)
}
}
hosts = []net.Addr{}
}
}
func TestMapTCPDummyListen(t *testing.T) {
pm := New()
dstIP := net.ParseIP("0.0.0.0")
dstAddr := &net.TCPAddr{IP: dstIP, Port: 80}
// no-op for dummy
srcAddr := &net.TCPAddr{Port: 1080, IP: net.ParseIP("172.16.0.1")}
addrEqual := func(addr1, addr2 net.Addr) bool {
return (addr1.Network() == addr2.Network()) && (addr1.String() == addr2.String())
}
if host, err := pm.Map(srcAddr, dstIP, 80, false); err != nil {
t.Fatalf("Failed to allocate port: %s", err)
} else if !addrEqual(dstAddr, host) {
t.Fatalf("Incorrect mapping result: expected %s:%s, got %s:%s",
dstAddr.String(), dstAddr.Network(), host.String(), host.Network())
}
if _, err := net.Listen("tcp", "0.0.0.0:80"); err == nil {
t.Fatal("Listen on mapped port without proxy should fail")
} else {
if !strings.Contains(err.Error(), "address already in use") {
t.Fatalf("Error should be about address already in use, got %v", err)
}
}
if _, err := net.Listen("tcp", "0.0.0.0:81"); err != nil {
t.Fatal(err)
}
if host, err := pm.Map(srcAddr, dstIP, 81, false); err == nil {
t.Fatalf("Bound port shouldn't be allocated, but it was on: %v", host)
} else {
if !strings.Contains(err.Error(), "address already in use") {
t.Fatalf("Error should be about address already in use, got %v", err)
}
}
}
func TestMapUDPDummyListen(t *testing.T) {
pm := New()
dstIP := net.ParseIP("0.0.0.0")
dstAddr := &net.UDPAddr{IP: dstIP, Port: 80}
// no-op for dummy
srcAddr := &net.UDPAddr{Port: 1080, IP: net.ParseIP("172.16.0.1")}
addrEqual := func(addr1, addr2 net.Addr) bool {
return (addr1.Network() == addr2.Network()) && (addr1.String() == addr2.String())
}
if host, err := pm.Map(srcAddr, dstIP, 80, false); err != nil {
t.Fatalf("Failed to allocate port: %s", err)
} else if !addrEqual(dstAddr, host) {
t.Fatalf("Incorrect mapping result: expected %s:%s, got %s:%s",
dstAddr.String(), dstAddr.Network(), host.String(), host.Network())
}
if _, err := net.ListenUDP("udp", &net.UDPAddr{IP: dstIP, Port: 80}); err == nil {
t.Fatal("Listen on mapped port without proxy should fail")
} else {
if !strings.Contains(err.Error(), "address already in use") {
t.Fatalf("Error should be about address already in use, got %v", err)
}
}
if _, err := net.ListenUDP("udp", &net.UDPAddr{IP: dstIP, Port: 81}); err != nil {
t.Fatal(err)
}
if host, err := pm.Map(srcAddr, dstIP, 81, false); err == nil {
t.Fatalf("Bound port shouldn't be allocated, but it was on: %v", host)
} else {
if !strings.Contains(err.Error(), "address already in use") {
t.Fatalf("Error should be about address already in use, got %v", err)
}
}
}

View File

@@ -0,0 +1,18 @@
package portmapper
import "net"
func newMockProxyCommand(proto string, hostIP net.IP, hostPort int, containerIP net.IP, containerPort int) userlandProxy {
return &mockProxyCommand{}
}
type mockProxyCommand struct {
}
func (p *mockProxyCommand) Start() error {
return nil
}
func (p *mockProxyCommand) Stop() error {
return nil
}

209
vendor/github.com/docker/libnetwork/portmapper/proxy.go generated vendored Normal file
View File

@@ -0,0 +1,209 @@
package portmapper
import (
"flag"
"fmt"
"io"
"io/ioutil"
"log"
"net"
"os"
"os/exec"
"os/signal"
"strconv"
"syscall"
"time"
"github.com/docker/docker/pkg/proxy"
"github.com/docker/docker/pkg/reexec"
)
const userlandProxyCommandName = "docker-proxy"
func init() {
reexec.Register(userlandProxyCommandName, execProxy)
}
type userlandProxy interface {
Start() error
Stop() error
}
// proxyCommand wraps an exec.Cmd to run the userland TCP and UDP
// proxies as separate processes.
type proxyCommand struct {
cmd *exec.Cmd
}
// execProxy is the reexec function that is registered to start the userland proxies
func execProxy() {
f := os.NewFile(3, "signal-parent")
host, container := parseHostContainerAddrs()
p, err := proxy.NewProxy(host, container)
if err != nil {
fmt.Fprintf(f, "1\n%s", err)
f.Close()
os.Exit(1)
}
go handleStopSignals(p)
fmt.Fprint(f, "0\n")
f.Close()
// Run will block until the proxy stops
p.Run()
}
// parseHostContainerAddrs parses the flags passed on reexec to create the TCP or UDP
// net.Addrs to map the host and container ports
func parseHostContainerAddrs() (host net.Addr, container net.Addr) {
var (
proto = flag.String("proto", "tcp", "proxy protocol")
hostIP = flag.String("host-ip", "", "host ip")
hostPort = flag.Int("host-port", -1, "host port")
containerIP = flag.String("container-ip", "", "container ip")
containerPort = flag.Int("container-port", -1, "container port")
)
flag.Parse()
switch *proto {
case "tcp":
host = &net.TCPAddr{IP: net.ParseIP(*hostIP), Port: *hostPort}
container = &net.TCPAddr{IP: net.ParseIP(*containerIP), Port: *containerPort}
case "udp":
host = &net.UDPAddr{IP: net.ParseIP(*hostIP), Port: *hostPort}
container = &net.UDPAddr{IP: net.ParseIP(*containerIP), Port: *containerPort}
default:
log.Fatalf("unsupported protocol %s", *proto)
}
return host, container
}
func handleStopSignals(p proxy.Proxy) {
s := make(chan os.Signal, 10)
signal.Notify(s, os.Interrupt, syscall.SIGTERM, syscall.SIGSTOP)
for _ = range s {
p.Close()
os.Exit(0)
}
}
func newProxyCommand(proto string, hostIP net.IP, hostPort int, containerIP net.IP, containerPort int) userlandProxy {
args := []string{
userlandProxyCommandName,
"-proto", proto,
"-host-ip", hostIP.String(),
"-host-port", strconv.Itoa(hostPort),
"-container-ip", containerIP.String(),
"-container-port", strconv.Itoa(containerPort),
}
return &proxyCommand{
cmd: &exec.Cmd{
Path: reexec.Self(),
Args: args,
SysProcAttr: &syscall.SysProcAttr{
Pdeathsig: syscall.SIGTERM, // send a sigterm to the proxy if the daemon process dies
},
},
}
}
func (p *proxyCommand) Start() error {
r, w, err := os.Pipe()
if err != nil {
return fmt.Errorf("proxy unable to open os.Pipe %s", err)
}
defer r.Close()
p.cmd.ExtraFiles = []*os.File{w}
if err := p.cmd.Start(); err != nil {
return err
}
w.Close()
errchan := make(chan error, 1)
go func() {
buf := make([]byte, 2)
r.Read(buf)
if string(buf) != "0\n" {
errStr, err := ioutil.ReadAll(r)
if err != nil {
errchan <- fmt.Errorf("Error reading exit status from userland proxy: %v", err)
return
}
errchan <- fmt.Errorf("Error starting userland proxy: %s", errStr)
return
}
errchan <- nil
}()
select {
case err := <-errchan:
return err
case <-time.After(16 * time.Second):
return fmt.Errorf("Timed out proxy starting the userland proxy")
}
}
func (p *proxyCommand) Stop() error {
if p.cmd.Process != nil {
if err := p.cmd.Process.Signal(os.Interrupt); err != nil {
return err
}
return p.cmd.Wait()
}
return nil
}
// dummyProxy just listen on some port, it is needed to prevent accidental
// port allocations on bound port, because without userland proxy we using
// iptables rules and not net.Listen
type dummyProxy struct {
listener io.Closer
addr net.Addr
}
func newDummyProxy(proto string, hostIP net.IP, hostPort int) userlandProxy {
switch proto {
case "tcp":
addr := &net.TCPAddr{IP: hostIP, Port: hostPort}
return &dummyProxy{addr: addr}
case "udp":
addr := &net.UDPAddr{IP: hostIP, Port: hostPort}
return &dummyProxy{addr: addr}
}
return nil
}
func (p *dummyProxy) Start() error {
switch addr := p.addr.(type) {
case *net.TCPAddr:
l, err := net.ListenTCP("tcp", addr)
if err != nil {
return err
}
p.listener = l
case *net.UDPAddr:
l, err := net.ListenUDP("udp", addr)
if err != nil {
return err
}
p.listener = l
default:
return fmt.Errorf("Unknown addr type: %T", p.addr)
}
return nil
}
func (p *dummyProxy) Stop() error {
if p.listener != nil {
return p.listener.Close()
}
return nil
}