VMware vSphere Integrated Containers provider (#206)

* Add Virtual Kubelet provider for VIC

Initial virtual kubelet provider for VMware VIC.  This provider currently
handles creating and starting of a pod VM via the VIC portlayer and persona
server.  Image store handling via the VIC persona server.  This provider
currently requires the feature/wolfpack branch of VIC.

* Added pod stop and delete.  Also added node capacity.

Added the ability to stop and delete pod VMs via VIC.  Also retrieve
node capacity information from the VCH.

* Cleanup and readme file

Some file clean up and added a Readme.md markdown file for the VIC
provider.

* Cleaned up errors, added function comments, moved operation code

1. Cleaned up error handling.  Set standard for creating errors.
2. Added method prototype comments for all interface functions.
3. Moved PodCreator, PodStarter, PodStopper, and PodDeleter to a new folder.

* Add mocking code and unit tests for podcache, podcreator, and podstarter

Used the unit test framework used in VIC to handle assertions in the provider's
unit test.  Mocking code generated using OSS project mockery, which is compatible
with the testify assertion framework.

* Vendored packages for the VIC provider

Requires feature/wolfpack branch of VIC and a few specific commit sha of
projects used within VIC.

* Implementation of POD Stopper and Deleter unit tests (#4)

* Updated files for initial PR
This commit is contained in:
Loc Nguyen
2018-06-04 15:41:32 -07:00
committed by Ria Bhatia
parent 98a111e8b7
commit 513cebe7b7
6296 changed files with 1123685 additions and 8 deletions

View File

@@ -0,0 +1,270 @@
package portallocator
import (
"bufio"
"errors"
"fmt"
"net"
"os"
"sync"
)
const (
// DefaultPortRangeStart indicates the first port in port range
DefaultPortRangeStart = 49153
// DefaultPortRangeEnd indicates the last port in port range
DefaultPortRangeEnd = 65535
)
type ipMapping map[string]protoMap
var (
// ErrAllPortsAllocated is returned when no more ports are available
ErrAllPortsAllocated = errors.New("all ports are allocated")
// ErrUnknownProtocol is returned when an unknown protocol was specified
ErrUnknownProtocol = errors.New("unknown protocol")
defaultIP = net.ParseIP("0.0.0.0")
once sync.Once
instance *PortAllocator
createInstance = func() { instance = newInstance() }
)
// ErrPortAlreadyAllocated is the returned error information when a requested port is already being used
type ErrPortAlreadyAllocated struct {
ip string
port int
}
func newErrPortAlreadyAllocated(ip string, port int) ErrPortAlreadyAllocated {
return ErrPortAlreadyAllocated{
ip: ip,
port: port,
}
}
// IP returns the address to which the used port is associated
func (e ErrPortAlreadyAllocated) IP() string {
return e.ip
}
// Port returns the value of the already used port
func (e ErrPortAlreadyAllocated) Port() int {
return e.port
}
// IPPort returns the address and the port in the form ip:port
func (e ErrPortAlreadyAllocated) IPPort() string {
return fmt.Sprintf("%s:%d", e.ip, e.port)
}
// Error is the implementation of error.Error interface
func (e ErrPortAlreadyAllocated) Error() string {
return fmt.Sprintf("Bind for %s:%d failed: port is already allocated", e.ip, e.port)
}
type (
// PortAllocator manages the transport ports database
PortAllocator struct {
mutex sync.Mutex
ipMap ipMapping
Begin int
End int
}
portRange struct {
begin int
end int
last int
}
portMap struct {
p map[int]struct{}
defaultRange string
portRanges map[string]*portRange
}
protoMap map[string]*portMap
)
// Get returns the default instance of PortAllocator
func Get() *PortAllocator {
// Port Allocator is a singleton
// Note: Long term solution will be each PortAllocator will have access to
// the OS so that it can have up to date view of the OS port allocation.
// When this happens singleton behavior will be removed. Clients do not
// need to worry about this, they will not see a change in behavior.
once.Do(createInstance)
return instance
}
func newInstance() *PortAllocator {
start, end, err := getDynamicPortRange()
if err != nil {
start, end = DefaultPortRangeStart, DefaultPortRangeEnd
}
return &PortAllocator{
ipMap: ipMapping{},
Begin: start,
End: end,
}
}
func getDynamicPortRange() (start int, end int, err error) {
const portRangeKernelParam = "/proc/sys/net/ipv4/ip_local_port_range"
portRangeFallback := fmt.Sprintf("using fallback port range %d-%d", DefaultPortRangeStart, DefaultPortRangeEnd)
file, err := os.Open(portRangeKernelParam)
if err != nil {
return 0, 0, fmt.Errorf("port allocator - %s due to error: %v", portRangeFallback, err)
}
defer file.Close()
n, err := fmt.Fscanf(bufio.NewReader(file), "%d\t%d", &start, &end)
if n != 2 || err != nil {
if err == nil {
err = fmt.Errorf("unexpected count of parsed numbers (%d)", n)
}
return 0, 0, fmt.Errorf("port allocator - failed to parse system ephemeral port range from %s - %s: %v", portRangeKernelParam, portRangeFallback, err)
}
return start, end, nil
}
// RequestPort requests new port from global ports pool for specified ip and proto.
// If port is 0 it returns first free port. Otherwise it checks port availability
// in proto's pool and returns that port or error if port is already busy.
func (p *PortAllocator) RequestPort(ip net.IP, proto string, port int) (int, error) {
return p.RequestPortInRange(ip, proto, port, port)
}
// RequestPortInRange requests new port from global ports pool for specified ip and proto.
// If portStart and portEnd are 0 it returns the first free port in the default ephemeral range.
// If portStart != portEnd it returns the first free port in the requested range.
// Otherwise (portStart == portEnd) it checks port availability in the requested proto's port-pool
// and returns that port or error if port is already busy.
func (p *PortAllocator) RequestPortInRange(ip net.IP, proto string, portStart, portEnd int) (int, error) {
p.mutex.Lock()
defer p.mutex.Unlock()
if proto != "tcp" && proto != "udp" {
return 0, ErrUnknownProtocol
}
if ip == nil {
ip = defaultIP
}
ipstr := ip.String()
protomap, ok := p.ipMap[ipstr]
if !ok {
protomap = protoMap{
"tcp": p.newPortMap(),
"udp": p.newPortMap(),
}
p.ipMap[ipstr] = protomap
}
mapping := protomap[proto]
if portStart > 0 && portStart == portEnd {
if _, ok := mapping.p[portStart]; !ok {
mapping.p[portStart] = struct{}{}
return portStart, nil
}
return 0, newErrPortAlreadyAllocated(ipstr, portStart)
}
port, err := mapping.findPort(portStart, portEnd)
if err != nil {
return 0, err
}
return port, nil
}
// ReleasePort releases port from global ports pool for specified ip and proto.
func (p *PortAllocator) ReleasePort(ip net.IP, proto string, port int) error {
p.mutex.Lock()
defer p.mutex.Unlock()
if ip == nil {
ip = defaultIP
}
protomap, ok := p.ipMap[ip.String()]
if !ok {
return nil
}
delete(protomap[proto].p, port)
return nil
}
func (p *PortAllocator) newPortMap() *portMap {
defaultKey := getRangeKey(p.Begin, p.End)
pm := &portMap{
p: map[int]struct{}{},
defaultRange: defaultKey,
portRanges: map[string]*portRange{
defaultKey: newPortRange(p.Begin, p.End),
},
}
return pm
}
// ReleaseAll releases all ports for all ips.
func (p *PortAllocator) ReleaseAll() error {
p.mutex.Lock()
p.ipMap = ipMapping{}
p.mutex.Unlock()
return nil
}
func getRangeKey(portStart, portEnd int) string {
return fmt.Sprintf("%d-%d", portStart, portEnd)
}
func newPortRange(portStart, portEnd int) *portRange {
return &portRange{
begin: portStart,
end: portEnd,
last: portEnd,
}
}
func (pm *portMap) getPortRange(portStart, portEnd int) (*portRange, error) {
var key string
if portStart == 0 && portEnd == 0 {
key = pm.defaultRange
} else {
key = getRangeKey(portStart, portEnd)
if portStart == portEnd ||
portStart == 0 || portEnd == 0 ||
portEnd < portStart {
return nil, fmt.Errorf("invalid port range: %s", key)
}
}
// Return existing port range, if already known.
if pr, exists := pm.portRanges[key]; exists {
return pr, nil
}
// Otherwise create a new port range.
pr := newPortRange(portStart, portEnd)
pm.portRanges[key] = pr
return pr, nil
}
func (pm *portMap) findPort(portStart, portEnd int) (int, error) {
pr, err := pm.getPortRange(portStart, portEnd)
if err != nil {
return 0, err
}
port := pr.last
for i := 0; i <= pr.end-pr.begin; i++ {
port++
if port > pr.end {
port = pr.begin
}
if _, ok := pm.p[port]; !ok {
pm.p[port] = struct{}{}
pr.last = port
return port, nil
}
}
return 0, ErrAllPortsAllocated
}

View File

@@ -0,0 +1,320 @@
package portallocator
import (
"net"
"testing"
_ "github.com/docker/libnetwork/testutils"
)
func resetPortAllocator() {
instance = newInstance()
}
func TestRequestNewPort(t *testing.T) {
p := Get()
defer resetPortAllocator()
port, err := p.RequestPort(defaultIP, "tcp", 0)
if err != nil {
t.Fatal(err)
}
if expected := p.Begin; port != expected {
t.Fatalf("Expected port %d got %d", expected, port)
}
}
func TestRequestSpecificPort(t *testing.T) {
p := Get()
defer resetPortAllocator()
port, err := p.RequestPort(defaultIP, "tcp", 5000)
if err != nil {
t.Fatal(err)
}
if port != 5000 {
t.Fatalf("Expected port 5000 got %d", port)
}
}
func TestReleasePort(t *testing.T) {
p := Get()
port, err := p.RequestPort(defaultIP, "tcp", 5000)
if err != nil {
t.Fatal(err)
}
if port != 5000 {
t.Fatalf("Expected port 5000 got %d", port)
}
if err := p.ReleasePort(defaultIP, "tcp", 5000); err != nil {
t.Fatal(err)
}
}
func TestReuseReleasedPort(t *testing.T) {
p := Get()
defer resetPortAllocator()
port, err := p.RequestPort(defaultIP, "tcp", 5000)
if err != nil {
t.Fatal(err)
}
if port != 5000 {
t.Fatalf("Expected port 5000 got %d", port)
}
if err := p.ReleasePort(defaultIP, "tcp", 5000); err != nil {
t.Fatal(err)
}
port, err = p.RequestPort(defaultIP, "tcp", 5000)
if err != nil {
t.Fatal(err)
}
}
func TestReleaseUnreadledPort(t *testing.T) {
p := Get()
defer resetPortAllocator()
port, err := p.RequestPort(defaultIP, "tcp", 5000)
if err != nil {
t.Fatal(err)
}
if port != 5000 {
t.Fatalf("Expected port 5000 got %d", port)
}
port, err = p.RequestPort(defaultIP, "tcp", 5000)
switch err.(type) {
case ErrPortAlreadyAllocated:
default:
t.Fatalf("Expected port allocation error got %s", err)
}
}
func TestUnknowProtocol(t *testing.T) {
if _, err := Get().RequestPort(defaultIP, "tcpp", 0); err != ErrUnknownProtocol {
t.Fatalf("Expected error %s got %s", ErrUnknownProtocol, err)
}
}
func TestAllocateAllPorts(t *testing.T) {
p := Get()
defer resetPortAllocator()
for i := 0; i <= p.End-p.Begin; i++ {
port, err := p.RequestPort(defaultIP, "tcp", 0)
if err != nil {
t.Fatal(err)
}
if expected := p.Begin + i; port != expected {
t.Fatalf("Expected port %d got %d", expected, port)
}
}
if _, err := p.RequestPort(defaultIP, "tcp", 0); err != ErrAllPortsAllocated {
t.Fatalf("Expected error %s got %s", ErrAllPortsAllocated, err)
}
_, err := p.RequestPort(defaultIP, "udp", 0)
if err != nil {
t.Fatal(err)
}
// release a port in the middle and ensure we get another tcp port
port := p.Begin + 5
if err := p.ReleasePort(defaultIP, "tcp", port); err != nil {
t.Fatal(err)
}
newPort, err := p.RequestPort(defaultIP, "tcp", 0)
if err != nil {
t.Fatal(err)
}
if newPort != port {
t.Fatalf("Expected port %d got %d", port, newPort)
}
// now pm.last == newPort, release it so that it's the only free port of
// the range, and ensure we get it back
if err := p.ReleasePort(defaultIP, "tcp", newPort); err != nil {
t.Fatal(err)
}
port, err = p.RequestPort(defaultIP, "tcp", 0)
if err != nil {
t.Fatal(err)
}
if newPort != port {
t.Fatalf("Expected port %d got %d", newPort, port)
}
}
func BenchmarkAllocatePorts(b *testing.B) {
p := Get()
defer resetPortAllocator()
for i := 0; i < b.N; i++ {
for i := 0; i <= p.End-p.Begin; i++ {
port, err := p.RequestPort(defaultIP, "tcp", 0)
if err != nil {
b.Fatal(err)
}
if expected := p.Begin + i; port != expected {
b.Fatalf("Expected port %d got %d", expected, port)
}
}
p.ReleaseAll()
}
}
func TestPortAllocation(t *testing.T) {
p := Get()
defer resetPortAllocator()
ip := net.ParseIP("192.168.0.1")
ip2 := net.ParseIP("192.168.0.2")
if port, err := p.RequestPort(ip, "tcp", 80); err != nil {
t.Fatal(err)
} else if port != 80 {
t.Fatalf("Acquire(80) should return 80, not %d", port)
}
port, err := p.RequestPort(ip, "tcp", 0)
if err != nil {
t.Fatal(err)
}
if port <= 0 {
t.Fatalf("Acquire(0) should return a non-zero port")
}
if _, err := p.RequestPort(ip, "tcp", port); err == nil {
t.Fatalf("Acquiring a port already in use should return an error")
}
if newPort, err := p.RequestPort(ip, "tcp", 0); err != nil {
t.Fatal(err)
} else if newPort == port {
t.Fatalf("Acquire(0) allocated the same port twice: %d", port)
}
if _, err := p.RequestPort(ip, "tcp", 80); err == nil {
t.Fatalf("Acquiring a port already in use should return an error")
}
if _, err := p.RequestPort(ip2, "tcp", 80); err != nil {
t.Fatalf("It should be possible to allocate the same port on a different interface")
}
if _, err := p.RequestPort(ip2, "tcp", 80); err == nil {
t.Fatalf("Acquiring a port already in use should return an error")
}
if err := p.ReleasePort(ip, "tcp", 80); err != nil {
t.Fatal(err)
}
if _, err := p.RequestPort(ip, "tcp", 80); err != nil {
t.Fatal(err)
}
port, err = p.RequestPort(ip, "tcp", 0)
if err != nil {
t.Fatal(err)
}
port2, err := p.RequestPort(ip, "tcp", port+1)
if err != nil {
t.Fatal(err)
}
port3, err := p.RequestPort(ip, "tcp", 0)
if err != nil {
t.Fatal(err)
}
if port3 == port2 {
t.Fatal("Requesting a dynamic port should never allocate a used port")
}
}
func TestPortAllocationWithCustomRange(t *testing.T) {
p := Get()
defer resetPortAllocator()
start, end := 8081, 8082
specificPort := 8000
//get an ephemeral port.
port1, err := p.RequestPortInRange(defaultIP, "tcp", 0, 0)
if err != nil {
t.Fatal(err)
}
//request invalid ranges
if _, err := p.RequestPortInRange(defaultIP, "tcp", 0, end); err == nil {
t.Fatalf("Expected error for invalid range %d-%d", 0, end)
}
if _, err := p.RequestPortInRange(defaultIP, "tcp", start, 0); err == nil {
t.Fatalf("Expected error for invalid range %d-%d", 0, end)
}
if _, err := p.RequestPortInRange(defaultIP, "tcp", 8081, 8080); err == nil {
t.Fatalf("Expected error for invalid range %d-%d", 0, end)
}
//request a single port
port, err := p.RequestPortInRange(defaultIP, "tcp", specificPort, specificPort)
if err != nil {
t.Fatal(err)
}
if port != specificPort {
t.Fatalf("Expected port %d, got %d", specificPort, port)
}
//get a port from the range
port2, err := p.RequestPortInRange(defaultIP, "tcp", start, end)
if err != nil {
t.Fatal(err)
}
if port2 < start || port2 > end {
t.Fatalf("Expected a port between %d and %d, got %d", start, end, port2)
}
//get another ephemeral port (should be > port1)
port3, err := p.RequestPortInRange(defaultIP, "tcp", 0, 0)
if err != nil {
t.Fatal(err)
}
if port3 < port1 {
t.Fatalf("Expected new port > %d in the ephemeral range, got %d", port1, port3)
}
//get another (and in this case the only other) port from the range
port4, err := p.RequestPortInRange(defaultIP, "tcp", start, end)
if err != nil {
t.Fatal(err)
}
if port4 < start || port4 > end {
t.Fatalf("Expected a port between %d and %d, got %d", start, end, port4)
}
if port4 == port2 {
t.Fatal("Allocated the same port from a custom range")
}
//request 3rd port from the range of 2
if _, err := p.RequestPortInRange(defaultIP, "tcp", start, end); err != ErrAllPortsAllocated {
t.Fatalf("Expected error %s got %s", ErrAllPortsAllocated, err)
}
}
func TestNoDuplicateBPR(t *testing.T) {
p := Get()
defer resetPortAllocator()
if port, err := p.RequestPort(defaultIP, "tcp", p.Begin); err != nil {
t.Fatal(err)
} else if port != p.Begin {
t.Fatalf("Expected port %d got %d", p.Begin, port)
}
if port, err := p.RequestPort(defaultIP, "tcp", 0); err != nil {
t.Fatal(err)
} else if port == p.Begin {
t.Fatalf("Acquire(0) allocated the same port twice: %d", port)
}
}