VMware vSphere Integrated Containers provider (#206)

* Add Virtual Kubelet provider for VIC

Initial virtual kubelet provider for VMware VIC.  This provider currently
handles creating and starting of a pod VM via the VIC portlayer and persona
server.  Image store handling via the VIC persona server.  This provider
currently requires the feature/wolfpack branch of VIC.

* Added pod stop and delete.  Also added node capacity.

Added the ability to stop and delete pod VMs via VIC.  Also retrieve
node capacity information from the VCH.

* Cleanup and readme file

Some file clean up and added a Readme.md markdown file for the VIC
provider.

* Cleaned up errors, added function comments, moved operation code

1. Cleaned up error handling.  Set standard for creating errors.
2. Added method prototype comments for all interface functions.
3. Moved PodCreator, PodStarter, PodStopper, and PodDeleter to a new folder.

* Add mocking code and unit tests for podcache, podcreator, and podstarter

Used the unit test framework used in VIC to handle assertions in the provider's
unit test.  Mocking code generated using OSS project mockery, which is compatible
with the testify assertion framework.

* Vendored packages for the VIC provider

Requires feature/wolfpack branch of VIC and a few specific commit sha of
projects used within VIC.

* Implementation of POD Stopper and Deleter unit tests (#4)

* Updated files for initial PR
This commit is contained in:
Loc Nguyen
2018-06-04 15:41:32 -07:00
committed by Ria Bhatia
parent 98a111e8b7
commit 513cebe7b7
6296 changed files with 1123685 additions and 8 deletions

View File

@@ -0,0 +1,121 @@
// Copyright 2016 VMware, Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package exec2
import (
"net/url"
"time"
"github.com/google/uuid"
"github.com/vmware/vic/pkg/vsphere/vm"
)
type ID uuid.UUID
func GenerateID() ID {
return ID(uuid.New())
}
func ParseID(idStr string) (ID, error) {
result, err := uuid.Parse(idStr)
return ID(result), err
}
func (id ID) String() string {
return uuid.UUID(id).String()
}
// Struct that represents the internal port-layer representation of a container
// All data in this struct must be data that is either immutable
// or can be relied upon without having to query either the container guest
// or the underlying infrastructure. Some of this state will be updated by events
type container struct {
ConstantConfig
vm vm.VirtualMachine
runState RunState
config Config
mainProcess ProcessConfig // container main process
execdProcess []ProcessConfig
filesToCopy []FileToCopy // cache if copy while stopped
}
// config that will be applied to a container on commit
// Needs to be public as it will be shared by net, storage etc
type PendingCommit struct {
ConstantConfig
runState RunState
config Config
mainProcess ProcessConfig
filesToCopy []FileToCopy
}
// config state that cannot change for the lifetime of the container
type ConstantConfig struct {
ContainerID ID
Created time.Time
}
// variable container configuration state
type Config struct {
Name string
Limits ResourceLimits
}
// configuration state of a container process
type ProcessConfig struct {
ProcessID ID
WorkDir string
ExecPath string
ExecArgs string
}
func NewProcessConfig(workDir string, execPath string, execArgs string) ProcessConfig {
return ProcessConfig{ProcessID: GenerateID(), WorkDir: workDir, ExecArgs: execArgs, ExecPath: execPath}
}
type ProcessStatus int
const (
_ ProcessStatus = iota
Started
Exited
)
// runtime status of a container process
type ProcessRunState struct {
ProcessID ID
Status ProcessStatus
GuestPid int
ExitCode int
ExitMsg string
StartedAt time.Time
FinishedAt time.Time
}
type FileToCopy struct {
target url.URL
perms int16
data []byte
}
type ResourceLimits struct {
MemoryMb int
CPUMhz int
}

View File

@@ -0,0 +1,117 @@
// Copyright 2016 VMware, Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package exec2
// ContainerLifecycle represents operations concerned with the creation, modification
// and deletion of containers
type ContainerLifecycle interface {
// CreateContainer creates a new container representation and returns a Handle
// The Handle can be used to configure the container before its actually created
// Calling Commit on the Handle will create the container
CreateContainer(name string) (Handle, error)
// GetHandle allows for an existing container to be modified
// The Handle can be used to reconfigure the container
// Calling Commit on the Handle will apply the reconfiguration
// Commit will fail if another client committed a modification after GetHandle was called
GetHandle(cid ID) (Handle, error)
// CopyTo copies a file into the container represented by the handle
// If the container is stopped, the file will be copied in when it is next started
CopyTo(handle Handle, targetDir string, fname string, perms int16, data []byte) (Handle, error)
// SetEntryPoint sets the entry point for the container
// This is the executable, the lifecycle of which is tied to the container lifecycle
SetEntryPoint(handle Handle, workDir string, execPath string, execArgs string) (Handle, error)
// SetLimits sets resource limits on the container
// A value of -1 implies a default value, not unlimited
// New limits will be ignored if committed to a running container
SetLimits(handle Handle, memoryMb int, cpuMhz int) (Handle, error)
// SetRunState allows for the running state of a container to be modified
// Created is not a valid state and will return an error
SetRunState(handle Handle, runState RunState) (Handle, error)
// Commit applies changes made to the Handle to either a new or running container
// Commit will fail if another client committed a modification after GetHandle was called
// Commit blocks until all changes have been committed
Commit(handle Handle) (ID, error)
// DestroyContainer destroys an stopped container
// It is up to the caller to put the container in stopped state before calling Destroy
DestroyContainer(cid ID) error
}
type ProcessLifecycle interface {
// ExecProcess executes a process in the container
// The lifecycle of the process is independent of the container main process
// The ID returned is a uuid handle to the process
ExecProcess(cid ID, workDir string, execPath string, execArgs string) (ID, error)
// Send a signal to the process
// Specifying a process ID will signal an exec'd process. Specifying the container ID will signal the main process
Signal(processID ID, signal int) error
}
// ContainerQuery represents queries that can be made against a Container
type ContainerQuery interface {
// ListContainers lists all container IDs for a given state
// If forState is nil, all containers are returned
ListContainers(forState RunState) ([]ID, error)
// GetConfig returns container and process config
GetContainerConfig(cid ID) (ContainerConfig, error)
// GetState returns the current state of the container and its processes
// This call will return a snapshot of the most recent state for each entity
GetContainerState(cid ID) (ContainerState, error)
// CopyFrom copies file data out of a running container
// Returns an error if the container is not running
CopyFrom(cid ID, sourceDir string, fname string) ([]byte, error)
}
// RunState represents the running state of a container
type RunState int
const (
_ RunState = iota
Created
Running
Stopped
)
// ContainerConfig is a type representing the configuration of a container and its processes
type ContainerConfig struct {
ConstantConfig
Config
MainProcess ProcessConfig
ExecdProcs []ProcessConfig
}
// ContainerState is a type representing the runtime state of a container and its processes
type ContainerState struct {
Status RunState
MainProcess ProcessState
ExecdProcs []ProcessState
}
// ProcessState is the runtime state of a process in a container
type ProcessState struct {
ProcessRunState
}

View File

@@ -0,0 +1,128 @@
// Copyright 2016 VMware, Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package exec2
import (
"fmt"
"net/url"
)
// PortLayerVsphere is a WIP implementation of the execution.go interfaces
type PortLayerVsphere struct {
vmomiGateway VmomiGateway
handles HandleFactory
containers map[ID]*container
}
func (p *PortLayerVsphere) getContainer(handle Handle) *container {
return p.containers[handle.(*PendingCommit).ContainerID]
}
func (p *PortLayerVsphere) newHandle(cid ID) *PendingCommit {
return p.handles.createHandle(cid).(*PendingCommit)
}
func (p *PortLayerVsphere) Init(gateway VmomiGateway, factory HandleFactory) error {
p.handles = factory
p.vmomiGateway = gateway
p.containers = make(map[ID]*container)
return nil
}
func (p *PortLayerVsphere) CreateContainer(name string) (Handle, error) {
cid := GenerateID()
handle := p.newHandle(cid)
handle.config.Name = name
handle.runState = Created
return handle, nil
}
func (p *PortLayerVsphere) GetHandle(cid ID) (Handle, error) {
c := p.containers[cid]
if c == nil {
return nil, fmt.Errorf("Invalid container ID")
}
return p.handles.createHandle(c.ContainerID), nil
}
func (p *PortLayerVsphere) SetEntryPoint(handle Handle, workDir string, execPath string, execArgs string) (Handle, error) {
resolvedHandle := handle.(*PendingCommit)
resolvedHandle.mainProcess = NewProcessConfig(workDir, execPath, execArgs)
return p.handles.refreshHandle(handle), nil
}
func (p *PortLayerVsphere) Commit(handle Handle) (ID, error) {
var err error
c := p.getContainer(handle)
if c == nil {
c, err = p.createContainer(handle)
} else {
// if c.vm == nil {
// return "", fmt.Errorf("Cannot modify container with no VM")
// }
err = p.modifyContainer(c.runState, handle)
}
// Handle will be garbage collected
return c.ContainerID, err
}
func (p *PortLayerVsphere) CopyTo(handle Handle, targetDir string, fname string, perms int16, data []byte) (Handle, error) {
var result Handle
resolvedHandle := handle.(*PendingCommit)
u, err := url.Parse("file://" + targetDir + "/" + fname)
if err == nil {
fileToCopy := FileToCopy{target: *u, perms: perms, data: data}
resolvedHandle.filesToCopy = append(resolvedHandle.filesToCopy, fileToCopy)
result = p.handles.refreshHandle(handle)
}
return result, err
}
func (p *PortLayerVsphere) SetLimits(handle Handle, memoryMb int, cpuMhz int) (Handle, error) {
resolvedHandle := handle.(*PendingCommit)
resolvedHandle.config.Limits = ResourceLimits{MemoryMb: memoryMb, CPUMhz: cpuMhz}
return p.handles.refreshHandle(handle), nil
}
func (p *PortLayerVsphere) SetRunState(handle Handle, runState RunState) (Handle, error) {
resolvedHandle := handle.(*PendingCommit)
resolvedHandle.runState = runState
return p.handles.refreshHandle(handle), nil
}
func (p *PortLayerVsphere) DestroyContainer(cid ID) error {
c := p.containers[cid]
if c == nil {
return fmt.Errorf("Invalid container ID")
}
delete(p.containers, cid)
return nil
}
func (p *PortLayerVsphere) createContainer(handle Handle) (*container, error) {
resolvedHandle := handle.(*PendingCommit)
c := container{}
p.containers[resolvedHandle.ContainerID] = &c
c.ContainerID = resolvedHandle.ContainerID
c.runState = resolvedHandle.runState
// followed by other transfer of state from pending to container
// fmt.Printf("Creating container for %v\n", pending)
return &c, nil
}
func (p *PortLayerVsphere) modifyContainer(runState RunState, handle Handle) error {
// fmt.Printf("Modifying container for %v\n", pending)
return nil
}

View File

@@ -0,0 +1,37 @@
// Copyright 2016 VMware, Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package exec2
/* A Handle should be completely opaque */
type Handle interface{}
type HandleFactory interface {
createHandle(cID ID) Handle
refreshHandle(oldHandle Handle) Handle
}
type BasicHandleFactory struct {
}
func (h *BasicHandleFactory) createHandle(cid ID) Handle {
newPc := &PendingCommit{}
newPc.ContainerID = cid
return newPc
}
// Basic handle resolver just passes back the handle passed in
func (h *BasicHandleFactory) refreshHandle(oldHandle Handle) Handle {
return oldHandle
}

View File

@@ -0,0 +1,126 @@
// Copyright 2016 VMware, Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package remote
import (
"encoding/gob"
"net/rpc"
"github.com/google/uuid"
"github.com/vmware/vic/lib/portlayer/exec2"
)
const serverAddress string = "localhost"
type PortLayerRPCClient struct {
client *rpc.Client
}
func (p *PortLayerRPCClient) Connect() error {
// Ignore Init args on the client - that is the server's responsibility
var err error
gob.Register(uuid.New())
p.client, err = rpc.DialHTTP("tcp", serverAddress+":1234")
return err
}
type CreateArgs struct {
Name string
}
func (p *PortLayerRPCClient) CreateContainer(name string) (exec2.Handle, error) {
args := &CreateArgs{Name: name}
var reply exec2.Handle
err := p.client.Call("PortLayerRPCServer.CreateContainer", args, &reply)
return reply, err
}
func (p *PortLayerRPCClient) GetHandle(cid exec2.ID) (exec2.Handle, error) {
var reply exec2.Handle
err := p.client.Call("PortLayerRPCServer.GetHandle", cid, &reply)
return reply, err
}
type CopyToArgs struct {
Handle exec2.Handle
TargetDir string
Fname string
Perms int16
Data []byte
}
func (p *PortLayerRPCClient) CopyTo(handle exec2.Handle, targetDir string, fname string, perms int16, data []byte) (exec2.Handle, error) {
args := &CopyToArgs{Handle: handle, TargetDir: targetDir, Fname: fname, Perms: perms, Data: data}
var reply exec2.Handle
err := p.client.Call("PortLayerRPCServer.CopyTo", args, &reply)
return reply, err
}
type SetEntryPointArgs struct {
Handle exec2.Handle
WorkDir string
ExecPath string
Args string
}
func (p *PortLayerRPCClient) SetEntryPoint(handle exec2.Handle, workDir string, execPath string, args string) (exec2.Handle, error) {
epArgs := &SetEntryPointArgs{Handle: handle, WorkDir: workDir, ExecPath: execPath, Args: args}
var reply exec2.Handle
err := p.client.Call("PortLayerRPCServer.SetEntryPoint", epArgs, &reply)
return reply, err
}
type SetLimitsArgs struct {
Handle exec2.Handle
MemoryMb int
CPUMhz int
}
func (p *PortLayerRPCClient) SetLimits(handle exec2.Handle, memoryMb int, cpuMhz int) (exec2.Handle, error) {
args := &SetLimitsArgs{Handle: handle, MemoryMb: memoryMb, CPUMhz: cpuMhz}
var reply exec2.Handle
err := p.client.Call("PortLayerRPCServer.SetLimits", args, &reply)
return reply, err
}
type SetRunStateArgs struct {
Handle exec2.Handle
RunState exec2.RunState
}
func (p *PortLayerRPCClient) SetRunState(handle exec2.Handle, runState exec2.RunState) (exec2.Handle, error) {
args := &SetRunStateArgs{Handle: handle, RunState: runState}
var reply exec2.Handle
err := p.client.Call("PortLayerRPCServer.SetRunState", args, &reply)
return reply, err
}
type CommitArgs struct {
Handle exec2.Handle
}
func (p *PortLayerRPCClient) Commit(handle exec2.Handle) (exec2.ID, error) {
args := &CommitArgs{Handle: handle}
var reply exec2.ID
err := p.client.Call("PortLayerRPCServer.Commit", args, &reply)
return reply, err
}
func (p *PortLayerRPCClient) DestroyContainer(cid exec2.ID) error {
/* Ignore the reply */
var reply exec2.ID
return p.client.Call("PortLayerRPCServer.DestroyContainer", cid, &reply)
}

View File

@@ -0,0 +1,133 @@
// Copyright 2016-2017 VMware, Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"encoding/gob"
"fmt"
"log"
"net"
"net/http"
"net/rpc"
"github.com/google/uuid"
"github.com/vmware/vic/lib/portlayer/exec2"
"github.com/vmware/vic/lib/portlayer/exec2/remote"
)
type PortLayerRPCServer struct {
}
// A Handle can be anything, so this takes advantage of that by creating a sparse handle
// to send to the client and using that sparse handle as a key in a hashtable which points to
// rich handles created by the HandleFactory
type SparseHandle exec2.Handle
var lcTarget exec2.ContainerLifecycle
//var lcQuery exec2.ContainerQuery
var handles map[SparseHandle]exec2.Handle
func init() {
pl := &exec2.PortLayerVsphere{}
pl.Init(nil, &exec2.BasicHandleFactory{})
lcTarget = pl
//lcQuery = lcTarget
handles = make(map[SparseHandle]exec2.Handle)
}
func main() {
gob.Register(uuid.New())
rpcServer := new(PortLayerRPCServer)
rpc.Register(rpcServer)
rpc.HandleHTTP()
// #nosec: Binds to all network interfaces
l, e := net.Listen("tcp", ":1234")
if e != nil {
log.Fatal("listen error:", e)
}
fmt.Println("Server listening")
http.Serve(l, nil)
}
// A sparse handle is simply a random string
func newSparseHandle() SparseHandle {
return SparseHandle(uuid.New())
}
func createSparseHandle(handle exec2.Handle) SparseHandle {
key := newSparseHandle()
handles[key] = handle
return key
}
func resolveSparseHandle(handle SparseHandle) exec2.Handle {
return handles[handle]
}
func refreshHandle(result *exec2.Handle, oldHandle SparseHandle, newHandle exec2.Handle, err error) error {
*result = createSparseHandle(newHandle)
delete(handles, oldHandle)
return err
}
func (*PortLayerRPCServer) CreateContainer(args remote.CreateArgs, result *exec2.Handle) error {
handle, err := lcTarget.CreateContainer(args.Name)
*result = createSparseHandle(handle)
return err
}
func (*PortLayerRPCServer) GetHandle(cid exec2.ID, result *exec2.Handle) error {
handle, err := lcTarget.GetHandle(cid)
*result = createSparseHandle(handle)
return err
}
func (*PortLayerRPCServer) CopyTo(args remote.CopyToArgs, result *exec2.Handle) error {
handle := resolveSparseHandle(args.Handle)
newHandle, err := lcTarget.CopyTo(handle, args.TargetDir, args.Fname, args.Perms, args.Data)
return refreshHandle(result, handle, newHandle, err)
}
func (*PortLayerRPCServer) SetEntryPoint(args remote.SetEntryPointArgs, result *exec2.Handle) error {
handle := resolveSparseHandle(args.Handle)
newHandle, err := lcTarget.SetEntryPoint(handle, args.WorkDir, args.ExecPath, args.Args)
return refreshHandle(result, handle, newHandle, err)
}
func (*PortLayerRPCServer) SetLimits(args remote.SetLimitsArgs, result *exec2.Handle) error {
handle := resolveSparseHandle(args.Handle)
newHandle, err := lcTarget.SetLimits(handle, args.MemoryMb, args.CPUMhz)
return refreshHandle(result, handle, newHandle, err)
}
func (*PortLayerRPCServer) SetRunState(args remote.SetRunStateArgs, result *exec2.Handle) error {
handle := resolveSparseHandle(args.Handle)
newHandle, err := lcTarget.SetRunState(handle, args.RunState)
return refreshHandle(result, handle, newHandle, err)
}
func (*PortLayerRPCServer) Commit(args remote.CommitArgs, result *exec2.ID) error {
cid, err := lcTarget.Commit(resolveSparseHandle(args.Handle))
*result = cid
return err
}
func (*PortLayerRPCServer) DestroyContainer(cid exec2.ID, result *exec2.ID) error {
err := lcTarget.DestroyContainer(cid)
*result = cid
return err
}

View File

@@ -0,0 +1,19 @@
// Copyright 2016 VMware, Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package exec2
// VmomiGateway represents an interface to a pre-authenticated Vmomi API
type VmomiGateway interface {
}