d8dfun/virtual-kubelet
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

VMware vSphere Integrated Containers provider (#206)

Browse Source 
* Add Virtual Kubelet provider for VIC

Initial virtual kubelet provider for VMware VIC.  This provider currently
handles creating and starting of a pod VM via the VIC portlayer and persona
server.  Image store handling via the VIC persona server.  This provider
currently requires the feature/wolfpack branch of VIC.

* Added pod stop and delete.  Also added node capacity.

Added the ability to stop and delete pod VMs via VIC.  Also retrieve
node capacity information from the VCH.

* Cleanup and readme file

Some file clean up and added a Readme.md markdown file for the VIC
provider.

* Cleaned up errors, added function comments, moved operation code

1. Cleaned up error handling.  Set standard for creating errors.
2. Added method prototype comments for all interface functions.
3. Moved PodCreator, PodStarter, PodStopper, and PodDeleter to a new folder.

* Add mocking code and unit tests for podcache, podcreator, and podstarter

Used the unit test framework used in VIC to handle assertions in the provider's
unit test.  Mocking code generated using OSS project mockery, which is compatible
with the testify assertion framework.

* Vendored packages for the VIC provider

Requires feature/wolfpack branch of VIC and a few specific commit sha of
projects used within VIC.

* Implementation of POD Stopper and Deleter unit tests (#4)

* Updated files for initial PR
This commit is contained in:
Loc Nguyen
2018-06-04 15:41:32 -07:00
committed by Ria Bhatia
parent 98a111e8b7
commit 513cebe7b7
6296 changed files with 1123685 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

446
vendor/github.com/vmware/vic/pkg/vsphere/performance/vm_collector.go generated vendored Normal file
View File

@@ -0,0 +1,446 @@
// Copyright 2017 VMware, Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package performance
import (
"context"
"strconv"
"sync"
"time"
"github.com/vmware/govmomi/performance"
"github.com/vmware/govmomi/vim25/types"
"github.com/vmware/vic/pkg/trace"
"github.com/vmware/vic/pkg/vsphere/session"
)
const (
// number of samples per collection
sampleSize = int32(2)
// number of seconds between sample collection
sampleInterval = int32(20)
// vSphere recommomends a maxiumum of 50 entities per query
maxEntityQuery = 50
)
// CPUUsage provides individual CPU metrics
type CPUUsage struct {
// processor id (0,1,2)
ID int
// MhzUsage is the MhZ consumed by a specific processor
MhzUsage int64
}
// CPUMetrics encapsulates available vm CPU metrics
type CPUMetrics struct {
// CPUs are the individual CPU metrics
CPUs []CPUUsage
// Usage is the percentage of total vm CPU usage
Usage float32
}
// MemoryMetrics encapsulates available vm memory metrics
type MemoryMetrics struct {
// Consumed memory of vm in bytes
Consumed int64
// Active memory of vm in bytes
Active int64
// Provisioned memory of vm in bytes
Provisioned int64
}
// NetworkUsage provides detailed network stats
type NetworkUsage struct {
Bytes uint64 // total bytes
Kbps int64 // KiloBytesPerSecond
Packets int64 // total packet count
Errors int64 // NOT CURRENTLY IMPLEMENTED
Dropped int64 // total dropped packet count
}
// Network provides metrics for individual network devices
type Network struct {
Name string
Rx NetworkUsage
Tx NetworkUsage
}
// DiskUsage provides detailed disk stats
type DiskUsage struct {
Bytes uint64 // total bytes for interval
Kbps int64 // KiloBytesPerSecond for interval
Op uint64 // Operation count for interval
Ops int64 // Operations per second
}
// VirtualDisk provides metrics for individual disks
type VirtualDisk struct {
Name string
Write DiskUsage
Read DiskUsage
}
// VMMetrics encapsulates the available metrics
type VMMetrics struct {
CPU CPUMetrics
Memory MemoryMetrics
Networks []Network
Disks []VirtualDisk
SampleTime time.Time
// interval of collection in seconds
Interval int32
}
// VMCollector is the VM metrics collector
type VMCollector struct {
perfMgr *performance.Manager
session *session.Session
timer *time.Ticker
stopper chan struct{}
// subscribers to streaming
mu sync.RWMutex
subs map[types.ManagedObjectReference]*vmSubscription
}
// newVMCollector will instantiate a new collector responsible for
// gathering VM metrics
func NewVMCollector(session *session.Session) *VMCollector {
return &VMCollector{
subs: make(map[types.ManagedObjectReference]*vmSubscription),
perfMgr: performance.NewManager(session.Vim25()),
session: session,
}
}
// Start will begin the collection polling process
func (vmc *VMCollector) Start() {
// create timer with sampleInterval alignment
vmc.timer = time.NewTicker(time.Duration(int64(sampleInterval)) * time.Second)
//create the stopper channel
vmc.stopper = make(chan struct{})
// loop on the channel
go func() {
ctx := context.Background()
collectorOp := trace.NewOperation(ctx, "VM metrics collector")
go vmc.collect(collectorOp)
for {
select {
case <-vmc.timer.C:
// collect metrics for current subscribers
vmc.collect(collectorOp)
case <-vmc.stopper:
// Ticker has been stopped, exit routine
collectorOp.Debugf("VM metrics collector complete")
return
}
}
}()
}
// Stop will stop the collection polling process
func (vmc *VMCollector) Stop() {
vmc.timer.Stop()
close(vmc.stopper)
}
// collect will query vSphere for VM metrics and return to the subscribers
func (vmc *VMCollector) collect(op trace.Operation) {
// gather the chunked morefs
vmReferences := vmc.subscriberReferences(op)
// iterate over the chunked references and sample vSphere
for i := range vmReferences {
// create a new operation so we can effectively log and measure the sample
sop := trace.NewOperation(op.Context, "sample operation")
sop.Debugf("parentOp[%s] sample(%d/%d) with %d morefs", op.ID(), i+1, len(vmReferences), len(vmReferences[i]))
go vmc.sample(sop, vmReferences[i])
}
}
// subscriberReferences will return a two dimensional array of VM managed object references. The
// references are chunked based on the maxEntityQuery limit.
func (vmc *VMCollector) subscriberReferences(op trace.Operation) [][]types.ManagedObjectReference {
var mos []types.ManagedObjectReference
var chunked [][]types.ManagedObjectReference
vmc.mu.Lock()
defer vmc.mu.Unlock()
op.Debugf("begin subscriberReferences: %d", len(vmc.subs))
// populate a two dimensional array chunked by the maxEntityQueryLimit
for mo := range vmc.subs {
mos = append(mos, mo)
if len(mos) == maxEntityQuery {
chunked = append(chunked, mos)
mos = make([]types.ManagedObjectReference, 0, maxEntityQuery)
}
}
// the initial loop will potentially miss the "remainder" morefs
if len(mos) > 0 && len(mos) < maxEntityQuery {
chunked = append(chunked, mos)
}
op.Debugf("end subscriberReferences: %d", len(chunked))
return chunked
}
// sample will query the vSphere performanceManager and publish the gather metrics
func (vmc *VMCollector) sample(op trace.Operation, mos []types.ManagedObjectReference) {
op.Debugf("begin sample for %d morefs", len(mos))
defer op.Debugf("end sample for %d morefs", len(mos))
// vSphere counters we are currently interested in monitoring
counters := []string{"cpu.usagemhz.average", "mem.active.average",
"virtualDisk.write.average", "virtualDisk.read.average",
"virtualDisk.numberReadAveraged.average", "virtualDisk.numberWriteAveraged.average",
"net.bytesRx.average", "net.bytesTx.average",
"net.droppedRx.summation", "net.droppedTx.summation",
"net.packetsRx.summation", "net.packetsTx.summation"}
// create the spec
spec := types.PerfQuerySpec{
Format: string(types.PerfFormatNormal),
MaxSample: sampleSize,
IntervalId: sampleInterval,
}
// retrieve sample based on counter names
sample, err := vmc.perfMgr.SampleByName(op.Context, spec, counters, mos)
if err != nil {
op.Errorf("unable to get metric sample: %s", err)
return
}
// convert to metrics
result, err := vmc.perfMgr.ToMetricSeries(op.Context, sample)
if err != nil {
op.Errorf("unable to convert metric sample to metric series: %s", err)
return
}
// iterate over results, convert to vic metrics and publish to subscribers
for i := range result {
met := result[i]
sub, exists := vmc.subs[met.Entity]
if !exists {
// the subscription is no longer valid go to the next result
continue
}
// convert the sample to a metric and publish
for s := range met.SampleInfo {
metric := &VMMetrics{
CPU: CPUMetrics{
CPUs: []CPUUsage{},
},
Memory: MemoryMetrics{},
SampleTime: met.SampleInfo[s].Timestamp,
Interval: sampleInterval,
}
// the series will have values for each sample
for _, v := range met.Value {
// skip the aggregate metric (empty string) and any negative values
if v.Instance == "" && v.Name != "mem.active.average" || v.Value[s] < 0 {
continue
}
switch v.Name {
case "cpu.usagemhz.average":
// convert cpu instance to int
cpu, err := instanceID(op, sub.ID(), v.Instance)
if err != nil {
// skipping this instance
continue
}
// specific vCPU metric
vcpu := CPUUsage{
ID: cpu,
MhzUsage: v.Value[s],
}
metric.CPU.CPUs = append(metric.CPU.CPUs, vcpu)
case "mem.active.average":
metric.Memory.Active = v.Value[s]
// DISK
case "virtualDisk.read.average":
disk := findDisk(metric, v.Instance)
// perfManager returns kbps -- convert to sum of bytes
sum := summation(v.Value[s]) * 1024
metric.Disks[disk].Read.Bytes = sum
metric.Disks[disk].Read.Kbps = v.Value[s]
case "virtualDisk.write.average":
disk := findDisk(metric, v.Instance)
// perfManager returns kbps -- convert to sum of bytes
sum := summation(v.Value[s]) * 1024
metric.Disks[disk].Write.Bytes = sum
metric.Disks[disk].Write.Kbps = v.Value[s]
case "virtualDisk.numberReadAveraged.average":
disk := findDisk(metric, v.Instance)
// sum of iop read average
metric.Disks[disk].Read.Op = summation(v.Value[s])
metric.Disks[disk].Read.Ops = v.Value[s]
case "virtualDisk.numberWriteAveraged.average":
disk := findDisk(metric, v.Instance)
// sum of iop write average
metric.Disks[disk].Write.Op = summation(v.Value[s])
metric.Disks[disk].Write.Ops = v.Value[s]
// NET
default:
name := sub.DeviceName(v.Instance)
// if we have a name gather the stats
if name != "" {
// get the network to update
net := findNetwork(metric, name)
switch v.Name {
case "net.bytesRx.average":
// perfManager returns kbps -- convert to sum of bytes
sum := summation(v.Value[s]) * 1024
metric.Networks[net].Rx.Bytes = sum
metric.Networks[net].Rx.Kbps = v.Value[s]
case "net.bytesTx.average":
// perfManager returns kbps -- convert to sum of bytes
sum := summation(v.Value[s]) * 1024
metric.Networks[net].Tx.Bytes = sum
metric.Networks[net].Tx.Kbps = v.Value[s]
case "net.droppedRx.summation":
metric.Networks[net].Rx.Dropped = v.Value[s]
case "net.droppedTx.summation":
metric.Networks[net].Tx.Dropped = v.Value[s]
case "net.packetsRx.summation":
metric.Networks[net].Rx.Packets = v.Value[s]
case "net.packetsTx.summation":
metric.Networks[net].Tx.Packets = v.Value[s]
}
}
}
}
sub.Publish(metric)
}
}
}
// Subscribe to a vm metric subscription
func (vmc *VMCollector) Subscribe(op trace.Operation, moref types.ManagedObjectReference, id string) (chan interface{}, error) {
vmc.mu.Lock()
defer vmc.mu.Unlock()
// used at end of func
subscriptionCount := len(vmc.subs)
// do we already have this subscription?
_, exists := vmc.subs[moref]
if !exists {
op.Debugf("Creating new subscription(%s)", id)
sub, err := newVMSubscription(op, vmc.session, moref, id)
if err != nil {
return nil, err
}
vmc.subs[moref] = sub
}
// get a subscriber channel
ch := vmc.subs[moref].Channel()
// This is first subscription so start collection
if subscriptionCount == 0 {
vmc.Start()
}
return ch, nil
}
// Unsubscribe from a vm metric subscription. The subscriber channel will
// be evicted and when no subscribers remain the subscription will be removed.
func (vmc *VMCollector) Unsubscribe(op trace.Operation, moref types.ManagedObjectReference, ch chan interface{}) {
vmc.mu.Lock()
defer vmc.mu.Unlock()
sub, exists := vmc.subs[moref]
if exists {
// remove the communication channel
sub.Evict(ch)
// do we have any subscribers to this subscription
if sub.Publishers() == 0 {
op.Debugf("Deleting metric subscription(%s)", sub.ID())
delete(vmc.subs, moref)
}
op.Debugf("Unsubscribed %s from metrics", sub.ID())
}
// no subscriptions, so stop the collection
if len(vmc.subs) == 0 {
vmc.Stop()
}
}
// instanceID coverts the ID or Key of a metric from a string to int.
func instanceID(op trace.Operation, subscriptionID string, instance string) (int, error) {
converted, err := strconv.Atoi(instance)
if err != nil {
// I don't expect this to ever happen, but if it does log and don't publish
op.Errorf("metrics failed to convert the subscription(%s) device id to an int - value(%#v): %s", subscriptionID, instance, err)
return converted, err
}
return converted, nil
}
// summation returns the product of the average * interval
func summation(avg int64) uint64 {
return uint64(avg) * uint64(sampleInterval)
}
// findDisk will iterate over the Disks and return the DiskUsage ordinal position
func findDisk(metric *VMMetrics, name string) int {
// find by name
for i := range metric.Disks {
if metric.Disks[i].Name == name {
return i
}
}
// Let's create the disk
d := VirtualDisk{
Name: name,
Read: DiskUsage{},
Write: DiskUsage{},
}
metric.Disks = append(metric.Disks, d)
return len(metric.Disks) - 1
}
// findNetwork will iterate over the Networks and return the NetworkUsage ordinal position
func findNetwork(metric *VMMetrics, name string) int {
// find by name
for i := range metric.Networks {
if metric.Networks[i].Name == name {
return i
}
}
// Let's create the disk
net := Network{
Name: name,
Rx: NetworkUsage{},
Tx: NetworkUsage{},
}
metric.Networks = append(metric.Networks, net)
return len(metric.Networks) - 1
}

186
vendor/github.com/vmware/vic/pkg/vsphere/performance/vm_subscription.go generated vendored Normal file
View File

@@ -0,0 +1,186 @@
// Copyright 2017 VMware, Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package performance
import (
"fmt"
"strconv"
"time"
"github.com/docker/docker/pkg/pubsub"
"github.com/vmware/govmomi/object"
"github.com/vmware/govmomi/vim25/types"
"github.com/vmware/vic/pkg/trace"
"github.com/vmware/vic/pkg/vsphere/session"
"github.com/vmware/vic/pkg/vsphere/vm"
)
// vmSubscription is a 1:1 relationship to a Virtual Machine and a
// 1:M relationship to subscribers
type vmSubscription struct {
vm *vm.VirtualMachine
id string
pub *pubsub.Publisher
devices object.VirtualDeviceList
deviceInstanceToKey map[string]string
diskNames []string // container's virtualDisk names
networkNames []string // container's network names
}
// DeviceName will return the name associated with the metric instance id
func (sub *vmSubscription) DeviceName(instance string) string {
var name string
// did we previously find this device
if name, exists := sub.deviceInstanceToKey[instance]; exists {
return name
}
// convert instance to key - we are expecting regular failures, so no logging
// or returning of error
key, err := strconv.Atoi(instance)
if err != nil {
// this is not a key, so return an empty string
return name
}
// find the device and get the name
device := sub.devices.FindByKey(int32(key))
if device != nil {
// get the name
name = sub.devices.Name(device)
// populate map
sub.deviceInstanceToKey[instance] = name
}
return name
}
// ID returns the subscription's id
func (sub *vmSubscription) ID() string {
return sub.id
}
// DeviceList retrieves the VMs devices and builds slices of the disk and network
// names
func (sub *vmSubscription) DeviceList(op trace.Operation) error {
list, err := sub.vm.Device(op.Context)
if err != nil {
op.Errorf("vm stats subscription(%s) unable to load devices: %s", sub.ID(), err)
return err
}
// populate slice for disk and network names
for i := range list {
switch list.Type(list[i]) {
case object.DeviceTypeDisk:
// disk names are presented by the performanceManager based on their relationship to the
// controller. So we need to create a disk name that aligns with the performance manager
// naming pattern
disk := list[i].GetVirtualDevice()
switch c := list.FindByKey(disk.ControllerKey).(type) {
case types.BaseVirtualSCSIController:
sub.diskNames = append(sub.diskNames, fmt.Sprintf("%s%d:%d", "scsi", c.GetVirtualSCSIController().BusNumber, *disk.UnitNumber))
}
case object.DeviceTypeEthernet:
sub.networkNames = append(sub.networkNames, list.Name(list[i]))
}
}
sub.devices = list
return nil
}
// Disks returns an initialized slice of the containers VirtualDisks
func (sub *vmSubscription) Disks() []VirtualDisk {
var disks []VirtualDisk
for i := range sub.diskNames {
d := VirtualDisk{
Name: sub.diskNames[i],
Read: DiskUsage{},
Write: DiskUsage{},
}
disks = append(disks, d)
}
return disks
}
// Networks returns an initialized slice of the containers networks
func (sub *vmSubscription) Networks() []Network {
var networks []Network
for i := range sub.networkNames {
n := Network{
Name: sub.networkNames[i],
Rx: NetworkUsage{},
Tx: NetworkUsage{},
}
networks = append(networks, n)
}
return networks
}
// Publish sends the metric to all channels subscribed
func (sub *vmSubscription) Publish(metric *VMMetrics) {
// if no disk / network reported then add the defaults
if len(metric.Disks) == 0 {
metric.Disks = sub.Disks()
}
if len(metric.Networks) == 0 {
metric.Networks = sub.Networks()
}
sub.pub.Publish(metric)
}
// Publishers returns the number of channels subscribed to the container
func (sub *vmSubscription) Publishers() int {
return sub.pub.Len()
}
// Channel provides the communication chan for metrics subscriptions
func (sub *vmSubscription) Channel() chan interface{} {
return sub.pub.Subscribe()
}
// Evict will remove the channel from the publisher ending the subscription
func (sub *vmSubscription) Evict(ch chan interface{}) {
sub.pub.Evict(ch)
}
// newVMSubscription is a helper func to convert the interface to a subscription
func newVMSubscription(op trace.Operation, session *session.Session, moref types.ManagedObjectReference, id string) (*vmSubscription, error) {
// ensure we have a valid moRef..we won't worry about inspecting the details
if moref.String() == "" {
err := fmt.Errorf("no vm associated with new stats subscription: %s", id)
op.Errorf("%s", err)
return nil, err
}
sub := &vmSubscription{
vm: vm.NewVirtualMachine(op.Context, session, moref),
deviceInstanceToKey: make(map[string]string),
}
err := sub.DeviceList(op)
if err != nil {
return nil, err
}
// create the publisher
sub.pub = pubsub.NewPublisher(100*time.Millisecond, 0)
return sub, nil
}