VMware vSphere Integrated Containers provider (#206)

* Add Virtual Kubelet provider for VIC

Initial virtual kubelet provider for VMware VIC.  This provider currently
handles creating and starting of a pod VM via the VIC portlayer and persona
server.  Image store handling via the VIC persona server.  This provider
currently requires the feature/wolfpack branch of VIC.

* Added pod stop and delete.  Also added node capacity.

Added the ability to stop and delete pod VMs via VIC.  Also retrieve
node capacity information from the VCH.

* Cleanup and readme file

Some file clean up and added a Readme.md markdown file for the VIC
provider.

* Cleaned up errors, added function comments, moved operation code

1. Cleaned up error handling.  Set standard for creating errors.
2. Added method prototype comments for all interface functions.
3. Moved PodCreator, PodStarter, PodStopper, and PodDeleter to a new folder.

* Add mocking code and unit tests for podcache, podcreator, and podstarter

Used the unit test framework used in VIC to handle assertions in the provider's
unit test.  Mocking code generated using OSS project mockery, which is compatible
with the testify assertion framework.

* Vendored packages for the VIC provider

Requires feature/wolfpack branch of VIC and a few specific commit sha of
projects used within VIC.

* Implementation of POD Stopper and Deleter unit tests (#4)

* Updated files for initial PR

vendor/github.com/vmware/vic/pkg/dio/doc.go

@@ -0,0 +1,18 @@
+// Copyright 2016 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package dio adds dynamic behaviour to the standard io package mutliX types
+package dio
+
+var verbose = true

vendor/github.com/vmware/vic/pkg/dio/multi_test.go

@@ -0,0 +1,430 @@
+// Copyright 2016-2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dio
+
+import (
+	"bytes"
+	"io"
+	"strings"
+	"sync"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+const (
+	count = 32
+
+	base    = "base functionality"
+	dynamic = "dynamic add/remove functionality"
+)
+
+type filterf func(idx int) bool
+
+func FilterBuffers(in []*bytes.Buffer, fn filterf) []*bytes.Buffer {
+	var out []*bytes.Buffer
+
+	for i := 0; i < len(in); i++ {
+		if fn(i) {
+			out = append(out, in[i])
+		}
+	}
+	return out
+}
+
+func FilterWriters(in []io.Writer, fn filterf) []io.Writer {
+	var out []io.Writer
+
+	for i := 0; i < len(in); i++ {
+		if fn(i) {
+			out = append(out, in[i])
+		}
+	}
+	return out
+}
+
+func FilterReaders(in []io.Reader, fn filterf) []io.Reader {
+	var out []io.Reader
+
+	for i := 0; i < len(in); i++ {
+		if fn(i) {
+			out = append(out, in[i])
+		}
+	}
+	return out
+}
+
+func write(t *testing.T, mwriter DynamicMultiWriter, p []byte) {
+	n, err := mwriter.Write(p)
+	if err != nil {
+		t.Errorf("write: %v", err)
+	}
+	if n != len(p) {
+		t.Errorf("short write: %d != %d", n, len(p))
+	}
+}
+
+func read(t *testing.T, mreader DynamicMultiReader, limit int) []byte {
+	total := 0
+
+	var buf = make([]byte, 32*1024)
+	for {
+		n, err := mreader.Read(buf[total:limit])
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			t.Errorf("read: %v", err)
+		}
+
+		total += n
+		if total >= limit {
+			break
+		}
+	}
+
+	return buf[:limit]
+}
+
+func each(t *testing.T, buffers []*bytes.Buffer, s string) {
+	for i := range buffers {
+		assert.Equal(t, buffers[i].String(), s)
+	}
+}
+
+// TestMultiWrite creates multi writers and writes to them, then removes some of them and writes again
+func TestMultiWrite(t *testing.T) {
+	var wg sync.WaitGroup
+
+	var writers []io.Writer
+	var buffers []*bytes.Buffer
+
+	// create & initialize writers and buffers
+	for i := 0; i < count; i++ {
+		var buffer bytes.Buffer
+
+		reader, writer := io.Pipe()
+
+		writers = append(writers, writer)
+		buffers = append(buffers, &buffer)
+
+		wg.Add(1)
+		go func() {
+			// set up a goroutine so we don't block writes
+			io.CopyN(&buffer, reader, int64(len(base)))
+
+			wg.Done()
+		}()
+	}
+
+	// create the multi writer
+	mwriter := MultiWriter(writers...)
+
+	// write and ensure io.Copy returns
+	write(t, mwriter, []byte(base))
+	wg.Wait()
+
+	each(t, buffers, base)
+}
+
+// TestMultiWrite creates bunch of multi writers and writes to them, then adds more and writes again
+func TestWriteAdd(t *testing.T) {
+	var wg sync.WaitGroup
+	var wgAdded sync.WaitGroup
+
+	var writers []io.Writer
+	var buffers []*bytes.Buffer
+
+	// create & initialize writers and buffers into two categories
+	// *Added ones will be added to multi writer later
+	for i := 0; i < count; i++ {
+		var buffer bytes.Buffer
+
+		reader, writer := io.Pipe()
+
+		writers = append(writers, writer)
+		buffers = append(buffers, &buffer)
+
+		if i%3 != 0 {
+			wg.Add(1)
+		}
+		wgAdded.Add(1)
+		go func(i int) {
+			if i%3 != 0 {
+				io.CopyN(&buffer, reader, int64(len(base)))
+				wg.Done()
+			}
+			io.CopyN(&buffer, reader, int64(len(dynamic)))
+
+			wgAdded.Done()
+		}(i)
+	}
+
+	writersAdded := FilterWriters(writers, func(i int) bool { return i%3 == 0 })
+	writersLeft := FilterWriters(writers, func(i int) bool { return i%3 != 0 })
+
+	buffersAdded := FilterBuffers(buffers, func(i int) bool { return i%3 == 0 })
+	buffersLeft := FilterBuffers(buffers, func(i int) bool { return i%3 != 0 })
+
+	// create the multi writer
+	mwriter := MultiWriter(writersLeft...)
+
+	// write and ensure io.Copy returns
+	write(t, mwriter, []byte(base))
+	wg.Wait()
+
+	each(t, buffersLeft, base)
+
+	// add skipped writers to the writer
+	mwriter.Add(writersAdded...)
+
+	// write and ensure io.Copy returns
+	write(t, mwriter, []byte(dynamic))
+	wgAdded.Wait()
+
+	each(t, buffersLeft, base+dynamic)
+	each(t, buffersAdded, dynamic)
+}
+
+// TestMultiWrite creates multi writers and writes to them, then removes some of them and writes again
+func TestWriteRemove(t *testing.T) {
+	var wg sync.WaitGroup
+	var wgRemoved sync.WaitGroup
+
+	var writers []io.Writer
+	var buffers []*bytes.Buffer
+
+	// create & initialize writers and buffers into two categories
+	// *Removed ones will be filtered out from multi writer later
+	for i := 0; i < count; i++ {
+		var buffer bytes.Buffer
+
+		reader, writer := io.Pipe()
+
+		writers = append(writers, writer)
+		buffers = append(buffers, &buffer)
+
+		// set up a goroutine so we don't block writes
+		wg.Add(1)
+		wgRemoved.Add(1)
+		go func(i int) {
+			// set up a goroutine so we don't block writes
+			io.CopyN(&buffer, reader, int64(len(base)))
+
+			wg.Done()
+
+			if i%3 == 0 {
+				wgRemoved.Done()
+				return
+			}
+
+			io.CopyN(&buffer, reader, int64(len(dynamic)))
+
+			wgRemoved.Done()
+		}(i)
+	}
+
+	// create the multi writer
+	mwriter := MultiWriter(writers...)
+
+	// write and ensure io.Copy returns
+	write(t, mwriter, []byte(base))
+	wg.Wait()
+
+	each(t, buffers, base)
+
+	// remove the writers
+	for i := 0; i < count; i++ {
+		if i%3 == 0 {
+			mwriter.Remove(writers[i])
+		}
+	}
+
+	// write and ensure io.Copy returns
+	write(t, mwriter, []byte(dynamic))
+	wgRemoved.Wait()
+
+	buffersLeft := FilterBuffers(buffers, func(i int) bool { return i%3 != 0 })
+	buffersRemoved := FilterBuffers(buffers, func(i int) bool { return i%3 == 0 })
+
+	each(t, buffersLeft, base+dynamic)
+	each(t, buffersRemoved, base)
+
+}
+
+// TestMultiRead creates multi readers and reads from them
+func TestMultiRead(t *testing.T) {
+	var wg sync.WaitGroup
+
+	var readers []io.Reader
+
+	// create & initialize writers and buffers
+	for i := 0; i < count; i++ {
+		reader, writer := io.Pipe()
+
+		readers = append(readers, reader)
+
+		wg.Add(1)
+		go func() {
+			// set up a  goroutine so we don't block reads
+			io.CopyN(writer, bytes.NewReader([]byte(base)), int64(len(base)))
+
+			wg.Done()
+		}()
+	}
+
+	// create the multi writer
+	mreader := MultiReader(readers...)
+
+	expected := strings.Repeat(base, count)
+	// read and ensure io.Copy returns
+	buffer := read(t, mreader, len(expected))
+	wg.Wait()
+
+	assert.Equal(t, expected, string(buffer))
+}
+
+// TestMultiRead creates multi readers and reads from them, then adds mores and reads again
+func TestReadAdd(t *testing.T) {
+	var wg sync.WaitGroup
+	var wgAdded sync.WaitGroup
+	var wgLeft sync.WaitGroup
+
+	var readers []io.Reader
+	var pipereaders []io.PipeReader
+
+	wgLeft.Add(1)
+	// create & initialize writers and buffers
+	for i := 0; i < count; i++ {
+		reader, writer := io.Pipe()
+
+		readers = append(readers, reader)
+
+		if i%3 != 0 {
+			pipereaders = append(pipereaders, *reader)
+
+			wg.Add(1)
+		}
+		wgAdded.Add(1)
+		go func(i int) {
+			if i%3 != 0 {
+				io.CopyN(writer, bytes.NewReader([]byte(base)), int64(len(base)))
+				wg.Done()
+			}
+			wgLeft.Wait()
+			io.CopyN(writer, bytes.NewReader([]byte(dynamic)), int64(len(dynamic)))
+
+			wgAdded.Done()
+		}(i)
+	}
+
+	readersAdded := FilterReaders(readers, func(i int) bool { return i%3 == 0 })
+	readersLeft := FilterReaders(readers, func(i int) bool { return i%3 != 0 })
+
+	// create the multi writer
+	mreader := MultiReader(readersLeft...)
+
+	expected := strings.Repeat(base, len(readersLeft))
+	// read and ensure io.Copy returns
+	buffer := read(t, mreader, len(expected))
+	wg.Wait()
+
+	assert.Equal(t, expected, string(buffer))
+
+	// close the initial set otherwise they will block
+	for i := range pipereaders {
+		pipereaders[i].Close()
+	}
+
+	wgLeft.Done()
+
+	// add the rest of the readers
+	mreader.Add(readersAdded...)
+
+	expected = strings.Repeat(dynamic, len(readersAdded))
+	// read and ensure io.Copy returns
+	buffer = read(t, mreader, len(expected))
+	wgAdded.Wait()
+
+	assert.Equal(t, expected, string(buffer))
+
+}
+
+// TestReadRemove creates multi readers and reads from them, then removes some and reads again
+func TestReadRemove(t *testing.T) {
+	var wg sync.WaitGroup
+	var wgRemoved sync.WaitGroup
+	var wgLeft sync.WaitGroup
+
+	var readers []io.Reader
+	var writers []io.Writer
+
+	wgLeft.Add(1)
+	// create & initialize writers and buffers
+	for i := 0; i < count; i++ {
+		reader, writer := io.Pipe()
+
+		readers = append(readers, reader)
+		writers = append(writers, writer)
+
+		// set up a goroutine so we don't block writes
+		wg.Add(1)
+		wgRemoved.Add(1)
+		go func(i int) {
+			// set up a goroutine so we don't block writes
+			io.CopyN(writer, bytes.NewReader([]byte(base)), int64(len(base)))
+
+			wg.Done()
+
+			if i%3 == 0 {
+				wgRemoved.Done()
+				return
+			}
+
+			wgLeft.Wait()
+
+			io.CopyN(writer, bytes.NewReader([]byte(dynamic)), int64(len(dynamic)))
+
+			wgRemoved.Done()
+		}(i)
+
+	}
+
+	// create the multi writer
+	mreader := MultiReader(readers...)
+
+	expected := strings.Repeat(base, count)
+	// read and ensure io.Copy returns
+	buffer := read(t, mreader, len(expected))
+	wg.Wait()
+
+	assert.Equal(t, expected, string(buffer))
+	wgLeft.Done()
+
+	readersLeft := FilterReaders(readers, func(i int) bool { return i%3 != 0 })
+	readersRemoved := FilterReaders(readers, func(i int) bool { return i%3 == 0 })
+
+	for i := range readersRemoved {
+		mreader.Remove(readersRemoved[i])
+	}
+
+	expected = strings.Repeat(dynamic, len(readersLeft))
+	// read and ensure io.Copy returns
+	buffer = read(t, mreader, len(expected))
+	wgRemoved.Wait()
+
+	assert.Equal(t, expected, string(buffer))
+
+}

vendor/github.com/vmware/vic/pkg/dio/reader.go

@@ -0,0 +1,220 @@
+// Copyright 2016-2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package dio adds dynamic behaviour to the standard io package mutliX types
+package dio
+
+import (
+	"io"
+	"sync"
+
+	log "github.com/Sirupsen/logrus"
+)
+
+// DynamicMultiReader adds dynamic add/remove to the base multireader behaviour
+type DynamicMultiReader interface {
+	io.Reader
+	Add(...io.Reader)
+	Remove(io.Reader)
+	Close() error
+	PropagateEOF(bool)
+}
+
+type multiReader struct {
+	mutex sync.Mutex
+
+	cond           *sync.Cond
+	err            error
+	readers        []io.Reader
+	honorInlineEOF bool
+}
+
+// PropagateEOF toggles whether to return EOF when all readers return EOF.
+// Setting this to true will result in an EOF if there are no readers available
+// when Read is next called
+func (t *multiReader) PropagateEOF(val bool) {
+	t.mutex.Lock()
+	t.honorInlineEOF = val
+	t.cond.Broadcast()
+	t.mutex.Unlock()
+}
+
+func (t *multiReader) Read(p []byte) (int, error) {
+	var n int
+	var err error
+	var rTmp []io.Reader
+
+	if verbose {
+		defer func() {
+			log.Debugf("[%p] read %q from %d readers (err: %#+v)", t, string(p[:n]), len(rTmp), err)
+		}()
+	}
+
+	t.mutex.Lock()
+	// stash a copy of the t.err
+	err = t.err
+	t.mutex.Unlock()
+
+	// Close sets this
+	if err == io.EOF || err == io.ErrClosedPipe {
+		if verbose {
+			log.Debugf("[%p] read from closed multi-reader, returning EOF", t)
+		}
+		return 0, io.EOF
+	}
+
+	// if there's no readers we are steady state - has to be after t.err check to
+	// get correct Close behaviour.
+	// Blocking behaviour!
+	t.mutex.Lock()
+	for len(t.readers) == 0 && t.err == nil {
+		log.Debugf("[%p] Going into sleep with %d readers", t, len(t.readers))
+		t.cond.Wait()
+		log.Debugf("[%p] Woken from sleep %d readers", t, len(t.readers))
+	}
+
+	// stash a copy of the readers slie to iterate later
+	rTmp = make([]io.Reader, len(t.readers))
+	copy(rTmp, t.readers)
+	// stash a copy of the t.err
+	err = t.err
+	t.mutex.Unlock()
+
+	if err != nil {
+		return 0, err
+	}
+
+	// eof counter
+	eof := 0
+	for _, r := range rTmp {
+		slice := p[n:]
+		if len(slice) == 0 {
+			// we've run out of target space and don't know what
+			// the remaining readers have, so not EOF
+			return n, nil
+		}
+
+		x, err := r.Read(slice)
+		n += x
+		if err != nil {
+			if err != io.EOF && err != io.ErrClosedPipe {
+				t.mutex.Lock()
+				// if there was an actual error, return that
+				t.err = err
+				t.mutex.Unlock()
+
+				return n, err
+			}
+			// increment the EOF counter and remove the reader that retured EOF
+			log.Debugf("[%p] removing reader due to EOF", t)
+			// Remove grabs the lock
+			t.Remove(r)
+
+			eof++
+		}
+	}
+
+	// This means readers closed/removed while we iterate
+	// if no data is to be returned, there's no major error, and the number of
+	// reported EOFs matches the number of readers on entry to the main loop
+	if n == 0 && t.err == nil && eof == len(rTmp) {
+		log.Debugf("[%p] All of the readers returned EOF (%d)", t, len(rTmp))
+		t.mutex.Lock()
+		// queue up an EOF for the next time around if no new readers are added
+		if t.honorInlineEOF {
+			t.err = io.EOF
+		}
+		t.mutex.Unlock()
+	}
+	return n, nil
+}
+
+func (t *multiReader) Add(reader ...io.Reader) {
+	t.mutex.Lock()
+	defer t.mutex.Unlock()
+
+	t.readers = append(t.readers, reader...)
+	// if we've got a new reader, we're not EOF any more until that reader EOFs
+	t.err = nil
+	t.cond.Broadcast()
+
+	if verbose {
+		log.Debugf("[%p] added reader - now %d readers", t, len(t.readers))
+
+		for i, r := range t.readers {
+			log.Debugf("[%p] Reader %d [%p]", t, i, r)
+		}
+	}
+}
+
+// TODO: add a WriteTo for more efficient copy
+
+func (t *multiReader) Close() error {
+	t.mutex.Lock()
+	defer t.mutex.Unlock()
+
+	log.Debugf("[%p] Close on readers", t)
+	for _, r := range t.readers {
+		if c, ok := r.(io.Closer); ok {
+			log.Debugf("[%p] Closing reader %+v", t, r)
+			c.Close()
+		}
+	}
+
+	t.err = io.EOF
+	t.cond.Broadcast()
+	return nil
+}
+
+// Remove doesn't return an error if element isn't found as the end result is
+// identical
+func (t *multiReader) Remove(reader io.Reader) {
+	t.mutex.Lock()
+	defer t.mutex.Unlock()
+
+	if verbose {
+		log.Debugf("[%p] removing reader - currently %d readers", t, len(t.readers))
+	}
+
+	for i, r := range t.readers {
+		if r == reader {
+			t.readers = append(t.readers[:i], t.readers[i+1:]...)
+			// using range directly means that we're looping up, so indexes are now invalid
+			if verbose {
+				log.Debugf("[%p] removed reader - now %d readers", t, len(t.readers))
+
+				for i, r := range t.readers {
+					log.Debugf("[%p] Reader %d [%p]", t, i, r)
+				}
+			}
+			break
+		}
+	}
+}
+
+// MultiReader returns a Reader that's the logical concatenation of
+// the provided input readers.  They're read sequentially.  Once all
+// inputs have returned EOF, Read will return EOF.  If any of the readers
+// return a non-nil, non-EOF error, Read will return that error.
+func MultiReader(readers ...io.Reader) DynamicMultiReader {
+	r := make([]io.Reader, len(readers))
+	copy(r, readers)
+	t := &multiReader{readers: r}
+	t.cond = sync.NewCond(&t.mutex)
+
+	if verbose {
+		log.Debugf("[%p] created multireader", t)
+	}
+	return t
+}

vendor/github.com/vmware/vic/pkg/dio/writer.go

@@ -0,0 +1,181 @@
+// Copyright 2016 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package dio adds dynamic behaviour to the standard io package mutliX types
+package dio
+
+import (
+	"io"
+	"os"
+	"sync"
+
+	log "github.com/Sirupsen/logrus"
+)
+
+// DynamicMultiWriter adds dynamic add/remove to the base multiwriter behaviour
+type DynamicMultiWriter interface {
+	io.Writer
+	Add(...io.Writer)
+	Remove(io.Writer)
+	Close() error
+}
+
+type multiWriter struct {
+	mutex     sync.Mutex
+	waitGroup sync.WaitGroup
+
+	writers []io.Writer
+}
+
+func (t *multiWriter) Write(p []byte) (int, error) {
+	var n int
+	var err error
+
+	var wTmp []io.Writer
+	if verbose {
+		defer func() {
+			log.Debugf("[%p] write %q to %d writers (err: %#+v)", t, string(p[:n]), len(wTmp), err)
+		}()
+	}
+
+	t.mutex.Lock()
+
+	t.waitGroup.Add(1)
+	defer t.waitGroup.Done()
+
+	// stash a local copy of the slice as we never want to write twice to a single writer
+	// if remove is called during this flow
+	wTmp = make([]io.Writer, len(t.writers))
+	copy(wTmp, t.writers)
+
+	t.mutex.Unlock()
+
+	eof := 0
+	// possibly want to add buffering or parallelize this
+	for _, w := range wTmp {
+		n, err = w.Write(p)
+		if err != nil {
+			// remove the writer
+			log.Debugf("[%p] removing writer %p due to %s", t, w, err.Error())
+
+			// Remove grabs the lock
+			t.Remove(w)
+
+			if err == io.EOF {
+				eof++
+			}
+		}
+
+		// FIXME: figure out what semantics we need here - currently we may not write to
+		// everything as we abort
+		if n != len(p) {
+			// remove the writer
+			log.Debugf("[%p] removing writer %p due to short write: %d != %d", t, w, n, len(p))
+
+			// Remove grabs the lock
+			t.Remove(w)
+		}
+	}
+
+	// This means writers closed/removed while we iterate
+	if eof != 0 && n == 0 && err == nil && eof == len(wTmp) {
+		log.Debugf("[%p] All of the writers returned EOF (%d)", t, len(wTmp))
+	}
+	return len(p), nil
+}
+
+func (t *multiWriter) Add(writer ...io.Writer) {
+	t.mutex.Lock()
+	defer t.mutex.Unlock()
+
+	t.writers = append(t.writers, writer...)
+	if verbose {
+		log.Debugf("[%p] added writer - now %d writers", t, len(t.writers))
+
+		for i, w := range t.writers {
+			log.Debugf("[%p] Writer %d [%p]", t, i, w)
+		}
+	}
+}
+
+// CloseWriter is an interface that implements structs
+// that close input streams to prevent from writing.
+type CloseWriter interface {
+	CloseWrite() error
+}
+
+// FIXME: provide a mechanism for selectively closing writers
+//  - currently this closes /dev/stdout and logging as well if present
+func (t *multiWriter) Close() error {
+	t.mutex.Lock()
+	defer t.mutex.Unlock()
+
+	// allow any pending writes to complete
+	t.waitGroup.Wait()
+
+	log.Debugf("[%p] Close on writers", t)
+	for _, w := range t.writers {
+		log.Debugf("[%p] Closing writer %+v", t, w)
+
+		if c, ok := w.(CloseWriter); ok {
+			log.Debugf("[%p] is a CloseWriter", t, w)
+			c.CloseWrite()
+		} else if c, ok := w.(io.Closer); ok && c != os.Stdout && c != os.Stderr {
+			log.Debugf("[%p] is a Closer", t, w)
+			// squash closing of stdout/err if bound
+			c.Close()
+		}
+	}
+
+	return nil
+}
+
+// TODO: add a ReadFrom for more efficient copy
+
+// Remove doesn't return an error if element isn't found as the end result is
+// identical
+func (t *multiWriter) Remove(writer io.Writer) {
+	t.mutex.Lock()
+	defer t.mutex.Unlock()
+
+	if verbose {
+		log.Debugf("[%p] removing writer %p - currently %d writers", t, writer, len(t.writers))
+	}
+	for i, w := range t.writers {
+		if w == writer {
+			t.writers = append(t.writers[:i], t.writers[i+1:]...)
+			if verbose {
+				log.Debugf("[%p] removed writer - now %d writers", t, len(t.writers))
+
+				for i, w := range t.writers {
+					log.Debugf("[%p] Writer %d [%p]", t, i, w)
+				}
+			}
+			break
+		}
+	}
+}
+
+// MultiWriter extends io.MultiWriter to allow add/remove of writers dynamically
+// without disrupting existing writing
+func MultiWriter(writers ...io.Writer) DynamicMultiWriter {
+	w := make([]io.Writer, len(writers))
+	copy(w, writers)
+	t := &multiWriter{writers: w}
+
+	if verbose {
+		log.Debugf("[%p] created multiwriter", t)
+	}
+	return t
+}