virtual-kubelet/providers/vic/proxy/isolation_proxy.go

package proxy

import (
	"fmt"
	"time"

	vicerrors "github.com/vmware/vic/lib/apiservers/engine/errors"
	"github.com/vmware/vic/lib/apiservers/portlayer/client"
	"github.com/vmware/vic/lib/apiservers/portlayer/client/containers"
	"github.com/vmware/vic/lib/apiservers/portlayer/client/interaction"
	"github.com/vmware/vic/lib/apiservers/portlayer/client/logging"
	"github.com/vmware/vic/lib/apiservers/portlayer/client/scopes"
	"github.com/vmware/vic/lib/apiservers/portlayer/client/storage"
	"github.com/vmware/vic/lib/apiservers/portlayer/client/tasks"
	"github.com/vmware/vic/lib/apiservers/portlayer/models"
	"github.com/vmware/vic/pkg/trace"
	"github.com/vmware/vic/pkg/vsphere/sys"

	"github.com/docker/docker/api/types/strslice"

	"github.com/virtual-kubelet/virtual-kubelet/providers/vic/cache"
	"github.com/virtual-kubelet/virtual-kubelet/providers/vic/constants"
)

type IsolationProxy interface {
	CreateHandle(op trace.Operation) (string, string, error)
	AddImageToHandle(op trace.Operation, handle, deltaID, layerID, imageID, imageName string) (string, error)
	CreateHandleTask(op trace.Operation, handle, id, layerID string, config IsolationContainerConfig) (string, error)
	AddHandleToScope(op trace.Operation, handle string, config IsolationContainerConfig) (string, error)
	AddInteractionToHandle(op trace.Operation, handle string) (string, error)
	AddLoggingToHandle(op trace.Operation, handle string) (string, error)
	CommitHandle(op trace.Operation, handle, containerID string, waitTime int32) error
	SetState(op trace.Operation, handle, name, state string) (string, error)

	BindScope(op trace.Operation, handle string, name string) (string, interface{}, error)
	UnbindScope(op trace.Operation, handle string, name string) (string, interface{}, error)

	Handle(op trace.Operation, id, name string) (string, error)
	State(op trace.Operation, id, name string) (string, error)
	Remove(op trace.Operation, id string, force bool) error
}

type VicIsolationProxy struct {
	client        *client.PortLayer
	imageStore    ImageStore
	podCache      cache.PodCache
	portlayerAddr string
	hostUUID      string
}

type PortBinding struct {
	HostIP   string
	HostPort string
}

type IsolationContainerConfig struct {
	ID        string
	ImageID   string
	LayerID   string
	ImageName string
	Name      string
	Namespace string

	Cmd        []string
	Path       string
	Entrypoint []string
	//Args       []string
	Env        []string
	WorkingDir string
	User       string
	StopSignal string

	Attach    bool
	StdinOnce bool
	OpenStdin bool
	Tty       bool

	CPUCount int64
	Memory   int64

	PortMap map[string]PortBinding
}

func NewIsolationProxy(plClient *client.PortLayer, portlayerAddr string, hostUUID string, imageStore ImageStore, podCache cache.PodCache) IsolationProxy {
	if plClient == nil {
		return nil
	}

	return &VicIsolationProxy{
		client:        plClient,
		imageStore:    imageStore,
		podCache:      podCache,
		portlayerAddr: portlayerAddr,
		hostUUID:      hostUUID,
	}
}

// CreateHandle creates a "manifest" that will be used by Commit() to create the actual
// isolation vm.
//
// returns:
//	(container/pod id, handle, error)
func (v *VicIsolationProxy) CreateHandle(op trace.Operation) (string, string, error) {
	defer trace.End(trace.Begin("CreateHandle", op))

	if v.client == nil {
		return "", "", vicerrors.NillPortlayerClientError("IsolationProxy")
	}

	// Call the Exec port layer to create the container
	var err error
	var hostUUID string
	if v.hostUUID != "" {
		hostUUID = v.hostUUID
	} else {
		hostUUID, err = sys.UUID()
	}

	if err != nil {
		return "", "", vicerrors.InternalServerError("IsolationProxy.CreateContainerHandle got unexpected error getting VCH UUID")
	}

	plCreateParams := initIsolationConfig(op, "", constants.DummyRepoName, constants.DummyImage, constants.DummyLayerID, hostUUID)
	createResults, err := v.client.Containers.Create(plCreateParams)
	if err != nil {
		if _, ok := err.(*containers.CreateNotFound); ok {
			cerr := fmt.Errorf("No such image: %s", constants.DummyImage)
			op.Errorf("%s (%s)", cerr, err)
			return "", "", vicerrors.NotFoundError(cerr.Error())
		}

		// If we get here, most likely something went wrong with the port layer API server
		return "", "", vicerrors.InternalServerError(err.Error())
	}

	id := createResults.Payload.ID
	h := createResults.Payload.Handle

	return id, h, nil
}

// Handle retrieves a handle to a VIC container.  Handles should be treated as opaque strings.
//
// returns:
//	(handle string, error)
func (v *VicIsolationProxy) Handle(op trace.Operation, id, name string) (string, error) {
	if v.client == nil {
		return "", vicerrors.NillPortlayerClientError("IsolationProxy")
	}

	resp, err := v.client.Containers.Get(containers.NewGetParamsWithContext(op).WithID(id))
	if err != nil {
		switch err := err.(type) {
		case *containers.GetNotFound:
			return "", vicerrors.NotFoundError(name)
		case *containers.GetDefault:
			return "", vicerrors.InternalServerError(err.Payload.Message)
		default:
			return "", vicerrors.InternalServerError(err.Error())
		}
	}
	return resp.Payload, nil
}

func (v *VicIsolationProxy) AddImageToHandle(op trace.Operation, handle, deltaID, layerID, imageID, imageName string) (string, error) {
	defer trace.End(trace.Begin(handle, op))

	if v.client == nil {
		return "", vicerrors.InternalServerError("IsolationProxy.AddImageToContainer failed to get the portlayer client")
	}

	var err error
	var hostUUID string
	if v.hostUUID != "" {
		hostUUID = v.hostUUID
	} else {
		hostUUID, err = sys.UUID()
	}

	if err != nil {
		return "", vicerrors.InternalServerError("IsolationProxy.AddImageToContainer got unexpected error getting VCH UUID")
	}

	response, err := v.client.Storage.ImageJoin(storage.NewImageJoinParamsWithContext(op).WithStoreName(hostUUID).WithID(layerID).
		WithConfig(&models.ImageJoinConfig{
			Handle:   handle,
			DeltaID:  deltaID,
			ImageID:  imageID,
			RepoName: imageName,
		}))
	if err != nil {
		return "", vicerrors.InternalServerError(err.Error())
	}
	handle, ok := response.Payload.Handle.(string)
	if !ok {
		return "", vicerrors.InternalServerError(fmt.Sprintf("Type assertion failed for %#+v", handle))
	}

	return handle, nil
}

func (v *VicIsolationProxy) CreateHandleTask(op trace.Operation, handle, id, layerID string, config IsolationContainerConfig) (string, error) {
	defer trace.End(trace.Begin(handle, op))

	if v.client == nil {
		return "", vicerrors.InternalServerError("IsolationProxy.CreateContainerTask failed to create a portlayer client")
	}

	op.Debugf("CreateHandleTask - %#v", config)

	plTaskParams := IsolationContainerConfigToTask(op, id, layerID, config)
	plTaskParams.Config.Handle = handle

	op.Debugf("*** CreateContainerTask - params = %#v", *plTaskParams.Config)
	responseJoin, err := v.client.Tasks.Join(plTaskParams)
	if err != nil {
		op.Errorf("Unable to join primary task to container: %+v", err)
		return "", vicerrors.InternalServerError(err.Error())
	}

	handle, ok := responseJoin.Payload.Handle.(string)
	if !ok {
		return "", vicerrors.InternalServerError(fmt.Sprintf("Type assertion failed on handle from task join: %#+v", handle))
	}

	plBindParams := tasks.NewBindParamsWithContext(op).WithConfig(&models.TaskBindConfig{Handle: handle, ID: id})
	responseBind, err := v.client.Tasks.Bind(plBindParams)
	if err != nil {
		op.Errorf("Unable to bind primary task to container: %+v", err)
		return "", vicerrors.InternalServerError(err.Error())
	}

	handle, ok = responseBind.Payload.Handle.(string)
	if !ok {
		return "", vicerrors.InternalServerError(fmt.Sprintf("Type assertion failed on handle from task bind %#+v", handle))
	}

	return handle, nil
}

// AddHandleToScope adds a container, referenced by handle, to a scope.
// If an error is return, the returned handle should not be used.
//
// returns:
//	modified handle
func (v *VicIsolationProxy) AddHandleToScope(op trace.Operation, handle string, config IsolationContainerConfig) (string, error) {
	defer trace.End(trace.Begin(handle, op))

	if v.client == nil {
		return "", vicerrors.NillPortlayerClientError("IsolationProxy")
	}

	// configure network
	netConf := networkConfigFromIsolationConfig(config)
	if netConf != nil {
		addContRes, err := v.client.Scopes.AddContainer(scopes.NewAddContainerParamsWithContext(op).
			WithScope(netConf.NetworkName).
			WithConfig(&models.ScopesAddContainerConfig{
				Handle:        handle,
				NetworkConfig: netConf,
			}))

		if err != nil {
			op.Errorf("IsolationProxy.AddContainerToScope: Scopes error: %s", err.Error())
			return handle, vicerrors.InternalServerError(err.Error())
		}

		defer func() {
			if err == nil {
				return
			}
			// roll back the AddContainer call
			if _, err2 := v.client.Scopes.RemoveContainer(scopes.NewRemoveContainerParamsWithContext(op).WithHandle(handle).WithScope(netConf.NetworkName)); err2 != nil {
				op.Warnf("could not roll back container add: %s", err2)
			}
		}()

		handle = addContRes.Payload
	}

	return handle, nil
}

// AddLoggingToHandle adds logging capability to the isolation vm, referenced by handle.
// If an error is return, the returned handle should not be used.
//
// returns:
//	modified handle
func (v *VicIsolationProxy) AddLoggingToHandle(op trace.Operation, handle string) (string, error) {
	defer trace.End(trace.Begin(handle, op))

	if v.client == nil {
		return "", vicerrors.NillPortlayerClientError("IsolationProxy")
	}

	response, err := v.client.Logging.LoggingJoin(logging.NewLoggingJoinParamsWithContext(op).
		WithConfig(&models.LoggingJoinConfig{
			Handle: handle,
		}))
	if err != nil {
		return "", vicerrors.InternalServerError(err.Error())
	}
	handle, ok := response.Payload.Handle.(string)
	if !ok {
		return "", vicerrors.InternalServerError(fmt.Sprintf("Type assertion failed for %#+v", handle))
	}

	return handle, nil
}

// AddInteractionToContainer adds interaction capabilities to a container, referenced by handle.
// If an error is return, the returned handle should not be used.
//
// returns:
//	modified handle
func (v *VicIsolationProxy) AddInteractionToHandle(op trace.Operation, handle string) (string, error) {
	defer trace.End(trace.Begin(handle, op))

	if v.client == nil {
		return "", vicerrors.NillPortlayerClientError("IsolationProxy")
	}

	response, err := v.client.Interaction.InteractionJoin(interaction.NewInteractionJoinParamsWithContext(op).
		WithConfig(&models.InteractionJoinConfig{
			Handle: handle,
		}))
	if err != nil {
		return "", vicerrors.InternalServerError(err.Error())
	}
	handle, ok := response.Payload.Handle.(string)
	if !ok {
		return "", vicerrors.InternalServerError(fmt.Sprintf("Type assertion failed for %#+v", handle))
	}

	return handle, nil
}

func (v *VicIsolationProxy) CommitHandle(op trace.Operation, handle, containerID string, waitTime int32) error {
	defer trace.End(trace.Begin(handle, op))

	if v.client == nil {
		return vicerrors.NillPortlayerClientError("IsolationProxy")
	}

	var commitParams *containers.CommitParams
	if waitTime > 0 {
		commitParams = containers.NewCommitParamsWithContext(op).WithHandle(handle).WithWaitTime(&waitTime)
	} else {
		commitParams = containers.NewCommitParamsWithContext(op).WithHandle(handle)
	}

	_, err := v.client.Containers.Commit(commitParams)
	if err != nil {
		switch err := err.(type) {
		case *containers.CommitNotFound:
			return vicerrors.NotFoundError(containerID)
		case *containers.CommitConflict:
			return vicerrors.ConflictError(err.Error())
		case *containers.CommitDefault:
			return vicerrors.InternalServerError(err.Payload.Message)
		default:
			return vicerrors.InternalServerError(err.Error())
		}
	}

	return nil
}

//TODO:  I don't think this function should be in here.
// BindNetwork binds the handle to the scope and returns endpoints.  Caller of the function does not need
// to interpret the return value.  In the event the caller want to unbind,
func (v *VicIsolationProxy) BindScope(op trace.Operation, handle string, name string) (string, interface{}, error) {
	defer trace.End(trace.Begin(handle, op))

	if v.client == nil {
		return "", nil, vicerrors.NillPortlayerClientError("IsolationProxy")
	}

	bindParams := scopes.NewBindContainerParamsWithContext(op).WithHandle(handle)
	bindRes, err := v.client.Scopes.BindContainer(bindParams)
	if err != nil {
		switch err := err.(type) {
		case *scopes.BindContainerNotFound:
			return "", nil, vicerrors.NotFoundError(name)
		case *scopes.BindContainerInternalServerError:
			return "", nil, vicerrors.InternalServerError(err.Payload.Message)
		default:
			return "", nil, vicerrors.InternalServerError(err.Error())
		}
	}

	return bindRes.Payload.Handle, bindRes.Payload.Endpoints, nil
}

func (v *VicIsolationProxy) UnbindScope(op trace.Operation, handle string, name string) (string, interface{}, error) {
	defer trace.End(trace.Begin(handle, op))

	if v.client == nil {
		return "", nil, vicerrors.NillPortlayerClientError("IsolationProxy")
	}

	unbindParams := scopes.NewUnbindContainerParamsWithContext(op).WithHandle(handle)
	resp, err := v.client.Scopes.UnbindContainer(unbindParams)
	if err != nil {
		switch err := err.(type) {
		case *scopes.UnbindContainerNotFound:
			return "", nil, vicerrors.NotFoundError(name)
		case *scopes.UnbindContainerInternalServerError:
			return "", nil, vicerrors.InternalServerError(err.Payload.Message)
		default:
			return "", nil, vicerrors.InternalServerError(err.Error())
		}
	}

	return resp.Payload.Handle, resp.Payload.Endpoints, nil
}

// SetState adds the desire state of the isolation unit once the handle is commited.
//
//   returns handle string and error
func (v *VicIsolationProxy) SetState(op trace.Operation, handle, name, state string) (string, error) {
	defer trace.End(trace.Begin(handle, op))

	if v.client == nil {
		return "", vicerrors.NillPortlayerClientError("IsolationProxy")
	}

	resp, err := v.client.Containers.StateChange(containers.NewStateChangeParamsWithContext(op).WithHandle(handle).WithState(state))
	if err != nil {
		switch err := err.(type) {
		case *containers.StateChangeNotFound:
			return "", vicerrors.NotFoundError(name)
		case *containers.StateChangeDefault:
			return "", vicerrors.InternalServerError(err.Payload.Message)
		default:
			return "", vicerrors.InternalServerError(err.Error())
		}
	}

	return resp.Payload, nil
}

func (v *VicIsolationProxy) State(op trace.Operation, id, name string) (string, error) {
	defer trace.End(trace.Begin(id, op))

	if v.client == nil {
		return "", vicerrors.NillPortlayerClientError("IsolationProxy")
	}

	results, err := v.client.Containers.GetContainerInfo(containers.NewGetContainerInfoParamsWithContext(op).WithID(id))
	if err != nil {
		switch err := err.(type) {
		case *containers.GetContainerInfoNotFound:
			return "", vicerrors.NotFoundError(name)
		case *containers.GetContainerInfoInternalServerError:
			return "", vicerrors.InternalServerError(err.Payload.Message)
		default:
			return "", vicerrors.InternalServerError(fmt.Sprintf("Unknown error from the interaction port layer: %s", err))
		}
	}

	state := results.Payload.ContainerConfig.State
	return state, nil
}

func (v *VicIsolationProxy) Remove(op trace.Operation, id string, force bool) error {
	defer trace.End(trace.Begin(id, op))

	if v.client == nil {
		return vicerrors.NillPortlayerClientError("IsolationProxy")
	}

	pForce := force
	params := containers.NewContainerRemoveParamsWithContext(op).
		WithID(id).
		WithForce(&pForce).
		WithTimeout(120 * time.Second)

	removeOK, err := v.client.Containers.ContainerRemove(params)
	op.Debugf("ContainerRemove returned %# +v", removeOK)
	return err
}

//------------------------------------
// Utility Functions
//------------------------------------

// Convert isolation container config to portlayer task param
func IsolationContainerConfigToTask(op trace.Operation, id, layerID string, ic IsolationContainerConfig) *tasks.JoinParams {
	config := &models.TaskJoinConfig{}

	var path string
	var args []string

	// we explicitly specify the ID for the primary task so that it's the same as the containerID
	config.ID = id

	// Set the filesystem namespace this task expects to run in
	config.Namespace = layerID

	// Expand cmd into entrypoint and args
	cmd := strslice.StrSlice(ic.Cmd)
	if len(ic.Entrypoint) != 0 {
		path, args = ic.Entrypoint[0], append(ic.Entrypoint[1:], cmd...)
	} else {
		path, args = cmd[0], cmd[1:]
	}

	// copy the path
	config.Path = path

	// copy the args
	config.Args = make([]string, len(args))
	copy(config.Args, args)

	// copy the env array
	config.Env = make([]string, len(ic.Env))
	copy(config.Env, ic.Env)

	// working dir
	config.WorkingDir = ic.WorkingDir

	// user
	config.User = ic.User

	// attach.  Always set to true otherwise we cannot attach later.
	// this tells portlayer container is attachable.
	config.Attach = true

	// openstdin
	config.OpenStdin = ic.OpenStdin

	// tty
	config.Tty = ic.Tty

	// container stop signal
	config.StopSignal = ic.StopSignal

	op.Debugf("dockerContainerCreateParamsToTask = %+v", config)

	return tasks.NewJoinParamsWithContext(op).WithConfig(config)
}

// initIsolationConfig returns a default config used to create the isolation unit handle
func initIsolationConfig(op trace.Operation, name, repoName, imageID, layerID, imageStore string) *containers.CreateParams {
	config := &models.ContainerCreateConfig{}

	config.NumCpus = constants.DefaultCPUs
	config.MemoryMB = constants.DefaultMemory

	// Layer/vmdk to use
	config.Layer = layerID

	// Image ID
	config.Image = imageID

	// Repo Requested
	config.RepoName = repoName

	//copy friendly name
	config.Name = name

	// image store
	config.ImageStore = &models.ImageStore{Name: imageStore}

	// network
	config.NetworkDisabled = true

	// hostname
	config.Hostname = constants.HostName
	//// domainname - https://github.com/moby/moby/issues/27067
	//config.Domainname = cc.Config.Domainname

	op.Debugf("dockerContainerCreateParamsToPortlayer = %+v", config)

	return containers.NewCreateParamsWithContext(op).WithCreateConfig(config)
}

func networkConfigFromIsolationConfig(config IsolationContainerConfig) *models.NetworkConfig {
	nc := &models.NetworkConfig{
		NetworkName: "default",
	}

	for key, val := range config.PortMap {
		nc.Ports = append(nc.Ports, fmt.Sprintf("%s:%s", val.HostPort, key))
	}

	return nc
}