Refactor provider init (#360)

* Refactor provider init

This moves provider init out of vkubelet setup, instead preferring to
initialize vkubelet with a provider.

* Split API server configuration from setup.

This makes sure that configuration (which is done primarily through env
vars) is separate from actually standing up the servers.

This also makes sure to abort daemon initialization if the API servers
are not able to start.

cmd/client.go

@@ -0,0 +1,35 @@
+package cmd
+
+import (
+	"os"
+
+	"github.com/pkg/errors"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+)
+
+func newClient(configPath string) (*kubernetes.Clientset, error) {
+	var config *rest.Config
+
+	// Check if the kubeConfig file exists.
+	if _, err := os.Stat(configPath); !os.IsNotExist(err) {
+		// Get the kubeconfig from the filepath.
+		config, err = clientcmd.BuildConfigFromFlags("", configPath)
+		if err != nil {
+			return nil, errors.Wrap(err, "error building client config")
+		}
+	} else {
+		// Set to in-cluster config.
+		config, err = rest.InClusterConfig()
+		if err != nil {
+			return nil, errors.Wrap(err, "error building in cluster config")
+		}
+	}
+
+	if masterURI := os.Getenv("MASTER_URI"); masterURI != "" {
+		config.Host = masterURI
+	}
+
+	return kubernetes.NewForConfig(config)
+}

cmd/root.go

@@ -16,18 +16,29 @@ package cmd
 
 import (
 	"context"
+	"fmt"
 	"os"
 	"path/filepath"
+	"strconv"
 	"strings"
 
 	"github.com/Sirupsen/logrus"
+	"github.com/cpuguy83/strongerrors"
 	homedir "github.com/mitchellh/go-homedir"
+	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	"github.com/virtual-kubelet/virtual-kubelet/log"
+	"github.com/virtual-kubelet/virtual-kubelet/manager"
 	"github.com/virtual-kubelet/virtual-kubelet/providers"
+	"github.com/virtual-kubelet/virtual-kubelet/providers/register"
 	vkubelet "github.com/virtual-kubelet/virtual-kubelet/vkubelet"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/client-go/kubernetes"
+)
+
+const (
+	defaultDaemonPort = "10250"
 )
 
 var kubeletConfig string
@@ -41,6 +52,11 @@ var taintKey string
 var disableTaint bool
 var logLevel string
 var metricsAddr string
+var taint *corev1.Taint
+var k8sClient *kubernetes.Clientset
+var p providers.Provider
+var rm *manager.ResourceManager
+var apiConfig vkubelet.APIConfig
 
 // RootCmd represents the base command when called without any subcommands
 var RootCmd = &cobra.Command{
@@ -50,11 +66,21 @@ var RootCmd = &cobra.Command{
 backend implementation allowing users to create kubernetes nodes without running the kubelet.
 This allows users to schedule kubernetes workloads on nodes that aren't running Kubernetes.`,
 	Run: func(cmd *cobra.Command, args []string) {
-		f, err := vkubelet.New(nodeName, operatingSystem, kubeNamespace, kubeConfig, provider, providerConfig, taintKey, disableTaint, metricsAddr)
+		ctx := context.Background()
+		f, err := vkubelet.New(ctx, vkubelet.Config{
+			Client:          k8sClient,
+			Namespace:       kubeNamespace,
+			NodeName:        nodeName,
+			Taint:           taint,
+			MetricsAddr:     metricsAddr,
+			Provider:        p,
+			ResourceManager: rm,
+			APIConfig:       apiConfig,
+		})
 		if err != nil {
 			log.L.WithError(err).Fatal("Error initializing virtual kubelet")
 		}
-		if err := f.Run(context.Background()); err != nil {
+		if err := f.Run(ctx); err != nil {
 			log.L.Fatal(err)
 		}
 	},
@@ -155,4 +181,61 @@ func initConfig() {
 	})
 	logger.Level = level
 	log.L = logger
+
+	if !disableTaint {
+		taint, err = getTaint(taintKey, provider)
+		if err != nil {
+			logger.WithError(err).Fatal("Error setting up desired kubernetes node taint")
+		}
+	}
+
+	k8sClient, err = newClient(kubeConfig)
+	if err != nil {
+		logger.WithError(err).Fatal("Error creating kubernetes client")
+	}
+
+	rm, err = manager.NewResourceManager(k8sClient)
+	if err != nil {
+		logger.WithError(err).Fatal("Error initializing resource manager")
+	}
+
+	daemonPortEnv := getEnv("KUBELET_PORT", defaultDaemonPort)
+	daemonPort, err := strconv.ParseInt(daemonPortEnv, 10, 32)
+	if err != nil {
+		logger.WithError(err).WithField("value", daemonPortEnv).Fatal("Invalid value from KUBELET_PORT in environment")
+	}
+
+	initConfig := register.InitConfig{
+		ConfigPath:      providerConfig,
+		NodeName:        nodeName,
+		OperatingSystem: operatingSystem,
+		ResourceManager: rm,
+		DaemonPort:      int32(daemonPort),
+		InternalIP:      os.Getenv("VKUBELET_POD_IP"),
+	}
+
+	p, err = register.GetProvider(provider, initConfig)
+	if err != nil {
+		logger.WithError(err).Fatal("Error initializing provider")
+	}
+
+	apiConfig, err = getAPIConfig()
+	if err != nil {
+		logger.WithError(err).Fatal("Error reading API config")
+	}
+}
+
+func getAPIConfig() (vkubelet.APIConfig, error) {
+	config := vkubelet.APIConfig{
+		CertPath: os.Getenv("APISERVER_CERT_LOCATION"),
+		KeyPath:  os.Getenv("APISERVER_KEY_LOCATION"),
+	}
+
+	port, err := strconv.Atoi(os.Getenv("KUBELET_PORT"))
+	if err != nil {
+		return vkubelet.APIConfig{}, strongerrors.InvalidArgument(errors.Wrap(err, "error parsing KUBELET_PORT variable"))
+	}
+	config.Addr = fmt.Sprintf(":%d", port)
+
+	return config, nil
 }

cmd/taint.go

@@ -0,0 +1,56 @@
+package cmd
+
+import (
+	"os"
+
+	"github.com/cpuguy83/strongerrors"
+
+	"github.com/pkg/errors"
+	corev1 "k8s.io/api/core/v1"
+)
+
+// Default taint values
+const (
+	DefaultTaintEffect = corev1.TaintEffectPreferNoSchedule
+	DefaultTaintKey    = "virtual-kubelet.io/provider"
+)
+
+func getEnv(key, defaultValue string) string {
+	value, found := os.LookupEnv(key)
+	if found {
+		return value
+	}
+	return defaultValue
+}
+
+// getTaint creates a taint using the provided key/value.
+// Taint effect is read from the environment
+// The taint key/value may be overwritten by the environment.
+func getTaint(key, value string) (*corev1.Taint, error) {
+	if key == "" {
+		key = DefaultTaintKey
+		value = provider
+	}
+
+	key = getEnv("VKUBELET_TAINT_KEY", key)
+	value = getEnv("VKUBELET_TAINT_VALUE", value)
+	effectEnv := getEnv("VKUBELET_TAINT_EFFECT", string(DefaultTaintEffect))
+
+	var effect corev1.TaintEffect
+	switch effectEnv {
+	case "NoSchedule":
+		effect = corev1.TaintEffectNoSchedule
+	case "NoExecute":
+		effect = corev1.TaintEffectNoExecute
+	case "PreferNoSchedule":
+		effect = corev1.TaintEffectPreferNoSchedule
+	default:
+		return nil, strongerrors.InvalidArgument(errors.Errorf("taint effect %q is not supported", effectEnv))
+	}
+
+	return &corev1.Taint{
+		Key:    key,
+		Value:  value,
+		Effect: effect,
+	}, nil
+}