From 8b01368f81951757a9540ddedbbec856b1358615 Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Wed, 26 Sep 2018 13:18:02 -0700 Subject: [PATCH] Refactor provider init (#360) * Refactor provider init This moves provider init out of vkubelet setup, instead preferring to initialize vkubelet with a provider. * Split API server configuration from setup. This makes sure that configuration (which is done primarily through env vars) is separate from actually standing up the servers. This also makes sure to abort daemon initialization if the API servers are not able to start. --- client.go | 35 ++++++++++++++++++++++ root.go | 87 +++++++++++++++++++++++++++++++++++++++++++++++++++++-- taint.go | 56 +++++++++++++++++++++++++++++++++++ 3 files changed, 176 insertions(+), 2 deletions(-) create mode 100644 client.go create mode 100644 taint.go diff --git a/client.go b/client.go new file mode 100644 index 0000000..d092157 --- /dev/null +++ b/client.go @@ -0,0 +1,35 @@ +package cmd + +import ( + "os" + + "github.com/pkg/errors" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" +) + +func newClient(configPath string) (*kubernetes.Clientset, error) { + var config *rest.Config + + // Check if the kubeConfig file exists. + if _, err := os.Stat(configPath); !os.IsNotExist(err) { + // Get the kubeconfig from the filepath. + config, err = clientcmd.BuildConfigFromFlags("", configPath) + if err != nil { + return nil, errors.Wrap(err, "error building client config") + } + } else { + // Set to in-cluster config. + config, err = rest.InClusterConfig() + if err != nil { + return nil, errors.Wrap(err, "error building in cluster config") + } + } + + if masterURI := os.Getenv("MASTER_URI"); masterURI != "" { + config.Host = masterURI + } + + return kubernetes.NewForConfig(config) +} diff --git a/root.go b/root.go index 6f2c0b3..8e23c88 100644 --- a/root.go +++ b/root.go @@ -16,18 +16,29 @@ package cmd import ( "context" + "fmt" "os" "path/filepath" + "strconv" "strings" "github.com/Sirupsen/logrus" + "github.com/cpuguy83/strongerrors" homedir "github.com/mitchellh/go-homedir" + "github.com/pkg/errors" "github.com/spf13/cobra" "github.com/spf13/viper" "github.com/virtual-kubelet/virtual-kubelet/log" + "github.com/virtual-kubelet/virtual-kubelet/manager" "github.com/virtual-kubelet/virtual-kubelet/providers" + "github.com/virtual-kubelet/virtual-kubelet/providers/register" vkubelet "github.com/virtual-kubelet/virtual-kubelet/vkubelet" corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/kubernetes" +) + +const ( + defaultDaemonPort = "10250" ) var kubeletConfig string @@ -41,6 +52,11 @@ var taintKey string var disableTaint bool var logLevel string var metricsAddr string +var taint *corev1.Taint +var k8sClient *kubernetes.Clientset +var p providers.Provider +var rm *manager.ResourceManager +var apiConfig vkubelet.APIConfig // RootCmd represents the base command when called without any subcommands var RootCmd = &cobra.Command{ @@ -50,11 +66,21 @@ var RootCmd = &cobra.Command{ backend implementation allowing users to create kubernetes nodes without running the kubelet. This allows users to schedule kubernetes workloads on nodes that aren't running Kubernetes.`, Run: func(cmd *cobra.Command, args []string) { - f, err := vkubelet.New(nodeName, operatingSystem, kubeNamespace, kubeConfig, provider, providerConfig, taintKey, disableTaint, metricsAddr) + ctx := context.Background() + f, err := vkubelet.New(ctx, vkubelet.Config{ + Client: k8sClient, + Namespace: kubeNamespace, + NodeName: nodeName, + Taint: taint, + MetricsAddr: metricsAddr, + Provider: p, + ResourceManager: rm, + APIConfig: apiConfig, + }) if err != nil { log.L.WithError(err).Fatal("Error initializing virtual kubelet") } - if err := f.Run(context.Background()); err != nil { + if err := f.Run(ctx); err != nil { log.L.Fatal(err) } }, @@ -155,4 +181,61 @@ func initConfig() { }) logger.Level = level log.L = logger + + if !disableTaint { + taint, err = getTaint(taintKey, provider) + if err != nil { + logger.WithError(err).Fatal("Error setting up desired kubernetes node taint") + } + } + + k8sClient, err = newClient(kubeConfig) + if err != nil { + logger.WithError(err).Fatal("Error creating kubernetes client") + } + + rm, err = manager.NewResourceManager(k8sClient) + if err != nil { + logger.WithError(err).Fatal("Error initializing resource manager") + } + + daemonPortEnv := getEnv("KUBELET_PORT", defaultDaemonPort) + daemonPort, err := strconv.ParseInt(daemonPortEnv, 10, 32) + if err != nil { + logger.WithError(err).WithField("value", daemonPortEnv).Fatal("Invalid value from KUBELET_PORT in environment") + } + + initConfig := register.InitConfig{ + ConfigPath: providerConfig, + NodeName: nodeName, + OperatingSystem: operatingSystem, + ResourceManager: rm, + DaemonPort: int32(daemonPort), + InternalIP: os.Getenv("VKUBELET_POD_IP"), + } + + p, err = register.GetProvider(provider, initConfig) + if err != nil { + logger.WithError(err).Fatal("Error initializing provider") + } + + apiConfig, err = getAPIConfig() + if err != nil { + logger.WithError(err).Fatal("Error reading API config") + } +} + +func getAPIConfig() (vkubelet.APIConfig, error) { + config := vkubelet.APIConfig{ + CertPath: os.Getenv("APISERVER_CERT_LOCATION"), + KeyPath: os.Getenv("APISERVER_KEY_LOCATION"), + } + + port, err := strconv.Atoi(os.Getenv("KUBELET_PORT")) + if err != nil { + return vkubelet.APIConfig{}, strongerrors.InvalidArgument(errors.Wrap(err, "error parsing KUBELET_PORT variable")) + } + config.Addr = fmt.Sprintf(":%d", port) + + return config, nil } diff --git a/taint.go b/taint.go new file mode 100644 index 0000000..279dd4d --- /dev/null +++ b/taint.go @@ -0,0 +1,56 @@ +package cmd + +import ( + "os" + + "github.com/cpuguy83/strongerrors" + + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" +) + +// Default taint values +const ( + DefaultTaintEffect = corev1.TaintEffectPreferNoSchedule + DefaultTaintKey = "virtual-kubelet.io/provider" +) + +func getEnv(key, defaultValue string) string { + value, found := os.LookupEnv(key) + if found { + return value + } + return defaultValue +} + +// getTaint creates a taint using the provided key/value. +// Taint effect is read from the environment +// The taint key/value may be overwritten by the environment. +func getTaint(key, value string) (*corev1.Taint, error) { + if key == "" { + key = DefaultTaintKey + value = provider + } + + key = getEnv("VKUBELET_TAINT_KEY", key) + value = getEnv("VKUBELET_TAINT_VALUE", value) + effectEnv := getEnv("VKUBELET_TAINT_EFFECT", string(DefaultTaintEffect)) + + var effect corev1.TaintEffect + switch effectEnv { + case "NoSchedule": + effect = corev1.TaintEffectNoSchedule + case "NoExecute": + effect = corev1.TaintEffectNoExecute + case "PreferNoSchedule": + effect = corev1.TaintEffectPreferNoSchedule + default: + return nil, strongerrors.InvalidArgument(errors.Errorf("taint effect %q is not supported", effectEnv)) + } + + return &corev1.Taint{ + Key: key, + Value: value, + Effect: effect, + }, nil +}