diff --git a/client.go b/client.go new file mode 100644 index 0000000..d092157 --- /dev/null +++ b/client.go @@ -0,0 +1,35 @@ +package cmd + +import ( + "os" + + "github.com/pkg/errors" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" +) + +func newClient(configPath string) (*kubernetes.Clientset, error) { + var config *rest.Config + + // Check if the kubeConfig file exists. + if _, err := os.Stat(configPath); !os.IsNotExist(err) { + // Get the kubeconfig from the filepath. + config, err = clientcmd.BuildConfigFromFlags("", configPath) + if err != nil { + return nil, errors.Wrap(err, "error building client config") + } + } else { + // Set to in-cluster config. + config, err = rest.InClusterConfig() + if err != nil { + return nil, errors.Wrap(err, "error building in cluster config") + } + } + + if masterURI := os.Getenv("MASTER_URI"); masterURI != "" { + config.Host = masterURI + } + + return kubernetes.NewForConfig(config) +} diff --git a/root.go b/root.go index 6f2c0b3..8e23c88 100644 --- a/root.go +++ b/root.go @@ -16,18 +16,29 @@ package cmd import ( "context" + "fmt" "os" "path/filepath" + "strconv" "strings" "github.com/Sirupsen/logrus" + "github.com/cpuguy83/strongerrors" homedir "github.com/mitchellh/go-homedir" + "github.com/pkg/errors" "github.com/spf13/cobra" "github.com/spf13/viper" "github.com/virtual-kubelet/virtual-kubelet/log" + "github.com/virtual-kubelet/virtual-kubelet/manager" "github.com/virtual-kubelet/virtual-kubelet/providers" + "github.com/virtual-kubelet/virtual-kubelet/providers/register" vkubelet "github.com/virtual-kubelet/virtual-kubelet/vkubelet" corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/kubernetes" +) + +const ( + defaultDaemonPort = "10250" ) var kubeletConfig string @@ -41,6 +52,11 @@ var taintKey string var disableTaint bool var logLevel string var metricsAddr string +var taint *corev1.Taint +var k8sClient *kubernetes.Clientset +var p providers.Provider +var rm *manager.ResourceManager +var apiConfig vkubelet.APIConfig // RootCmd represents the base command when called without any subcommands var RootCmd = &cobra.Command{ @@ -50,11 +66,21 @@ var RootCmd = &cobra.Command{ backend implementation allowing users to create kubernetes nodes without running the kubelet. This allows users to schedule kubernetes workloads on nodes that aren't running Kubernetes.`, Run: func(cmd *cobra.Command, args []string) { - f, err := vkubelet.New(nodeName, operatingSystem, kubeNamespace, kubeConfig, provider, providerConfig, taintKey, disableTaint, metricsAddr) + ctx := context.Background() + f, err := vkubelet.New(ctx, vkubelet.Config{ + Client: k8sClient, + Namespace: kubeNamespace, + NodeName: nodeName, + Taint: taint, + MetricsAddr: metricsAddr, + Provider: p, + ResourceManager: rm, + APIConfig: apiConfig, + }) if err != nil { log.L.WithError(err).Fatal("Error initializing virtual kubelet") } - if err := f.Run(context.Background()); err != nil { + if err := f.Run(ctx); err != nil { log.L.Fatal(err) } }, @@ -155,4 +181,61 @@ func initConfig() { }) logger.Level = level log.L = logger + + if !disableTaint { + taint, err = getTaint(taintKey, provider) + if err != nil { + logger.WithError(err).Fatal("Error setting up desired kubernetes node taint") + } + } + + k8sClient, err = newClient(kubeConfig) + if err != nil { + logger.WithError(err).Fatal("Error creating kubernetes client") + } + + rm, err = manager.NewResourceManager(k8sClient) + if err != nil { + logger.WithError(err).Fatal("Error initializing resource manager") + } + + daemonPortEnv := getEnv("KUBELET_PORT", defaultDaemonPort) + daemonPort, err := strconv.ParseInt(daemonPortEnv, 10, 32) + if err != nil { + logger.WithError(err).WithField("value", daemonPortEnv).Fatal("Invalid value from KUBELET_PORT in environment") + } + + initConfig := register.InitConfig{ + ConfigPath: providerConfig, + NodeName: nodeName, + OperatingSystem: operatingSystem, + ResourceManager: rm, + DaemonPort: int32(daemonPort), + InternalIP: os.Getenv("VKUBELET_POD_IP"), + } + + p, err = register.GetProvider(provider, initConfig) + if err != nil { + logger.WithError(err).Fatal("Error initializing provider") + } + + apiConfig, err = getAPIConfig() + if err != nil { + logger.WithError(err).Fatal("Error reading API config") + } +} + +func getAPIConfig() (vkubelet.APIConfig, error) { + config := vkubelet.APIConfig{ + CertPath: os.Getenv("APISERVER_CERT_LOCATION"), + KeyPath: os.Getenv("APISERVER_KEY_LOCATION"), + } + + port, err := strconv.Atoi(os.Getenv("KUBELET_PORT")) + if err != nil { + return vkubelet.APIConfig{}, strongerrors.InvalidArgument(errors.Wrap(err, "error parsing KUBELET_PORT variable")) + } + config.Addr = fmt.Sprintf(":%d", port) + + return config, nil } diff --git a/taint.go b/taint.go new file mode 100644 index 0000000..279dd4d --- /dev/null +++ b/taint.go @@ -0,0 +1,56 @@ +package cmd + +import ( + "os" + + "github.com/cpuguy83/strongerrors" + + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" +) + +// Default taint values +const ( + DefaultTaintEffect = corev1.TaintEffectPreferNoSchedule + DefaultTaintKey = "virtual-kubelet.io/provider" +) + +func getEnv(key, defaultValue string) string { + value, found := os.LookupEnv(key) + if found { + return value + } + return defaultValue +} + +// getTaint creates a taint using the provided key/value. +// Taint effect is read from the environment +// The taint key/value may be overwritten by the environment. +func getTaint(key, value string) (*corev1.Taint, error) { + if key == "" { + key = DefaultTaintKey + value = provider + } + + key = getEnv("VKUBELET_TAINT_KEY", key) + value = getEnv("VKUBELET_TAINT_VALUE", value) + effectEnv := getEnv("VKUBELET_TAINT_EFFECT", string(DefaultTaintEffect)) + + var effect corev1.TaintEffect + switch effectEnv { + case "NoSchedule": + effect = corev1.TaintEffectNoSchedule + case "NoExecute": + effect = corev1.TaintEffectNoExecute + case "PreferNoSchedule": + effect = corev1.TaintEffectPreferNoSchedule + default: + return nil, strongerrors.InvalidArgument(errors.Errorf("taint effect %q is not supported", effectEnv)) + } + + return &corev1.Taint{ + Key: key, + Value: value, + Effect: effect, + }, nil +}