Add default provider taint and taint configuration options

This allows for more specificity when setting taint tolerations for
workloads. Three new env variables are introduced:

VKUBELET_TAINT_KEY (defaults to `virtual-kubelet.io/provider`)
VKUBELET_TAINT_VALUE (defaults to provider name)
VKUBELET_TAINT_EFFECT (defaults to `NoSchedule`)

BREAKING CHANGES:
- The default taint key of `azure.com/aci` is now
  `virtual-kubelet.io/provider`.
- Specifying a custom taint key is now done via an environment variable
  rather than the `--taint` command line flag.

charts/virtual-kubelet/Chart.yaml

@@ -1,5 +1,5 @@
 name: virtual-kubelet
-version: 0.2.0
+version: 0.3.0
 appVersion: 0.3
 description: A Helm chart to install virtual kubelet inside a Kubernetes cluster.
 icon: https://avatars2.githubusercontent.com/u/34250142

charts/virtual-kubelet/templates/deployment.yaml

@@ -28,6 +28,12 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: status.podIP
+        - name: VKUBELET_TAINT_KEY
+          value: {{ .Values.taint.key }}
+        - name: VKUBELET_TAINT_VALUE
+          value: {{ tpl .Values.taint.value $ }}
+        - name: VKUBELET_TAINT_EFFECT
+          value: {{ .Values.taint.effect }}
 {{- if eq .Values.provider "azure" }}
 {{- with .Values.providers.azure }}
 {{- if .loganalytics.enabled }}
@@ -73,11 +79,13 @@ spec:
 {{- end }}
         command: ["virtual-kubelet"]
         args: [
+{{- if not .Values.taint.enabled }}
+          "--disable-taint", "true",
+{{- end }}
           "--provider", "{{ required "provider is required" .Values.provider }}",
           "--namespace", "{{ .Values.monitoredNamespace }}",
           "--nodename", "{{ required "nodeName is required" .Values.nodeName }}",
-          "--os", "{{ .Values.nodeOsType }}",
-          "--taint", "{{ .Values.nodeTaint }}"
+          "--os", "{{ .Values.nodeOsType }}"
         ]
       volumes:
       - name: credentials

charts/virtual-kubelet/templates/tests/helloworld.yaml

@@ -23,5 +23,8 @@ spec:
     kubernetes.io/hostname: "{{ .Values.nodeName }}"
   restartPolicy: Never
   tolerations:
-  - key: "{{ .Values.nodeTaint }}"
-    effect: NoSchedule
+{{- if .Values.taint.enabled }}
+  - key: "{{ .Values.taint.key }}"
+    value: "{{ tpl .Values.taint.value $ }}"
+    effect: "{{ .Values.taint.effect }}"
+{{- end }}

charts/virtual-kubelet/values.yaml

@@ -6,12 +6,18 @@ image:
 ## `provider` should be one of aws, azure, azurebatch, etc...
 provider: 
 nodeName: "virtual-kubelet"
-nodeTaint: "azure.com/aci"
 nodeOsType: "Linux"
 monitoredNamespace: ""
 apiserverCert: 
 apiserverKey: 
 
+taint:
+  enabled: true
+  key: virtual-kubelet.io/provider
+  value: "{{ .Values.provider }}"
+  ## `effect` must be `NoSchedule`, `PreferNoSchedule` or `NoExecute`.
+  effect: NoSchedule
+
 providers:
   azure:
     ## Set to true if deploying to Azure Kubernetes Service (AKS), otherwise false

cmd/root.go

@@ -36,7 +36,7 @@ var nodeName string
 var operatingSystem string
 var provider string
 var providerConfig string
-var taint string
+var disableTaint bool
 
 // RootCmd represents the base command when called without any subcommands
 var RootCmd = &cobra.Command{
@@ -47,7 +47,7 @@ backend implementation allowing users to create kubernetes nodes without running
 This allows users to schedule kubernetes workloads on nodes that aren't running Kubernetes.`,
 	Run: func(cmd *cobra.Command, args []string) {
 		fmt.Println(kubeConfig)
-		f, err := vkubelet.New(nodeName, operatingSystem, kubeNamespace, kubeConfig, taint, provider, providerConfig)
+		f, err := vkubelet.New(nodeName, operatingSystem, kubeNamespace, kubeConfig, provider, providerConfig, disableTaint)
 		if err != nil {
 			log.Fatal(err)
 		}
@@ -82,7 +82,7 @@ func init() {
 	RootCmd.PersistentFlags().StringVar(&nodeName, "nodename", defaultNodeName, "kubernetes node name")
 	RootCmd.PersistentFlags().StringVar(&operatingSystem, "os", "Linux", "Operating System (Linux/Windows)")
 	RootCmd.PersistentFlags().StringVar(&provider, "provider", "", "cloud provider")
-	RootCmd.PersistentFlags().StringVar(&taint, "taint", "", "apply taint to node, making scheduling explicit")
+	RootCmd.PersistentFlags().BoolVar(&disableTaint, "disable-taint", false, "disable the virtual-kubelet node taint")
 	RootCmd.PersistentFlags().StringVar(&providerConfig, "provider-config", "", "cloud provider configuration file")
 
 	// Cobra also supports local flags, which will only run

examples/busyecho.yaml

@@ -25,5 +25,5 @@ spec:
     beta.kubernetes.io/os: linux
     type: virtual-kubelet
   tolerations:
-  - key: azure.com/aci
-    effect: NoSchedule
+  - key: virtual-kubelet.io/provider
+    operator: Exists

examples/iis-pod.yaml

@@ -27,5 +27,5 @@ spec:
     type: virtual-kubelet
   automountServiceAccountToken: false
   tolerations:
-  - key: azure.com/aci
-    effect: NoSchedule
+  - key: virtual-kubelet.io/provider
+    operator: Exists

examples/nanoserver.yaml

@@ -16,5 +16,5 @@ spec:
     type: virtual-kubelet
   automountServiceAccountToken: false
   tolerations:
-  - key: azure.com/aci
-    effect: NoSchedule
+  - key: virtual-kubelet.io/provider
+    operator: Exists

examples/nginx-pod.yaml

@@ -19,5 +19,5 @@ spec:
     beta.kubernetes.io/os: linux
     type: virtual-kubelet
   tolerations:
-  - key: azure.com/aci
-    effect: NoSchedule
+  - key: virtual-kubelet.io/provider
+    operator: Exists

examples/pause.yaml

@@ -23,5 +23,5 @@ spec:
     beta.kubernetes.io/os: linux
     type: virtual-kubelet
   tolerations:
-  - key: azure.com/aci
-    effect: NoSchedule
+  - key: virtual-kubelet.io/provider
+    operator: Exists

providers/azure/README.md

@@ -320,7 +320,8 @@ spec:
   dnsPolicy: ClusterFirst
   nodeName: virtual-kubelet-myconnector-linux
   tolerations:
-  - key: azure.com/aci
+  - key: virtual-kubelet.io/provider
+    value: azure
     effect: NoSchedule
 ```
 
@@ -328,7 +329,8 @@ Notice that Virtual-Kubelet nodes are tainted by default to avoid unexpected pod
 
 ```
   tolerations:
-  - key: azure.com/aci
+  - key: virtual-kubelet.io/provider
+    value: azure
     effect: NoSchedule
 ```
 

vkubelet/vkubelet.go

@@ -30,14 +30,23 @@ type Server struct {
 	nodeName        string
 	namespace       string
 	k8sClient       *kubernetes.Clientset
-	taint           string
+	taint           corev1.Taint
+	disableTaint    bool
 	provider        Provider
 	podWatcher      watch.Interface
 	resourceManager *manager.ResourceManager
 }
 
+func getEnv(key, defaultValue string) string {
+	value, found := os.LookupEnv(key)
+	if found {
+		return value
+	}
+	return defaultValue
+}
+
 // New creates a new virtual-kubelet server.
-func New(nodeName, operatingSystem, namespace, kubeConfig, taint, provider, providerConfig string) (*Server, error) {
+func New(nodeName, operatingSystem, namespace, kubeConfig, provider, providerConfig string, disableTaint bool) (*Server, error) {
 	var config *rest.Config
 
 	// Check if the kubeConfig file exists.
@@ -71,6 +80,27 @@ func New(nodeName, operatingSystem, namespace, kubeConfig, taint, provider, prov
 
 	internalIP := os.Getenv("VKUBELET_POD_IP")
 
+	vkTaintKey := getEnv("VKUBELET_TAINT_KEY", "virtual-kubelet.io/provider")
+	vkTaintValue := getEnv("VKUBELET_TAINT_VALUE", provider)
+	vkTaintEffectEnv := getEnv("VKUBELET_TAINT_EFFECT", "NoSchedule")
+	var vkTaintEffect corev1.TaintEffect
+	switch vkTaintEffectEnv {
+	case "NoSchedule":
+		vkTaintEffect = corev1.TaintEffectNoSchedule
+	case "NoExecute":
+		vkTaintEffect = corev1.TaintEffectNoExecute
+	case "PreferNoSchedule":
+		vkTaintEffect = corev1.TaintEffectPreferNoSchedule
+	default:
+		fmt.Printf("Taint effect '%s' is not supported\n", vkTaintEffectEnv)
+	}
+
+	taint := corev1.Taint{
+		Key:    vkTaintKey,
+		Value:  vkTaintValue,
+		Effect: vkTaintEffect,
+	}
+
 	p, err = lookupProvider(provider, providerConfig, rm, nodeName, operatingSystem, internalIP, daemonEndpointPort)
 	if err != nil {
 		return nil, err
@@ -80,6 +110,7 @@ func New(nodeName, operatingSystem, namespace, kubeConfig, taint, provider, prov
 		namespace:       namespace,
 		nodeName:        nodeName,
 		taint:           taint,
+		disableTaint:    disableTaint,
 		k8sClient:       clientset,
 		resourceManager: rm,
 		provider:        p,
@@ -106,11 +137,8 @@ func New(nodeName, operatingSystem, namespace, kubeConfig, taint, provider, prov
 func (s *Server) registerNode() error {
 	taints := make([]corev1.Taint, 0)
 
-	if s.taint != "" {
-		taints = append(taints, corev1.Taint{
-			Key:    s.taint,
-			Effect: corev1.TaintEffectNoSchedule,
-		})
+	if !s.disableTaint {
+		taints = append(taints, s.taint)
 	}
 
 	node := &corev1.Node{