diff --git a/charts/virtual-kubelet-0.3.0.tgz b/charts/virtual-kubelet-0.3.0.tgz new file mode 100644 index 000000000..084e0f1a6 Binary files /dev/null and b/charts/virtual-kubelet-0.3.0.tgz differ diff --git a/charts/virtual-kubelet-latest.tgz b/charts/virtual-kubelet-latest.tgz index 7ec58eb1f..084e0f1a6 100644 Binary files a/charts/virtual-kubelet-latest.tgz and b/charts/virtual-kubelet-latest.tgz differ diff --git a/charts/virtual-kubelet/Chart.yaml b/charts/virtual-kubelet/Chart.yaml index fddfc59d8..c1334eaf2 100644 --- a/charts/virtual-kubelet/Chart.yaml +++ b/charts/virtual-kubelet/Chart.yaml @@ -1,5 +1,5 @@ name: virtual-kubelet -version: 0.2.0 +version: 0.3.0 appVersion: 0.3 description: A Helm chart to install virtual kubelet inside a Kubernetes cluster. icon: https://avatars2.githubusercontent.com/u/34250142 diff --git a/charts/virtual-kubelet/templates/deployment.yaml b/charts/virtual-kubelet/templates/deployment.yaml index 5470efe2c..533883576 100644 --- a/charts/virtual-kubelet/templates/deployment.yaml +++ b/charts/virtual-kubelet/templates/deployment.yaml @@ -28,6 +28,12 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP + - name: VKUBELET_TAINT_KEY + value: {{ .Values.taint.key }} + - name: VKUBELET_TAINT_VALUE + value: {{ tpl .Values.taint.value $ }} + - name: VKUBELET_TAINT_EFFECT + value: {{ .Values.taint.effect }} {{- if eq .Values.provider "azure" }} {{- with .Values.providers.azure }} {{- if .loganalytics.enabled }} @@ -73,11 +79,13 @@ spec: {{- end }} command: ["virtual-kubelet"] args: [ +{{- if not .Values.taint.enabled }} + "--disable-taint", "true", +{{- end }} "--provider", "{{ required "provider is required" .Values.provider }}", "--namespace", "{{ .Values.monitoredNamespace }}", "--nodename", "{{ required "nodeName is required" .Values.nodeName }}", - "--os", "{{ .Values.nodeOsType }}", - "--taint", "{{ .Values.nodeTaint }}" + "--os", "{{ .Values.nodeOsType }}" ] volumes: - name: credentials diff --git a/charts/virtual-kubelet/templates/tests/helloworld.yaml b/charts/virtual-kubelet/templates/tests/helloworld.yaml index 4ce276f0f..5704a0c79 100644 --- a/charts/virtual-kubelet/templates/tests/helloworld.yaml +++ b/charts/virtual-kubelet/templates/tests/helloworld.yaml @@ -23,5 +23,8 @@ spec: kubernetes.io/hostname: "{{ .Values.nodeName }}" restartPolicy: Never tolerations: - - key: "{{ .Values.nodeTaint }}" - effect: NoSchedule +{{- if .Values.taint.enabled }} + - key: "{{ .Values.taint.key }}" + value: "{{ tpl .Values.taint.value $ }}" + effect: "{{ .Values.taint.effect }}" +{{- end }} diff --git a/charts/virtual-kubelet/values.yaml b/charts/virtual-kubelet/values.yaml index f126e0806..6009bd4fe 100644 --- a/charts/virtual-kubelet/values.yaml +++ b/charts/virtual-kubelet/values.yaml @@ -6,12 +6,18 @@ image: ## `provider` should be one of aws, azure, azurebatch, etc... provider: nodeName: "virtual-kubelet" -nodeTaint: "azure.com/aci" nodeOsType: "Linux" monitoredNamespace: "" apiserverCert: apiserverKey: +taint: + enabled: true + key: virtual-kubelet.io/provider + value: "{{ .Values.provider }}" + ## `effect` must be `NoSchedule`, `PreferNoSchedule` or `NoExecute`. + effect: NoSchedule + providers: azure: ## Set to true if deploying to Azure Kubernetes Service (AKS), otherwise false diff --git a/cmd/root.go b/cmd/root.go index 3b703571a..4013337be 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -36,7 +36,8 @@ var nodeName string var operatingSystem string var provider string var providerConfig string -var taint string +var taintKey string +var disableTaint bool // RootCmd represents the base command when called without any subcommands var RootCmd = &cobra.Command{ @@ -47,7 +48,7 @@ backend implementation allowing users to create kubernetes nodes without running This allows users to schedule kubernetes workloads on nodes that aren't running Kubernetes.`, Run: func(cmd *cobra.Command, args []string) { fmt.Println(kubeConfig) - f, err := vkubelet.New(nodeName, operatingSystem, kubeNamespace, kubeConfig, taint, provider, providerConfig) + f, err := vkubelet.New(nodeName, operatingSystem, kubeNamespace, kubeConfig, provider, providerConfig, taintKey, disableTaint) if err != nil { log.Fatal(err) } @@ -82,8 +83,10 @@ func init() { RootCmd.PersistentFlags().StringVar(&nodeName, "nodename", defaultNodeName, "kubernetes node name") RootCmd.PersistentFlags().StringVar(&operatingSystem, "os", "Linux", "Operating System (Linux/Windows)") RootCmd.PersistentFlags().StringVar(&provider, "provider", "", "cloud provider") - RootCmd.PersistentFlags().StringVar(&taint, "taint", "", "apply taint to node, making scheduling explicit") + RootCmd.PersistentFlags().BoolVar(&disableTaint, "disable-taint", false, "disable the virtual-kubelet node taint") RootCmd.PersistentFlags().StringVar(&providerConfig, "provider-config", "", "cloud provider configuration file") + RootCmd.PersistentFlags().StringVar(&taintKey, "taint", "", "Set node taint key") + RootCmd.PersistentFlags().MarkDeprecated("taint", "Taint key should now be configured using the VK_TAINT_KEY environment variable") // Cobra also supports local flags, which will only run // when this action is called directly. diff --git a/examples/busyecho.yaml b/examples/busyecho.yaml index 0c400830a..94a4b080a 100644 --- a/examples/busyecho.yaml +++ b/examples/busyecho.yaml @@ -25,5 +25,5 @@ spec: beta.kubernetes.io/os: linux type: virtual-kubelet tolerations: - - key: azure.com/aci - effect: NoSchedule + - key: virtual-kubelet.io/provider + operator: Exists diff --git a/examples/iis-pod.yaml b/examples/iis-pod.yaml index 230249120..4cba4165a 100644 --- a/examples/iis-pod.yaml +++ b/examples/iis-pod.yaml @@ -27,5 +27,5 @@ spec: type: virtual-kubelet automountServiceAccountToken: false tolerations: - - key: azure.com/aci - effect: NoSchedule + - key: virtual-kubelet.io/provider + operator: Exists diff --git a/examples/nanoserver.yaml b/examples/nanoserver.yaml index 2097c50f5..830aa720a 100644 --- a/examples/nanoserver.yaml +++ b/examples/nanoserver.yaml @@ -16,5 +16,5 @@ spec: type: virtual-kubelet automountServiceAccountToken: false tolerations: - - key: azure.com/aci - effect: NoSchedule + - key: virtual-kubelet.io/provider + operator: Exists diff --git a/examples/nginx-pod.yaml b/examples/nginx-pod.yaml index c4bf7631e..9c541215e 100644 --- a/examples/nginx-pod.yaml +++ b/examples/nginx-pod.yaml @@ -19,5 +19,5 @@ spec: beta.kubernetes.io/os: linux type: virtual-kubelet tolerations: - - key: azure.com/aci - effect: NoSchedule + - key: virtual-kubelet.io/provider + operator: Exists diff --git a/examples/pause.yaml b/examples/pause.yaml index 19e1372e8..6936274b4 100644 --- a/examples/pause.yaml +++ b/examples/pause.yaml @@ -23,5 +23,5 @@ spec: beta.kubernetes.io/os: linux type: virtual-kubelet tolerations: - - key: azure.com/aci - effect: NoSchedule + - key: virtual-kubelet.io/provider + operator: Exists diff --git a/providers/azure/README.md b/providers/azure/README.md index 3d58eba2d..741901207 100644 --- a/providers/azure/README.md +++ b/providers/azure/README.md @@ -320,7 +320,8 @@ spec: dnsPolicy: ClusterFirst nodeName: virtual-kubelet-myconnector-linux tolerations: - - key: azure.com/aci + - key: virtual-kubelet.io/provider + value: azure effect: NoSchedule ``` @@ -328,7 +329,8 @@ Notice that Virtual-Kubelet nodes are tainted by default to avoid unexpected pod ``` tolerations: - - key: azure.com/aci + - key: virtual-kubelet.io/provider + value: azure effect: NoSchedule ``` diff --git a/vkubelet/vkubelet.go b/vkubelet/vkubelet.go index af40d46e8..a1480702d 100644 --- a/vkubelet/vkubelet.go +++ b/vkubelet/vkubelet.go @@ -30,14 +30,23 @@ type Server struct { nodeName string namespace string k8sClient *kubernetes.Clientset - taint string + taint corev1.Taint + disableTaint bool provider Provider podWatcher watch.Interface resourceManager *manager.ResourceManager } +func getEnv(key, defaultValue string) string { + value, found := os.LookupEnv(key) + if found { + return value + } + return defaultValue +} + // New creates a new virtual-kubelet server. -func New(nodeName, operatingSystem, namespace, kubeConfig, taint, provider, providerConfig string) (*Server, error) { +func New(nodeName, operatingSystem, namespace, kubeConfig, provider, providerConfig, taintKey string, disableTaint bool) (*Server, error) { var config *rest.Config // Check if the kubeConfig file exists. @@ -71,6 +80,33 @@ func New(nodeName, operatingSystem, namespace, kubeConfig, taint, provider, prov internalIP := os.Getenv("VKUBELET_POD_IP") + var defaultTaintKey string + if taintKey != "" { + defaultTaintKey = taintKey + } else { + defaultTaintKey = "virtual-kubelet.io/provider" + } + vkTaintKey := getEnv("VKUBELET_TAINT_KEY", defaultTaintKey) + vkTaintValue := getEnv("VKUBELET_TAINT_VALUE", provider) + vkTaintEffectEnv := getEnv("VKUBELET_TAINT_EFFECT", "NoSchedule") + var vkTaintEffect corev1.TaintEffect + switch vkTaintEffectEnv { + case "NoSchedule": + vkTaintEffect = corev1.TaintEffectNoSchedule + case "NoExecute": + vkTaintEffect = corev1.TaintEffectNoExecute + case "PreferNoSchedule": + vkTaintEffect = corev1.TaintEffectPreferNoSchedule + default: + fmt.Printf("Taint effect '%s' is not supported\n", vkTaintEffectEnv) + } + + taint := corev1.Taint{ + Key: vkTaintKey, + Value: vkTaintValue, + Effect: vkTaintEffect, + } + p, err = lookupProvider(provider, providerConfig, rm, nodeName, operatingSystem, internalIP, daemonEndpointPort) if err != nil { return nil, err @@ -80,6 +116,7 @@ func New(nodeName, operatingSystem, namespace, kubeConfig, taint, provider, prov namespace: namespace, nodeName: nodeName, taint: taint, + disableTaint: disableTaint, k8sClient: clientset, resourceManager: rm, provider: p, @@ -106,11 +143,8 @@ func New(nodeName, operatingSystem, namespace, kubeConfig, taint, provider, prov func (s *Server) registerNode() error { taints := make([]corev1.Taint, 0) - if s.taint != "" { - taints = append(taints, corev1.Taint{ - Key: s.taint, - Effect: corev1.TaintEffectNoSchedule, - }) + if !s.disableTaint { + taints = append(taints, s.taint) } node := &corev1.Node{