wuo

README.md: add go.dev badge

.circleci/config.yml

@@ -1,150 +0,0 @@
-version: 2.0
-jobs:
-  validate:
-    resource_class: xlarge
-    docker:
-      - image: circleci/golang:1.15
-        environment:
-          GO111MODULE: "on"
-          GOPROXY: https://proxy.golang.org
-    working_directory: /go/src/github.com/virtual-kubelet/virtual-kubelet
-    steps:
-      - checkout
-      - restore_cache:
-          keys:
-            - validate-{{ checksum "go.mod" }}-{{ checksum "go.sum" }}
-      - run:
-          name: go vet
-          command: V=1 CI=1 make vet
-      - run:
-          name: Lint
-          command: make lint
-      - run:
-          name: Dependencies
-          command: scripts/validate/gomod.sh
-      - save_cache:
-          key: validate-{{ checksum "go.mod" }}-{{ checksum "go.sum" }}
-          paths:
-            - "/go/pkg/mod"
-
-  test:
-    resource_class: xlarge
-    docker:
-      - image: circleci/golang:1.15
-        environment:
-          GO111MODULE: "on"
-    working_directory: /go/src/github.com/virtual-kubelet/virtual-kubelet
-    steps:
-      - checkout
-      - restore_cache:
-          keys:
-            - test-{{ checksum "go.mod" }}-{{ checksum "go.sum" }}
-      - run:
-          name: Build
-          command: V=1 make build
-      - run:
-          name: Tests
-          command: V=1 CI=1 make test envtest
-      - save_cache:
-          key: test-{{ checksum "go.mod" }}-{{ checksum "go.sum" }}
-          paths:
-            - "/go/pkg/mod"
-
-  e2e:
-    machine:
-      image: ubuntu-1604:202010-01
-    working_directory: /home/circleci/go/src/github.com/virtual-kubelet/virtual-kubelet
-    environment:
-      CHANGE_MINIKUBE_NONE_USER: true
-      GOPATH: /home/circleci/go
-      KUBECONFIG: /home/circleci/.kube/config
-      KUBERNETES_VERSION: v1.20.1
-      MINIKUBE_HOME: /home/circleci
-      MINIKUBE_VERSION: v1.16.0
-      MINIKUBE_WANTUPDATENOTIFICATION: false
-      MINIKUBE_WANTREPORTERRORPROMPT: false
-      SKAFFOLD_VERSION: v1.17.2
-      GO111MODULE: "on"
-    steps:
-      - checkout
-      - run:
-          name: Install kubectl
-          command: |
-            curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/${KUBERNETES_VERSION}/bin/linux/amd64/kubectl
-            chmod +x kubectl
-            sudo mv kubectl /usr/local/bin/
-            mkdir -p ${HOME}/.kube
-            touch ${HOME}/.kube/config
-      - run:
-          name: Install Skaffold
-          command: |
-            curl -Lo skaffold https://storage.googleapis.com/skaffold/releases/${SKAFFOLD_VERSION}/skaffold-linux-amd64
-            chmod +x skaffold
-            sudo mv skaffold /usr/local/bin/
-      - run:
-          name: Install Minikube dependencies
-          command: |
-            sudo apt-get update && sudo apt-get install -y apt-transport-https curl
-            curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -
-            cat <<EOF | sudo tee /etc/apt/sources.list.d/kubernetes.list
-            deb https://apt.kubernetes.io/ kubernetes-xenial main
-            EOF
-            sudo apt-get update
-            sudo apt-get install -y kubelet # systemd unit is disabled
-      - run:
-          name: Install Minikube
-          command: |
-            curl -Lo minikube https://storage.googleapis.com/minikube/releases/${MINIKUBE_VERSION}/minikube-linux-amd64
-            chmod +x minikube
-            sudo mv minikube /usr/local/bin/
-      - run:
-          name: Start Minikube
-          command: |
-            sudo -E minikube start --vm-driver=none --cpus 2 --memory 2048 --kubernetes-version=${KUBERNETES_VERSION}
-      - run:
-          name: Wait for Minikube
-          command: |
-            JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}';
-            until kubectl get nodes -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do
-              sleep 1;
-            done
-      - run:
-          name: Watch pods
-          command: kubectl get pods -o json --watch
-          background: true
-      - run:
-          name: Watch nodes
-          command: kubectl get nodes -o json --watch
-          background: true
-      - restore_cache:
-          keys:
-            - e2e-{{ checksum "go.mod" }}-{{ checksum "go.sum" }}-2
-      - run:
-          name: Run the end-to-end test suite
-          command: |
-            mkdir $HOME/.go
-            export PATH=$HOME/.go/bin:${PATH}
-            curl -fsSL -o "/tmp/go.tar.gz" "https://dl.google.com/go/go1.15.6.linux-amd64.tar.gz"
-            tar -C $HOME/.go --strip-components=1 -xzf "/tmp/go.tar.gz"
-            go version
-            make e2e
-      - save_cache:
-          key: e2e-{{ checksum "go.mod" }}-{{ checksum "go.sum" }}-2
-          paths:
-            - "/home/circleci/go/pkg/mod"
-      - run:
-          name: Collect logs on failure from vkubelet-mock-0
-          command: |
-            kubectl logs vkubelet-mock-0
-          when: on_fail
-
-workflows:
-  version: 2
-  validate_and_test:
-    jobs:
-      - validate
-      - test
-      - e2e:
-          requires:
-            - validate
-            - test

.dockerignore

@@ -1,4 +1,6 @@
 .vscode
 private.env
 *.private.*
-providers/azurebatch/deployment/
+providers/azurebatch/deployment/
+Dockerfile
+.dockerignore

.github/dependabot.yml

@@ -0,0 +1,10 @@
+version: 2
+updates:
+  - package-ecosystem: "gomod"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"

.github/workflows/ci.yml

@@ -0,0 +1,110 @@
+name: CI
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+
+env:
+  GO_VERSION: "1.18"
+
+jobs:
+  lint:
+    name: Lint
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+      - uses: actions/setup-go@v3
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - uses: actions/checkout@v3
+      - uses: golangci/golangci-lint-action@v3
+        with:
+          version: v1.48.0
+          args: --timeout=5m
+
+  unit-tests:
+    name: Unit Tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+      - uses: actions/setup-go@v3
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - uses: actions/checkout@v3
+      - name: Run Tests
+        run: make test
+
+  env-tests:
+    name: Envtest Tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+      - uses: actions/setup-go@v3
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - uses: actions/checkout@v3
+      - name: Run Tests
+        run: make envtest
+
+  e2e:
+    name: E2E
+    runs-on: ubuntu-22.04
+    timeout-minutes: 10
+    env:
+      CHANGE_MINIKUBE_NONE_USER: true
+      KUBERNETES_VERSION: v1.20.1
+      MINIKUBE_HOME: /home/circleci
+      MINIKUBE_VERSION: v1.16.0
+      MINIKUBE_WANTUPDATENOTIFICATION: false
+      MINIKUBE_WANTREPORTERRORPROMPT: false
+      SKAFFOLD_VERSION: v1.17.2
+      GO111MODULE: "on"
+
+    steps:
+      - uses: actions/setup-go@v3
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Checkout repository
+        uses: actions/checkout@v3
+      - name: Install Skaffold
+        run: |
+          curl -Lo skaffold https://storage.googleapis.com/skaffold/releases/${SKAFFOLD_VERSION}/skaffold-linux-amd64
+          chmod +x skaffold
+          sudo mv skaffold /usr/local/bin/
+          echo /usr/local/bin >> $GITHUB_PATH
+      - name: Install Minikube dependencies
+        run: |
+          sudo apt-get update && sudo apt-get install -y apt-transport-https curl
+          curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -
+          cat <<EOF | sudo tee /etc/apt/sources.list.d/kubernetes.list
+          deb https://apt.kubernetes.io/ kubernetes-xenial main
+          EOF
+          sudo apt-get update
+          sudo apt-get install -y kubelet # systemd unit is disabled
+      - name: Install Minikube
+        run: |
+          curl -Lo minikube https://storage.googleapis.com/minikube/releases/${MINIKUBE_VERSION}/minikube-linux-amd64
+          chmod +x minikube
+          sudo mv minikube /usr/local/bin/
+      - name: Start Minikube
+        run: |
+          sudo -E PATH=$PATH minikube start --vm-driver=none --cpus 2 --memory 2048 --kubernetes-version=${KUBERNETES_VERSION}
+      - name: Wait for Minikube
+        run: |
+          JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}';
+          until kubectl get nodes -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do
+            sleep 1;
+          done
+      - name: Run Tests
+        run: make e2e

.github/workflows/codeql-analysis.yml

@@ -1,56 +1,59 @@
 name: "CodeQL"
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 on:
   push:
-    branches: [ master ]
-  pull_request:
-    # The branches below must be a subset of the branches above
-    branches: [ master ]
+    branches: [master]
   schedule:
-    - cron: '19 18 * * 3'
+    - cron: "19 18 * * 3"
 
 jobs:
   analyze:
     name: Analyze
     runs-on: ubuntu-latest
+    permissions:
+      security-events: write
 
     strategy:
       fail-fast: false
       matrix:
-        language: [ 'go' ]
+        language: ["go"]
         # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
         # Learn more:
         # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
 
     steps:
-    - name: Checkout repository
-      uses: actions/checkout@v2
+      - name: Checkout repository
+        uses: actions/checkout@v3
 
-    # Initializes the CodeQL tools for scanning.
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
-      with:
-        languages: ${{ matrix.language }}
-        # If you wish to specify custom queries, you can do so here or in a config file.
-        # By default, queries listed here will override any specified in a config file.
-        # Prefix the list here with "+" to use these queries and those in the config file.
-        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+      # Initializes the CodeQL tools for scanning.
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v2
+        with:
+          languages: ${{ matrix.language }}
+          # If you wish to specify custom queries, you can do so here or in a config file.
+          # By default, queries listed here will override any specified in a config file.
+          # Prefix the list here with "+" to use these queries and those in the config file.
+          # queries: ./path/to/local/query, your-org/your-repo/queries@main
 
-    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
-    # If this step fails, then you should remove it and run the build manually (see below)
-    - name: Autobuild
-      uses: github/codeql-action/autobuild@v1
+      # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+      # If this step fails, then you should remove it and run the build manually (see below)
+      - name: Autobuild
+        uses: github/codeql-action/autobuild@v2
 
-    # ℹ️ Command-line programs to run using the OS shell.
-    # 📚 https://git.io/JvXDl
+      # ℹ️ Command-line programs to run using the OS shell.
+      # 📚 https://git.io/JvXDl
 
-    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
-    #    and modify them (or add more) to build your code if your project
-    #    uses a compiled language
+      # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+      #    and modify them (or add more) to build your code if your project
+      #    uses a compiled language
 
-    #- run: |
-    #   make bootstrap
-    #   make release
+      #- run: |
+      #   make bootstrap
+      #   make release
 
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v2

.golangci.yml

@@ -2,28 +2,38 @@ linter-settings:
   lll:
     line-length: 200
 
+timeout: 10m
+
 run:
   skip-dirs:
     # This directory contains copy code from upstream kubernetes/kubernetes, skip it.
     - internal/kubernetes
+    # This is mostly copied from upstream, rather than fixing that code here just ignore the errors.
+    - internal/podutils
 
 linters:
   enable:
     - errcheck
-    - govet
-    - ineffassign
-    - golint
-    - goconst
-    - goimports
-    - unused
+    - structcheck
     - varcheck
-    - deadcode
+    - staticcheck
+    - unconvert
+    - gofmt
+    - goimports
+    - ineffassign
+    - vet
+    - unused
     - misspell
-    - nolintlint
-    - gocritic
+    - gosec
+    - exportloopref # Checks for pointers to enclosing loop variables
+    - tenv # Detects using os.Setenv instead of t.Setenv since Go 1.17
 
+linters-settings:
+  gosec:
+    excludes:
+      - G304
 issues:
   exclude-use-default: false
   exclude:
     # EXC0001 errcheck: Almost all programs ignore errors on these functions and in most cases it's ok
-    - Error return value of .((os\.)?std(out|err)\..*|.*Close|.*Flush|os\.Remove(All)?|.*print(f|ln)?|os\.(Un)?Setenv). is not checked
+    - Error return value of .((os\.)?std(out|err)\..*|.*Close|.*Flush|os\.Remove(All)?|.*print(f|ln)?|os\.(Un)?Setenv). (is not checked|Errors unhandled)

Dockerfile

@@ -1,4 +1,6 @@
-FROM golang:1.15 as builder
+ARG GOLANG_CI_LINT_VERSION
+
+FROM golang:1.18 as builder
 ENV PATH /go/bin:/usr/local/go/bin:$PATH
 ENV GOPATH /go
 COPY . /go/src/github.com/virtual-kubelet/virtual-kubelet
@@ -7,6 +9,22 @@ ARG BUILD_TAGS=""
 RUN make VK_BUILD_TAGS="${BUILD_TAGS}" build
 RUN cp bin/virtual-kubelet /usr/bin/virtual-kubelet
 
+FROM golangci/golangci-lint:${GOLANG_CI_LINT_VERSION} as lint
+WORKDIR /app
+COPY go.mod ./
+COPY go.sum ./
+RUN \
+    --mount=type=cache,target=/root/.cache/go-build \
+    --mount=type=cache,target=/go/pkg/mod \
+    go mod download
+COPY . .
+ARG OUT_FORMAT
+RUN \
+    --mount=type=cache,target=/root/.cache/go-build \
+    --mount=type=cache,target=/go/pkg/mod \
+    --mount=type=cache,target=/root/.cache/golangci-lint \
+    golangci-lint run -v --out-format="${OUT_FORMAT:-colored-line-number}"
+
 FROM scratch
 COPY --from=builder /usr/bin/virtual-kubelet /usr/bin/virtual-kubelet
 COPY --from=builder /etc/ssl/certs/ /etc/ssl/certs

Makefile

@@ -5,6 +5,8 @@ exec := $(DOCKER_IMAGE)
 github_repo := virtual-kubelet/virtual-kubelet
 binary := virtual-kubelet
 
+GOTEST ?= go test $(if $V,-v)
+
 export GO111MODULE ?= on
 
 include Makefile.e2e
@@ -71,36 +73,28 @@ vet:
 	@echo "go vet'ing..."
 ifndef CI
 	@echo "go vet'ing Outside CI..."
-	go vet $(allpackages)
+	go vet $(TESTDIRS)
 else
 	@echo "go vet'ing in CI..."
 	mkdir -p test
-	( go vet $(allpackages); echo $$? ) | \
+	( go vet $(TESTDIRS); echo $$? ) | \
        tee test/vet.txt | sed '$$ d'; exit $$(tail -1 test/vet.txt)
 endif
 
 test:
-ifndef CI
-	@echo "Testing..."
-	go test  $(if $V,-v) $(allpackages)
-else
-	@echo "Testing in CI..."
-	mkdir -p test
-	( GODEBUG=cgocheck=2 go test -timeout=9m -v $(allpackages); echo $$? ) | \
-       tee test/output.txt | sed '$$ d'; exit $$(tail -1 test/output.txt)
-endif
+	$(GOTEST) $(TESTDIRS)
 
 list:
 	@echo "List..."
-	@echo $(allpackages)
+	@echo $(TESTDIRS)
 
 cover: gocovmerge
 	@echo "Coverage Report..."
 	@echo "NOTE: make cover does not exit 1 on failure, don't use it to check for tests success!"
 	rm -f .GOPATH/cover/*.out cover/all.merged
 	$(if $V,@echo "-- go test -coverpkg=./... -coverprofile=cover/... ./...")
-	@for MOD in $(allpackages); do \
-        go test -coverpkg=`echo $(allpackages)|tr " " ","` \
+	@for MOD in $(TESTDIRS); do \
+        go test -coverpkg=`echo $(TESTDIRS)|tr " " ","` \
             -coverprofile=cover/unit-`echo $$MOD|tr "/" "_"`.out \
             $$MOD 2>&1 | grep -v "no packages being tested depend on"; \
     done
@@ -142,11 +136,7 @@ VERSION          := $(shell git describe --tags --always --dirty="-dev")
 DATE             := $(shell date -u '+%Y-%m-%d-%H:%M UTC')
 VERSION_FLAGS    := -ldflags='-X "main.buildVersion=$(VERSION)" -X "main.buildTime=$(DATE)"'
 
-# assuming go 1.9 here!!
-_allpackages = $(shell go list ./...)
-
-# memoize allpackages, so that it's executed only once and only if used
-allpackages = $(if $(__allpackages),,$(eval __allpackages := $$(_allpackages)))$(__allpackages)
+TESTDIRS ?= ./...
 
 .PHONY: goimports
 goimports: $(gobin_tool)
@@ -187,12 +177,16 @@ kubebuilder_2.3.1_${TEST_OS}_${TEST_ARCH}: kubebuilder_2.3.1_${TEST_OS}_${TEST_A
 envtest: kubebuilder_2.3.1_${TEST_OS}_${TEST_ARCH}
 	# You can add klog flags for debugging, like: -klog.v=10 -klog.logtostderr
 	# klogv2 flags just wraps our existing logrus.
-	KUBEBUILDER_ASSETS=$(PWD)/kubebuilder_2.3.1_${TEST_OS}_${TEST_ARCH}/bin go test -v -run=TestEnvtest ./node -envtest=true
+	KUBEBUILDER_ASSETS=$(PWD)/kubebuilder_2.3.1_${TEST_OS}_${TEST_ARCH}/bin $(GOTEST) -run=TestEnvtest ./node -envtest=true
 
 .PHONY: fmt
 fmt:
 	goimports -w $(shell go list -f '{{.Dir}}' ./...)
 
+
+export GOLANG_CI_LINT_VERSION ?= v1.48.0
+DOCKER_BUILD ?= docker buildx build
+
 .PHONY: lint
-lint: $(gobin_tool)
-	gobin -run github.com/golangci/golangci-lint/cmd/golangci-lint@v1.33.0 run ./...
+lint:
+	$(DOCKER_BUILD) --target=lint --build-arg GOLANG_CI_LINT_VERSION --build-arg OUT_FORMAT .

Makefile.e2e

@@ -39,7 +39,7 @@ e2e: NODE_NAME := vkubelet-mock-0
 e2e: export VK_BUILD_TAGS += mock_provider
 e2e: e2e.clean bin/e2e/virtual-kubelet skaffold/run
 	@echo Running tests...
-	cd $(PWD)/internal/test/e2e && go test -v -timeout 5m -tags e2e ./... \
+	cd $(PWD)/internal/test/e2e && $(GOTEST) -timeout 5m -tags e2e ./... \
 		-kubeconfig=$(KUBECONFIG) \
 		-namespace=$(NAMESPACE) \
 		-node-name=$(NODE_NAME)

README.md

@@ -1,5 +1,7 @@
 # Virtual Kubelet
 
+[![Go Reference](https://pkg.go.dev/badge/github.com/virtual-kubelet/virtual-kubelet.svg)](https://pkg.go.dev/github.com/virtual-kubelet/virtual-kubelet)
+
 Virtual Kubelet is an open source [Kubernetes kubelet](https://kubernetes.io/docs/reference/generated/kubelet/)
 implementation that masquerades as a kubelet for the purposes of connecting Kubernetes to other APIs.
 This allows the nodes to be backed by other services like ACI, AWS Fargate, [IoT Edge](https://github.com/Azure/iot-edge-virtual-kubelet-provider), [Tensile Kube](https://github.com/virtual-kubelet/tensile-kube) etc. The primary scenario for VK is enabling the extension of the Kubernetes API into serverless container platforms like ACI and Fargate, though we are open to others. However, it should be noted that VK is explicitly not intended to be an alternative to Kubernetes federation.
@@ -23,6 +25,7 @@ The best description is "Kubernetes API on top, programmable back."
     + [AWS Fargate Provider](#aws-fargate-provider)
     + [Elotl Kip](#elotl-kip)
 	+ [HashiCorp Nomad](#hashicorp-nomad-provider)
+    + [Liqo](#liqo-provider)
     + [OpenStack Zun](#openstack-zun-provider)
     + [Tensile Kube Provider](#tensile-kube-provider)
     + [Adding a New Provider via the Provider Interface](#adding-a-new-provider-via-the-provider-interface)
@@ -46,7 +49,7 @@ project to build a custom Kubernetes node agent.
 See godoc for up to date instructions on consuming this project:
 https://godoc.org/github.com/virtual-kubelet/virtual-kubelet
 
-There are implementations available for several provides (listed above), see
+There are implementations available for [several providers](#providers), see
 those repos for details on how to deploy.
 
 ## Current Features
@@ -134,6 +137,12 @@ would on a Kubernetes node.
 
 For detailed instructions, follow the guide [here](https://github.com/virtual-kubelet/nomad/blob/master/README.md).
 
+### Liqo Provider
+
+[Liqo](https://liqo.io) implements a provider for Virtual Kubelet designed to transparently offload pods and services to "peered" Kubernetes remote cluster. Liqo is capable of discovering neighbor clusters (using DNS, mDNS) and "peer" with them, or in other words, establish a relationship to share part of the cluster resources. When a cluster has established a peering, a new instance of the Liqo Virtual Kubelet is spawned to seamlessly extend the capacity of the cluster, by providing an abstraction of the resources of the remote cluster. The provider combined with the Liqo network fabric extends the cluster networking by enabling Pod-to-Pod traffic and multi-cluster east-west services, supporting endpoints on both clusters.
+
+For detailed instruction, follow the guide [here](https://github.com/liqotech/liqo/blob/master/README.md)
+
 ### OpenStack Zun Provider
 
 OpenStack [Zun](https://docs.openstack.org/zun/latest/) provider for Virtual Kubelet connects
@@ -264,6 +273,11 @@ One of the roles of a Kubelet is to accept requests from the API server for
 things like `kubectl logs` and  `kubectl exec`. Helpers for setting this up are
 provided [here](https://godoc.org/github.com/virtual-kubelet/virtual-kubelet/node/api)
 
+#### Scrape Pod metrics
+
+If you want to use HPA(Horizontal Pod Autoscaler) in your cluster, the provider should implement the `GetStatsSummary` function. Then metrics-server will be able to get the metrics of the pods on virtual-kubelet. Otherwise, you may see `No metrics for pod ` on metrics-server, which means the metrics of the pods on virtual-kubelet are not collected.
+
+
 ## Testing
 
 ### Unit tests
@@ -297,5 +311,4 @@ Monthly Virtual Kubelet Office Hours are held at 10am PST on the last Thursday o
 
 Our google drive with design specifications and meeting notes are [here](https://drive.google.com/drive/folders/19Ndu11WBCCBDowo9CrrGUHoIfd2L8Ueg?usp=sharing).
 
-We also have a community slack channel named virtual-kubelet in the Kubernetes slack. You can also connect with the Virtual Kubelet community via the [mailing list](virtualkubelet-dev@lists.cncf.io).
-
+We also have a community slack channel named virtual-kubelet in the Kubernetes slack. You can also connect with the Virtual Kubelet community via the [mailing list](https://lists.cncf.io/g/virtualkubelet-dev).

charts/README.md

@@ -1,14 +0,0 @@
-# The Virtual Kubelet Helm chart
-
-Each version of Virtual Kubelet has a dedicated [Helm](https://helm.sh) chart. Those charts are served as static assets directly from GitHub.
-
-## The `index.yaml` file
-
-This subdirectory has an `index.yaml` file, which is necessary for it to act as a Helm chart repository. To re-generate the `index.yaml` file (assuming that you have Helm installed):
-
-```shell
-cd /path/to/virtual-kubelet
-helm repo index charts
-```
-
-The `index.yaml` then needs to be committed to Git and merged to `master`.

charts/index.yaml

@@ -1,231 +0,0 @@
-apiVersion: v1
-entries:
-  virtual-kubelet:
-  - appVersion: "0.5"
-    created: 2019-01-28T11:18:23.622097-08:00
-    description: A Helm chart to install virtual kubelet inside a Kubernetes cluster.
-    digest: b4af3cc07e1a914b862ebcd05facbf155b16e098a138fcd7f5dc8ac715aabfa2
-    icon: https://avatars2.githubusercontent.com/u/34250142
-    maintainers:
-    - email: junjiez@microsoft.com
-      name: Robbie Zhang
-    name: virtual-kubelet
-    sources:
-    - https://github.com/virtual-kubelet/virtual-kubelet
-    urls:
-    - virtual-kubelet-0.5.0.tgz
-    version: 0.5.0
-  - appVersion: "0.5"
-    created: 2019-01-28T11:18:23.62762-08:00
-    description: A Helm chart to install virtual kubelet inside a Kubernetes cluster.
-    digest: b4af3cc07e1a914b862ebcd05facbf155b16e098a138fcd7f5dc8ac715aabfa2
-    icon: https://avatars2.githubusercontent.com/u/34250142
-    maintainers:
-    - email: junjiez@microsoft.com
-      name: Robbie Zhang
-    name: virtual-kubelet
-    sources:
-    - https://github.com/virtual-kubelet/virtual-kubelet
-    urls:
-    - virtual-kubelet-latest.tgz
-    version: 0.5.0
-  - appVersion: "0.4"
-    created: 2019-01-28T11:18:23.62156-08:00
-    description: A Helm chart to install virtual kubelet inside a Kubernetes cluster.
-    digest: ae4b8be9d69129f1002ea2228848ec790ea1280d9ff0f7dd99d0d6e3e13922f2
-    icon: https://avatars2.githubusercontent.com/u/34250142
-    maintainers:
-    - email: junjiez@microsoft.com
-      name: Robbie Zhang
-    name: virtual-kubelet
-    sources:
-    - https://github.com/virtual-kubelet/virtual-kubelet
-    urls:
-    - virtual-kubelet-0.4.0.tgz
-    version: 0.4.0
-  - appVersion: "0.3"
-    created: 2019-01-28T11:18:23.620905-08:00
-    description: A Helm chart to install virtual kubelet inside a Kubernetes cluster.
-    digest: f472f181c0724420d4f12218f8fc7f65d09fa02a8292f418d1537bf71231d643
-    icon: https://avatars2.githubusercontent.com/u/34250142
-    maintainers:
-    - email: junjiez@microsoft.com
-      name: Robbie Zhang
-    name: virtual-kubelet
-    sources:
-    - https://github.com/virtual-kubelet/virtual-kubelet
-    urls:
-    - virtual-kubelet-0.3.0.tgz
-    version: 0.3.0
-  - appVersion: "0.3"
-    created: 2019-01-28T11:18:23.6202-08:00
-    description: A Helm chart to install virtual kubelet inside a Kubernetes cluster.
-    digest: 5a2d0a269620ffe49498686161e853f49761ca4f905577fe1b8e552ab85fcaca
-    icon: https://avatars2.githubusercontent.com/u/34250142
-    maintainers:
-    - email: junjiez@microsoft.com
-      name: Robbie Zhang
-    name: virtual-kubelet
-    sources:
-    - https://github.com/virtual-kubelet/virtual-kubelet
-    urls:
-    - virtual-kubelet-0.2.0.tgz
-    version: 0.2.0
-  - created: 2019-01-28T11:18:23.619614-08:00
-    description: a Helm chart to install virtual kubelet inside a Kubernetes cluster.
-    digest: be2778949548cfb0cebe638cbe044d89045eb34ffc8d62180b86ff2f0f30b323
-    maintainers:
-    - email: junjiez@microsoft.com
-      name: Robbie Zhang
-    name: virtual-kubelet
-    sources:
-    - https://github.com/virtual-kubelet/virtual-kubelet
-    urls:
-    - virtual-kubelet-0.1.3.tgz
-    version: 0.1.3
-  - created: 2019-01-28T11:18:23.618974-08:00
-    description: a Helm chart to install virtual kubelet inside a Kubernetes cluster.
-    digest: b8519c66766d06a68671a2b8940cf44cccdf9321f144dead543f62d16325331e
-    maintainers:
-    - email: junjiez@microsoft.com
-      name: Robbie Zhang
-    name: virtual-kubelet
-    sources:
-    - https://github.com/virtual-kubelet/virtual-kubelet
-    urls:
-    - virtual-kubelet-0.1.2.tgz
-    version: 0.1.2
-  - created: 2019-01-28T11:18:23.618406-08:00
-    description: a Helm chart to install virtual kubelet inside a Kubernetes cluster.
-    digest: b15b3d5acde2cc264b5e8624c3bafcc249440c662ae353ef7012493eb0b6abf6
-    maintainers:
-    - email: junjiez@microsoft.com
-      name: Robbie Zhang
-    name: virtual-kubelet
-    sources:
-    - https://github.com/virtual-kubelet/virtual-kubelet
-    urls:
-    - virtual-kubelet-0.1.1.tgz
-    version: 0.1.1
-  - created: 2019-01-28T11:18:23.617865-08:00
-    description: a Helm chart to install virtual kubelet inside a Kubernetes cluster.
-    digest: 22c60ad2f7ea71abf58d65e52b91c2b9feca1ad6a15091321f7f12e5c43ecf81
-    maintainers:
-    - email: junjiez@microsoft.com
-      name: Robbie Zhang
-    name: virtual-kubelet
-    sources:
-    - https://github.com/virtual-kubelet/virtual-kubelet
-    urls:
-    - virtual-kubelet-0.1.0.tgz
-    version: 0.1.0
-  virtual-kubelet-for-aks:
-  - created: 2019-01-28T11:18:23.622632-08:00
-    description: a Helm chart to install virtual kubelet in an AKS or ACS cluster.
-    digest: 345cda5aeb537129e1ecc3d23c0cf47651454f672694a4402a7bbb5d3f3a91ba
-    maintainers:
-    - email: junjiez@microsoft.com
-      name: Robbie Zhang
-    name: virtual-kubelet-for-aks
-    sources:
-    - https://github.com/virtual-kubelet/virtual-kubelet
-    urls:
-    - virtual-kubelet-for-aks-0.1.10.tgz
-    version: 0.1.10
-  - created: 2019-01-28T11:18:23.627017-08:00
-    description: a Helm chart to install virtual kubelet in an AKS or ACS cluster.
-    digest: 345cda5aeb537129e1ecc3d23c0cf47651454f672694a4402a7bbb5d3f3a91ba
-    maintainers:
-    - email: junjiez@microsoft.com
-      name: Robbie Zhang
-    name: virtual-kubelet-for-aks
-    sources:
-    - https://github.com/virtual-kubelet/virtual-kubelet
-    urls:
-    - virtual-kubelet-for-aks-latest.tgz
-    version: 0.1.10
-  - created: 2019-01-28T11:18:23.626524-08:00
-    description: a Helm chart to install virtual kubelet in an AKS or ACS cluster.
-    digest: 4a30821657ff4ef4522060c9905e5659656c035126a7a9e650dd9bb54621b9eb
-    maintainers:
-    - email: junjiez@microsoft.com
-      name: Robbie Zhang
-    name: virtual-kubelet-for-aks
-    sources:
-    - https://github.com/virtual-kubelet/virtual-kubelet
-    urls:
-    - virtual-kubelet-for-aks-0.1.9.tgz
-    version: 0.1.9
-  - created: 2019-01-28T11:18:23.626081-08:00
-    description: a Helm chart to install virtual kubelet in an AKS or ACS cluster.
-    digest: 18d988bfb4d24b3674c6c690c0445b85cf3969471cfce74f7c49e87483cb497d
-    maintainers:
-    - email: junjiez@microsoft.com
-      name: Robbie Zhang
-    name: virtual-kubelet-for-aks
-    sources:
-    - https://github.com/virtual-kubelet/virtual-kubelet
-    urls:
-    - virtual-kubelet-for-aks-0.1.8.tgz
-    version: 0.1.8
-  - created: 2019-01-28T11:18:23.625442-08:00
-    description: a Helm chart to install virtual kubelet in an AKS or ACS cluster.
-    digest: 21c0719fe330981a170810ee4c3e84375106c176de08894827c8208a66f52112
-    maintainers:
-    - email: junjiez@microsoft.com
-      name: Robbie Zhang
-    name: virtual-kubelet-for-aks
-    sources:
-    - https://github.com/virtual-kubelet/virtual-kubelet
-    urls:
-    - virtual-kubelet-for-aks-0.1.7.tgz
-    version: 0.1.7
-  - created: 2019-01-28T11:18:23.624524-08:00
-    description: a Helm chart to install virtual kubelet in an AKS or ACS cluster.
-    digest: b5e72d03f04113a46350fa800135a67f5af9b4ffe3d6650bdeebd271b885e5aa
-    maintainers:
-    - email: junjiez@microsoft.com
-      name: Robbie Zhang
-    name: virtual-kubelet-for-aks
-    sources:
-    - https://github.com/virtual-kubelet/virtual-kubelet
-    urls:
-    - virtual-kubelet-for-aks-0.1.6.tgz
-    version: 0.1.6
-  - created: 2019-01-28T11:18:23.624071-08:00
-    description: a Helm chart to install virtual kubelet in an AKS or ACS cluster.
-    digest: 5b04abcc63dcc71b5ad25c8368ad1b114accf721e9c475c5a7f962a48fb0ed65
-    maintainers:
-    - email: junjiez@microsoft.com
-      name: Robbie Zhang
-    name: virtual-kubelet-for-aks
-    sources:
-    - https://github.com/virtual-kubelet/virtual-kubelet
-    urls:
-    - virtual-kubelet-for-aks-0.1.5.tgz
-    version: 0.1.5
-  - created: 2019-01-28T11:18:23.623547-08:00
-    description: a Helm chart to install virtual kubelet in an AKS or ACS cluster.
-    digest: b639126041dc4f3d0307f6a678d22021ba149f28612eb0bfc3903c75cf1ea414
-    maintainers:
-    - email: junjiez@microsoft.com
-      name: Robbie Zhang
-    name: virtual-kubelet-for-aks
-    sources:
-    - https://github.com/virtual-kubelet/virtual-kubelet
-    urls:
-    - virtual-kubelet-for-aks-0.1.4.tgz
-    version: 0.1.4
-  - created: 2019-01-28T11:18:23.623097-08:00
-    description: a Helm chart to install virtual kubelet in an AKS or ACS cluster.
-    digest: 1b428fd99c667482681c83b61753e7327b85ec2644ef4fa0e9366492a0e73cd7
-    maintainers:
-    - email: junjiez@microsoft.com
-      name: Robbie Zhang
-    name: virtual-kubelet-for-aks
-    sources:
-    - https://github.com/virtual-kubelet/virtual-kubelet
-    urls:
-    - virtual-kubelet-for-aks-0.1.3.tgz
-    version: 0.1.3
-generated: 2019-01-28T11:18:23.615432-08:00

charts/virtual-kubelet-for-aks/Chart.yaml

@@ -1,8 +0,0 @@
-name: virtual-kubelet-for-aks
-version: 0.1.10
-description: a Helm chart to install virtual kubelet in an AKS or ACS cluster.
-sources:
-  - https://github.com/virtual-kubelet/virtual-kubelet
-maintainers:
-  - name: Robbie Zhang
-    email: junjiez@microsoft.com

charts/virtual-kubelet-for-aks/templates/NOTES.txt

@@ -1,12 +0,0 @@
-The virtual kubelet is getting deployed on your cluster.
-
-To verify that virtual kubelet has started, run:
-
-  kubectl --namespace={{ .Release.Namespace }} get pods -l "app={{ template "fullname" . }}"
-
-{{- if (not .Values.env.apiserverCert) and (not .Values.env.apiserverKey) }}
-
-Note: 
-TLS key pair not provided for VK HTTP listener. A key pair was generated for you. This generated key pair is not suitable for production use.
-
-{{- end }}

charts/virtual-kubelet-for-aks/templates/_helpers.tpl

@@ -1,16 +0,0 @@
-{{/* vim: set filetype=mustache: */}}
-{{/*
-Expand the name of the chart.
-*/}}
-{{- define "name" -}}
-{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
-{{- end -}}
-
-{{/*
-Create a default fully qualified app name.
-We truncate at 24 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
-*/}}
-{{- define "fullname" -}}
-{{- $name := default .Chart.Name .Values.nameOverride -}}
-{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
-{{- end -}}

charts/virtual-kubelet-for-aks/templates/clusterrolebinding.yaml

@@ -1,14 +0,0 @@
-{{ if .Values.rbac.install }}
-apiVersion: "rbac.authorization.k8s.io/{{ .Values.rbac.apiVersion }}"
-kind: ClusterRoleBinding
-metadata:
-  name: {{ template "fullname" . }}
-subjects:
-- kind: ServiceAccount
-  name: {{ template "fullname" . }}
-  namespace: {{ .Release.Namespace }}
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: {{ .Values.rbac.roleRef }}
-{{ end }}

charts/virtual-kubelet-for-aks/templates/deployment.yaml

@@ -1,79 +0,0 @@
-apiVersion: extensions/v1beta1
-kind: Deployment
-metadata:
-  name: {{ template "fullname" . }}
-spec:
-  replicas: 1
-  template:
-    metadata:
-      labels:
-        app: {{ template "fullname" . }}
-    spec:
-      containers:
-      - name: {{ template "fullname" . }}
-        image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
-        imagePullPolicy: {{ .Values.image.pullPolicy }}
-        env:
-        - name: KUBELET_PORT
-          value: "10250"
-        - name: ACS_CREDENTIAL_LOCATION
-          value: /etc/acs/azure.json
-        - name: AZURE_TENANT_ID
-          value: {{ .Values.env.azureTenantId }}
-        - name: AZURE_SUBSCRIPTION_ID
-          value: {{ .Values.env.azureSubscriptionId }}
-        - name: AZURE_CLIENT_ID
-          value: {{ .Values.env.azureClientId }}
-        - name: AZURE_CLIENT_SECRET
-          valueFrom:
-            secretKeyRef:
-              name: {{ template "fullname" . }}
-              key: clientSecret
-        - name: ACI_RESOURCE_GROUP
-          value: {{ .Values.env.aciResourceGroup }}
-        - name: ACI_REGION
-          value: {{ default "westus" .Values.env.aciRegion }}
-        - name: APISERVER_CERT_LOCATION
-          value: /etc/virtual-kubelet/cert.pem
-        - name: APISERVER_KEY_LOCATION
-          value: /etc/virtual-kubelet/key.pem
-        - name: VKUBELET_POD_IP
-          valueFrom:
-            fieldRef:
-              fieldPath: status.podIP
-        - name: ACI_EXTRA_USER_AGENT
-          value: {{ printf "helm-chart/aks/%s/%s" .Chart.Name .Chart.Version }}
-        - name: ACI_SUBNET_NAME
-          value: {{ .Values.env.aciVnetSubnetName }}
-        - name: ACI_SUBNET_CIDR
-          value: {{ .Values.env.aciVnetSubnetCIDR }}
-        - name: MASTER_URI
-          value: {{ .Values.env.masterUri }}
-        - name: CLUSTER_CIDR
-          value: {{ .Values.env.clusterCIDR }}
-        - name: KUBE_DNS_IP
-          value: {{ .Values.env.kubeDnsIP }}
-        {{ if .Values.loganalytics.enabled }}
-        - name: LOG_ANALYTICS_AUTH_LOCATION
-          value: /etc/virtual-kubelet/loganalytics.json
-        {{ end }}
-        volumeMounts:
-        - name: credentials
-          mountPath: "/etc/virtual-kubelet"
-        - name: acs-credential
-          mountPath: "/etc/acs/azure.json"
-        command: ["virtual-kubelet"]
-        args: ["--provider", "azure", "--namespace", {{ default "" .Values.env.monitoredNamespace | quote }}, "--nodename", {{ default "virtual-kubelet" .Values.env.nodeName | quote }} , "--os", {{ default "Linux" .Values.env.nodeOsType | quote }} ]
-      volumes:
-      - name: credentials
-        secret:
-          secretName: {{ template "fullname" . }}
-      - name: acs-credential
-        hostPath:
-          path: /etc/kubernetes/azure.json
-          type: File
-      {{ if .Values.rbac.install }}
-      serviceAccountName: {{ template "fullname" . }}
-      {{ end }}
-      nodeSelector:
-        beta.kubernetes.io/os: linux

charts/virtual-kubelet-for-aks/templates/secrets.yaml

@@ -1,22 +0,0 @@
-apiVersion: v1
-kind: Secret
-metadata:
-  name: {{ template "fullname" . }}
-type: Opaque
-data:
-  {{- if (not .Values.env.apiserverCert) and (not .Values.env.apiserverKey) }}
-  {{- $ca := genCA "virtual-kubelet-ca" 3650 }}
-  {{- $cn := printf "%s-virtual-kubelet-apiserver" .Release.Name }}
-  {{- $altName1 := printf "%s-virtual-kubelet-apiserver.%s" .Release.Name .Release.Namespace }}
-  {{- $altName2 := printf "%s-virtual-kubelet-apiserver.%s.svc" .Release.Name .Release.Namespace }}
-  {{- $cert := genSignedCert $cn nil (list $altName1 $altName2) 3650 $ca }}
-  cert.pem: {{ b64enc $cert.Cert }}
-  key.pem: {{ b64enc $cert.Key }}
-  {{ else }}
-  cert.pem: {{ quote .Values.env.apiserverCert }}
-  key.pem: {{ quote .Values.env.apiserverKey }}
-  {{ end}}
-  clientSecret: {{ default "" .Values.env.azureClientKey | b64enc | quote }}
-  {{ if .Values.loganalytics.enabled }}
-  loganalytics.json: {{ printf "{\"workspaceID\": \"%s\",\"workspaceKey\": \"%s\"}" (required "workspaceID is required for loganalytics" .Values.loganalytics.workspaceID ) (required "workspaceKey is required for loganalytics" .Values.loganalytics.workspaceKey ) | b64enc | quote }}
-  {{ end }}

charts/virtual-kubelet-for-aks/templates/serviceaccount.yaml

@@ -1,6 +0,0 @@
-{{ if .Values.rbac.install }}
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: {{ template "fullname" . }}
-{{ end }}

charts/virtual-kubelet-for-aks/values.yaml

@@ -1,34 +0,0 @@
-image:
-  repository: microsoft/virtual-kubelet
-  tag: latest
-  pullPolicy: Always
-env:
-  azureClientId: 
-  azureClientKey: 
-  azureTenantId: 
-  azureSubscriptionId: 
-  aciResourceGroup: 
-  aciRegion: 
-  nodeName: 
-  nodeTaint: 
-  nodeOsType: 
-  apiserverCert: 
-  apiserverKey: 
-  monitoredNamespace:
-  aciVnetSubnetName:
-  aciVnetSubnetCidr:
-  masterUri:
-  clusterCidr:
-  kubeDnsIp:
-loganalytics:
-  enabled: false
-  workspaceID:
-  workspaceKey:
-
-# Install Default RBAC roles and bindings
-rbac:
-  install: true
-  ## RBAC api version
-  apiVersion: v1beta1
-  # Cluster role reference
-  roleRef: cluster-admin

charts/virtual-kubelet/Chart.yaml

@@ -1,10 +0,0 @@
-name: virtual-kubelet
-version: 0.5.0
-appVersion: 0.5
-description: A Helm chart to install virtual kubelet inside a Kubernetes cluster.
-icon: https://avatars2.githubusercontent.com/u/34250142
-sources:
-  - https://github.com/virtual-kubelet/virtual-kubelet
-maintainers:
-  - name: Robbie Zhang
-    email: junjiez@microsoft.com

charts/virtual-kubelet/templates/NOTES.txt

@@ -1,12 +0,0 @@
-The virtual kubelet is getting deployed on your cluster.
-
-To verify that virtual kubelet has started, run:
-
-  kubectl --namespace={{ .Release.Namespace }} get pods -l "app={{ template "vk.name" . }}"
-
-{{- if (not .Values.apiserverCert) and (not .Values.apiserverKey) }}
-
-Note: 
-TLS key pair not provided for VK HTTP listener. A key pair was generated for you. This generated key pair is not suitable for production use.
-
-{{- end }}

charts/virtual-kubelet/templates/_helpers.tpl

@@ -1,29 +0,0 @@
-{{/* vim: set filetype=mustache: */}}
-{{/*
-Expand the name of the chart.
-*/}}
-{{- define "vk.name" -}}
-{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
-{{- end -}}
-
-{{/*
-Create a default fully qualified app name.
-We truncate at 24 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
-*/}}
-{{- define "vk.fullname" -}}
-{{- $name := default .Chart.Name .Values.nameOverride -}}
-{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
-{{- end -}}
-
-{{/*
-Standard labels for helm resources
-*/}}
-{{- define "vk.labels" -}}
-labels:
-  heritage: "{{ .Release.Service }}"
-  release: "{{ .Release.Name }}"
-  revision: "{{ .Release.Revision }}"
-  chart: "{{ .Chart.Name }}"
-  chartVersion: "{{ .Chart.Version }}"
-  app: {{ template "vk.name" . }}
-{{- end -}}

charts/virtual-kubelet/templates/clusterrolebinding.yaml

@@ -1,15 +0,0 @@
-{{ if .Values.rbac.install }}
-apiVersion: "rbac.authorization.k8s.io/{{ .Values.rbac.apiVersion }}"
-kind: ClusterRoleBinding
-metadata:
-  name: {{ template "vk.fullname" . }}
-{{ include "vk.labels" . | indent 2 }}
-subjects:
-- kind: ServiceAccount
-  name: {{ template "vk.fullname" . }}
-  namespace: {{ .Release.Namespace }}
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: {{ .Values.rbac.roleRef }}
-{{ end }}

charts/virtual-kubelet/templates/deployment.yaml

@@ -1,159 +0,0 @@
-apiVersion: extensions/v1beta1
-kind: Deployment
-metadata:
-  name: {{ template "vk.fullname" . }}
-{{ include "vk.labels" . | indent 2 }}
-    component: kubelet
-spec:
-  replicas: 1
-  template:
-    metadata:
-{{ include "vk.labels" . | indent 6 }}
-        component: kubelet
-      annotations:
-        checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }}
-    spec:
-      containers:
-{{- if eq .Values.trace.exporter "jaeger" }}
-{{- with .Values.traceExporters.jaeger }}
-{{- if eq .endpoint "" }}
-      - name: {{ tpl .name $ }}
-        image: {{ .image.repository}}:{{.image.tag}}
-        imagePullPolicy: {{ .image.pullPolicy }}
-        ports:
-          - containerPort: 16686
-{{- end }}
-{{- end }}
-{{- end }}
-      - name: {{ template "vk.fullname" . }}
-        image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
-        imagePullPolicy: {{ .Values.image.pullPolicy }}
-        env:
-        - name: KUBELET_PORT
-          value: "10250"
-        - name: APISERVER_CERT_LOCATION
-          value: /etc/virtual-kubelet/cert.pem
-        - name: APISERVER_KEY_LOCATION
-          value: /etc/virtual-kubelet/key.pem
-        - name: VKUBELET_POD_IP
-          valueFrom:
-            fieldRef:
-              fieldPath: status.podIP
-        - name: VKUBELET_TAINT_KEY
-          value: {{ .Values.taint.key }}
-        - name: VKUBELET_TAINT_VALUE
-          value: {{ tpl .Values.taint.value $ }}
-        - name: VKUBELET_TAINT_EFFECT
-          value: {{ .Values.taint.effect }}
-{{- if eq (required "You must specify a Virtual Kubelet provider" .Values.provider) "azure" }}
-{{- with .Values.providers.azure }}
-{{- if .loganalytics.enabled }}
-        - name: LOG_ANALYTICS_AUTH_LOCATION
-          value: /etc/virtual-kubelet/loganalytics.json
-        - name: CLUSTER_RESOURCE_ID
-          value:  {{ .loganalytics.clusterResourceId }}
-{{- end }}
-{{- if .targetAKS }}
-        - name: ACS_CREDENTIAL_LOCATION
-          value: /etc/acs/azure.json
-        - name: AZURE_TENANT_ID
-          value: {{ .tenantId }}
-        - name: AZURE_SUBSCRIPTION_ID
-          value: {{ .subscriptionId }}
-        - name: AZURE_CLIENT_ID
-          value: {{ .clientId }}
-        - name: AZURE_CLIENT_SECRET
-          valueFrom:
-            secretKeyRef:
-              name: {{ template "vk.fullname" $ }}
-              key: clientSecret
-        - name: ACI_RESOURCE_GROUP
-          value: {{ .aciResourceGroup }}
-        - name: ACI_REGION
-          value: {{ .aciRegion }}
-        - name: ACI_EXTRA_USER_AGENT
-          value: {{ printf "helm-chart/aks/%s/%s" $.Chart.Name $.Chart.Version }}
-{{- else }}
-        - name: AZURE_AUTH_LOCATION
-          value: /etc/virtual-kubelet/credentials.json
-        - name: ACI_RESOURCE_GROUP
-          value: {{ required "aciResourceGroup is required" .aciResourceGroup }}
-        - name: ACI_REGION
-          value: {{ required "aciRegion is required" .aciRegion }}
-        - name: ACI_EXTRA_USER_AGENT
-          value: {{ printf "helm-chart/other/%s/%s" $.Chart.Name $.Chart.Version }}
-{{- end }}
-{{- if .vnet.enabled }}
-        - name: ACI_SUBNET_NAME
-          value: {{ required "subnetName is required" .vnet.subnetName }}
-        - name: ACI_SUBNET_CIDR
-          value: {{ .vnet.subnetCidr }}
-        - name: MASTER_URI
-          value: {{ required "masterUri is required" .masterUri }}
-        - name: CLUSTER_CIDR
-          value: {{ required "clusterCidr is required" .vnet.clusterCidr }}
-        - name: KUBE_DNS_IP
-          value: {{ required "kubeDnsIp is required" .vnet.kubeDnsIp }}
-{{- else }}
-        - name: MASTER_URI
-          value: {{ .masterUri }}
-{{- end }}          
-{{- end }}
-{{- end }}
-{{- if eq .Values.trace.exporter "jaeger" }}
-        - name: JAEGER_ENDPOINT
-{{- with .Values.traceExporters.jaeger }}
-{{- if eq .endpoint "" }}
-          value: "http://127.0.0.1:14268"
-{{- else }}
-          value: {{.endpoint}}
-{{- end }}
-{{- end }}
-{{- end }}
-        volumeMounts:
-        - name: credentials
-          mountPath: "/etc/virtual-kubelet"
-{{- if eq (required "You must specify a Virtual Kubelet provider" .Values.provider) "azure" }}
-{{- if .Values.providers.azure.targetAKS }}
-        - name: acs-credential
-          mountPath: "/etc/acs/azure.json"
-{{- end }}
-{{- end }}
-        command: ["virtual-kubelet"]
-        args: [
-{{- if not .Values.taint.enabled }}
-          "--disable-taint", "true",
-{{- end }}
-          "--provider", "{{ required "You must specify a Virtual Kubelet provider" .Values.provider }}",
-          "--namespace", "{{ .Values.monitoredNamespace }}",
-          "--nodename", "{{ required "nodeName is required" .Values.nodeName }}",
-          {{- if .Values.logLevel }}
-          "--log-level", "{{.Values.logLevel}}",
-          {{- end }}
-{{- if ne .Values.trace.exporter "" }}
-          "--trace-exporter", "{{ .Values.trace.exporter }}",
-{{- if gt .Values.trace.sampleRate 0.0 }}
-          "--trace-sample-rate", "{{ .Values.trace.sampleRate }}",
-{{- end }}
-{{- $serviceName := tpl .Values.trace.serviceName $ }}
-{{- if ne $serviceName "" }}
-          "--trace-service-name", "{{ $serviceName }}",
-{{- end}}
-{{- end}}
-          "--os", "{{ .Values.nodeOsType }}"
-        ]
-      volumes:
-      - name: credentials
-        secret:
-          secretName: {{ template "vk.fullname" . }}
-{{- if eq (required "You must specify a Virtual Kubelet provider" .Values.provider) "azure" }}
-{{- if .Values.providers.azure.targetAKS }}
-      - name: acs-credential
-        hostPath:
-          path: /etc/kubernetes/azure.json
-          type: File
-{{- end }}
-{{- end }}
-      serviceAccountName: {{ if .Values.rbac.install }} "{{ template "vk.fullname" . }}" {{ end }}
-      nodeSelector:
-        beta.kubernetes.io/os: linux

charts/virtual-kubelet/templates/secret.yaml

@@ -1,31 +0,0 @@
-apiVersion: v1
-kind: Secret
-metadata:
-  name: {{ template "vk.fullname" . }}
-{{ include "vk.labels" . | indent 2 }}
-type: Opaque
-data:
-{{- if (not .Values.apiserverCert) and (not .Values.apiserverKey) }}
-{{- $ca := genCA "virtual-kubelet-ca" 3650 }}
-{{- $cn := printf "%s-virtual-kubelet-apiserver" .Release.Name }}
-{{- $altName1 := printf "%s-virtual-kubelet-apiserver.%s" .Release.Name .Release.Namespace }}
-{{- $altName2 := printf "%s-virtual-kubelet-apiserver.%s.svc" .Release.Name .Release.Namespace }}
-{{- $cert := genSignedCert $cn nil (list $altName1 $altName2) 3650 $ca }}
-  cert.pem: {{ b64enc $cert.Cert }}
-  key.pem: {{ b64enc $cert.Key }}
-{{- else }}
-  cert.pem: {{ quote .Values.apiserverCert }}
-  key.pem: {{ quote .Values.apiserverKey }}
-{{- end }}
-{{- if eq (required "You must specify a Virtual Kubelet provider" .Values.provider) "azure" }}
-{{- with .Values.providers.azure }}
-{{- if .loganalytics.enabled }}
-  loganalytics.json: {{ printf "{\"workspaceID\": \"%s\",\"workspaceKey\": \"%s\"}" (required "workspaceId is required for loganalytics" .loganalytics.workspaceId ) (required "workspaceKey is required for loganalytics" .loganalytics.workspaceKey ) | b64enc | quote }}
-{{- end }}
-{{- if .targetAKS }}
-  clientSecret: {{ default "" .clientKey | b64enc | quote }}
-{{- else }}
-  credentials.json: {{ printf "{ \"clientId\": \"%s\", \"clientSecret\": \"%s\", \"subscriptionId\": \"%s\", \"tenantId\": \"%s\", \"activeDirectoryEndpointUrl\": \"https://login.microsoftonline.com/\", \"resourceManagerEndpointUrl\": \"https://management.azure.com/\", \"activeDirectoryGraphResourceId\": \"https://graph.windows.net/\", \"sqlManagementEndpointUrl\": \"database.windows.net\", \"galleryEndpointUrl\": \"https://gallery.azure.com/\", \"managementEndpointUrl\": \"https://management.core.windows.net/\" }" (default "MISSING" .clientId) (default "MISSING" .clientKey) (default "MISSING" .subscriptionId) (default "MISSING" .tenantId) | b64enc | quote }}
-{{- end }}
-{{- end }}
-{{- end }}

charts/virtual-kubelet/templates/serviceaccount.yaml

@@ -1,7 +0,0 @@
-{{ if .Values.rbac.install }}
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: {{ template "vk.fullname" . }}
-{{ include "vk.labels" . | indent 2 }}
-{{ end }}

charts/virtual-kubelet/templates/tests/helloworld.yaml

@@ -1,30 +0,0 @@
-apiVersion: v1
-kind: Pod
-metadata:
-  name: "{{ .Release.Name }}-{{ .Release.Revision }}-test"
-{{ include "vk.labels" . | indent 2 }}
-    component: test
-  annotations:
-    "helm.sh/hook": test-success
-spec:
-  containers:
-  - image: hello-world:linux
-    imagePullPolicy: Always
-    name: helloworld
-    resources:
-      requests:
-        memory: "0.1G"
-        cpu: 10m
-      limits:
-        memory: "0.1G"
-        cpu: 10m
-  dnsPolicy: ClusterFirst
-  nodeSelector:
-    kubernetes.io/hostname: "{{ .Values.nodeName }}"
-  restartPolicy: Never
-  tolerations:
-{{- if .Values.taint.enabled }}
-  - key: "{{ .Values.taint.key }}"
-    value: "{{ tpl .Values.taint.value $ }}"
-    effect: "{{ .Values.taint.effect }}"
-{{- end }}

charts/virtual-kubelet/values.yaml

@@ -1,65 +0,0 @@
-image:
-  repository: microsoft/virtual-kubelet
-  tag: latest
-  pullPolicy: Always
-
-nodeName: "virtual-kubelet"
-nodeOsType: "Linux"
-monitoredNamespace: ""
-apiserverCert: 
-apiserverKey: 
-logLevel: 
-
-taint:
-  enabled: true
-  key: virtual-kubelet.io/provider
-  value: "{{ .Values.provider }}"
-  ## `effect` must be `NoSchedule`, `PreferNoSchedule` or `NoExecute`.
-  effect: NoSchedule
-
-trace:
-  exporter: ""
-  serviceName: "{{ .Values.nodeName }}"
-  sampleRate: 0
-
-traceExporters:
-  jaeger:
-    name: "{{ .Values.trace.exporter }}"
-    endpoint: ""
-    image:
-      repository: jaegertracing/all-in-one
-      tag: 1.8
-      pullPolicy: Always
-
-providers:
-  azure:
-    ## Set to true if deploying to Azure Kubernetes Service (AKS), otherwise false
-    targetAKS: true
-    clientId: 
-    clientKey: 
-    tenantId: 
-    subscriptionId: 
-    ## `aciResourceGroup` and `aciRegion` are required only for non-AKS deployments
-    aciResourceGroup: 
-    aciRegion: 
-    masterUri: 
-    loganalytics:
-      enabled: false
-      workspaceId: 
-      workspaceKey: 
-      clusterResourceId: 
-    vnet:
-      enabled: false
-      subnetName: 
-      subnetCidr: 
-      clusterCidr: 
-      kubeDnsIp: 
-
-## Install Default RBAC roles and bindings
-rbac:
-  install: true
-  serviceAccountName: virtual-kubelet
-  ## RBAC api version
-  apiVersion: v1beta1
-  ## Cluster role reference
-  roleRef: cluster-admin

cmd/virtual-kubelet/internal/commands/root/flag.go

@@ -22,7 +22,7 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/spf13/pflag"
-	"k8s.io/klog"
+	klog "k8s.io/klog/v2"
 )
 
 type mapVar map[string]string
@@ -59,7 +59,13 @@ func (mv mapVar) Type() string {
 
 func installFlags(flags *pflag.FlagSet, c *Opts) {
 	flags.StringVar(&c.KubeConfigPath, "kubeconfig", c.KubeConfigPath, "kube config file to use for connecting to the Kubernetes API server")
+
 	flags.StringVar(&c.KubeNamespace, "namespace", c.KubeNamespace, "kubernetes namespace (default is 'all')")
+	/* #nosec */
+	flags.MarkDeprecated("namespace", "Nodes must watch for pods in all namespaces. This option is now ignored.") //nolint:errcheck
+	/* #nosec */
+	flags.MarkHidden("namespace") //nolint:errcheck
+
 	flags.StringVar(&c.KubeClusterDomain, "cluster-domain", c.KubeClusterDomain, "kubernetes cluster-domain (default is 'cluster.local')")
 	flags.StringVar(&c.NodeName, "nodename", c.NodeName, "kubernetes node name")
 	flags.StringVar(&c.OperatingSystem, "os", c.OperatingSystem, "Operating System (Linux/Windows)")
@@ -68,11 +74,18 @@ func installFlags(flags *pflag.FlagSet, c *Opts) {
 	flags.StringVar(&c.MetricsAddr, "metrics-addr", c.MetricsAddr, "address to listen for metrics/stats requests")
 
 	flags.StringVar(&c.TaintKey, "taint", c.TaintKey, "Set node taint key")
+
 	flags.BoolVar(&c.DisableTaint, "disable-taint", c.DisableTaint, "disable the virtual-kubelet node taint")
+	/* #nosec */
 	flags.MarkDeprecated("taint", "Taint key should now be configured using the VK_TAINT_KEY environment variable") //nolint:errcheck
 
 	flags.IntVar(&c.PodSyncWorkers, "pod-sync-workers", c.PodSyncWorkers, `set the number of pod synchronization workers`)
+
 	flags.BoolVar(&c.EnableNodeLease, "enable-node-lease", c.EnableNodeLease, `use node leases (1.13) for node heartbeats`)
+	/* #nosec */
+	flags.MarkDeprecated("enable-node-lease", "leases are always enabled") //nolint:errcheck
+	/* #nosec */
+	flags.MarkHidden("enable-node-lease") //nolint:errcheck
 
 	flags.StringSliceVar(&c.TraceExporters, "trace-exporter", c.TraceExporters, fmt.Sprintf("sets the tracing exporter to use, available exporters: %s", AvailableTraceExporters()))
 	flags.StringVar(&c.TraceConfig.ServiceName, "trace-service-name", c.TraceConfig.ServiceName, "sets the name of the service used to register with the trace exporter")

cmd/virtual-kubelet/internal/commands/root/http.go

@@ -15,140 +15,15 @@
 package root
 
 import (
-	"context"
-	"crypto/tls"
 	"fmt"
-	"io"
-	"net"
-	"net/http"
 	"os"
 	"time"
-
-	"github.com/pkg/errors"
-	"github.com/virtual-kubelet/virtual-kubelet/cmd/virtual-kubelet/internal/provider"
-	"github.com/virtual-kubelet/virtual-kubelet/log"
-	"github.com/virtual-kubelet/virtual-kubelet/node/api"
 )
 
-// AcceptedCiphers is the list of accepted TLS ciphers, with known weak ciphers elided
-// Note this list should be a moving target.
-var AcceptedCiphers = []uint16{
-	tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
-	tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
-	tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
-	tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
-
-	tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
-	tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
-	tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
-	tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
-}
-
-func loadTLSConfig(certPath, keyPath string) (*tls.Config, error) {
-	cert, err := tls.LoadX509KeyPair(certPath, keyPath)
-	if err != nil {
-		return nil, errors.Wrap(err, "error loading tls certs")
-	}
-
-	return &tls.Config{
-		Certificates:             []tls.Certificate{cert},
-		MinVersion:               tls.VersionTLS12,
-		PreferServerCipherSuites: true,
-		CipherSuites:             AcceptedCiphers,
-	}, nil
-}
-
-func setupHTTPServer(ctx context.Context, p provider.Provider, cfg *apiServerConfig, getPodsFromKubernetes api.PodListerFunc) (_ func(), retErr error) {
-	var closers []io.Closer
-	cancel := func() {
-		for _, c := range closers {
-			c.Close()
-		}
-	}
-	defer func() {
-		if retErr != nil {
-			cancel()
-		}
-	}()
-
-	if cfg.CertPath == "" || cfg.KeyPath == "" {
-		log.G(ctx).
-			WithField("certPath", cfg.CertPath).
-			WithField("keyPath", cfg.KeyPath).
-			Error("TLS certificates not provided, not setting up pod http server")
-	} else {
-		tlsCfg, err := loadTLSConfig(cfg.CertPath, cfg.KeyPath)
-		if err != nil {
-			return nil, err
-		}
-		l, err := tls.Listen("tcp", cfg.Addr, tlsCfg)
-		if err != nil {
-			return nil, errors.Wrap(err, "error setting up listener for pod http server")
-		}
-
-		mux := http.NewServeMux()
-
-		podRoutes := api.PodHandlerConfig{
-			RunInContainer:        p.RunInContainer,
-			GetContainerLogs:      p.GetContainerLogs,
-			GetPodsFromKubernetes: getPodsFromKubernetes,
-			GetPods:               p.GetPods,
-			StreamIdleTimeout:     cfg.StreamIdleTimeout,
-			StreamCreationTimeout: cfg.StreamCreationTimeout,
-		}
-
-		api.AttachPodRoutes(podRoutes, mux, true)
-
-		s := &http.Server{
-			Handler:   mux,
-			TLSConfig: tlsCfg,
-		}
-		go serveHTTP(ctx, s, l, "pods")
-		closers = append(closers, s)
-	}
-
-	if cfg.MetricsAddr == "" {
-		log.G(ctx).Info("Pod metrics server not setup due to empty metrics address")
-	} else {
-		l, err := net.Listen("tcp", cfg.MetricsAddr)
-		if err != nil {
-			return nil, errors.Wrap(err, "could not setup listener for pod metrics http server")
-		}
-
-		mux := http.NewServeMux()
-
-		var summaryHandlerFunc api.PodStatsSummaryHandlerFunc
-		if mp, ok := p.(provider.PodMetricsProvider); ok {
-			summaryHandlerFunc = mp.GetStatsSummary
-		}
-		podMetricsRoutes := api.PodMetricsConfig{
-			GetStatsSummary: summaryHandlerFunc,
-		}
-		api.AttachPodMetricsRoutes(podMetricsRoutes, mux)
-		s := &http.Server{
-			Handler: mux,
-		}
-		go serveHTTP(ctx, s, l, "pod metrics")
-		closers = append(closers, s)
-	}
-
-	return cancel, nil
-}
-
-func serveHTTP(ctx context.Context, s *http.Server, l net.Listener, name string) {
-	if err := s.Serve(l); err != nil {
-		select {
-		case <-ctx.Done():
-		default:
-			log.G(ctx).WithError(err).Errorf("Error setting up %s http server", name)
-		}
-	}
-	l.Close()
-}
-
 type apiServerConfig struct {
 	CertPath              string
 	KeyPath               string
+	CACertPath            string
 	Addr                  string
 	MetricsAddr           string
 	StreamIdleTimeout     time.Duration
@@ -157,8 +32,9 @@ type apiServerConfig struct {
 
 func getAPIConfig(c Opts) (*apiServerConfig, error) {
 	config := apiServerConfig{
-		CertPath: os.Getenv("APISERVER_CERT_LOCATION"),
-		KeyPath:  os.Getenv("APISERVER_KEY_LOCATION"),
+		CertPath:   os.Getenv("APISERVER_CERT_LOCATION"),
+		KeyPath:    os.Getenv("APISERVER_KEY_LOCATION"),
+		CACertPath: os.Getenv("APISERVER_CA_CERT_LOCATION"),
 	}
 
 	config.Addr = fmt.Sprintf(":%d", c.ListenPort)

cmd/virtual-kubelet/internal/commands/root/node.go

@@ -15,54 +15,10 @@
 package root
 
 import (
-	"context"
-	"strings"
-
-	"github.com/virtual-kubelet/virtual-kubelet/cmd/virtual-kubelet/internal/provider"
 	"github.com/virtual-kubelet/virtual-kubelet/errdefs"
 	corev1 "k8s.io/api/core/v1"
-	v1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-const osLabel = "beta.kubernetes.io/os"
-
-// NodeFromProvider builds a kubernetes node object from a provider
-// This is a temporary solution until node stuff actually split off from the provider interface itself.
-func NodeFromProvider(ctx context.Context, name string, taint *v1.Taint, p provider.Provider, version string) *v1.Node {
-	taints := make([]v1.Taint, 0)
-
-	if taint != nil {
-		taints = append(taints, *taint)
-	}
-
-	node := &v1.Node{
-		ObjectMeta: metav1.ObjectMeta{
-			Name: name,
-			Labels: map[string]string{
-				"type":                   "virtual-kubelet",
-				"kubernetes.io/role":     "agent",
-				"kubernetes.io/hostname": name,
-			},
-		},
-		Spec: v1.NodeSpec{
-			Taints: taints,
-		},
-		Status: v1.NodeStatus{
-			NodeInfo: v1.NodeSystemInfo{
-				Architecture:   "amd64",
-				KubeletVersion: version,
-			},
-		},
-	}
-
-	p.ConfigureNode(ctx, node)
-	if _, ok := node.ObjectMeta.Labels[osLabel]; !ok {
-		node.ObjectMeta.Labels[osLabel] = strings.ToLower(node.Status.NodeInfo.OperatingSystem)
-	}
-	return node
-}
-
 // getTaint creates a taint using the provided key/value.
 // Taint effect is read from the environment
 // The taint key/value may be overwritten by the environment.
@@ -80,7 +36,7 @@ func getTaint(c Opts) (*corev1.Taint, error) {
 
 	key = getEnv("VKUBELET_TAINT_KEY", key)
 	value = getEnv("VKUBELET_TAINT_VALUE", value)
-	effectEnv := getEnv("VKUBELET_TAINT_EFFECT", string(c.TaintEffect))
+	effectEnv := getEnv("VKUBELET_TAINT_EFFECT", c.TaintEffect)
 
 	var effect corev1.TaintEffect
 	switch effectEnv {

cmd/virtual-kubelet/internal/commands/root/opts.go

@@ -15,6 +15,7 @@
 package root
 
 import (
+	"fmt"
 	"os"
 	"path/filepath"
 	"strconv"
@@ -28,7 +29,7 @@ import (
 // Defaults for root command options
 const (
 	DefaultNodeName             = "virtual-kubelet"
-	DefaultOperatingSystem      = "Linux"
+	DefaultOperatingSystem      = "linux"
 	DefaultInformerResyncPeriod = 1 * time.Minute
 	DefaultMetricsAddr          = ":10255"
 	DefaultListenPort           = 10250 // TODO(cpuguy83)(VK1.0): Change this to an addr instead of just a port.. we should not be listening on all interfaces.
@@ -95,6 +96,8 @@ type Opts struct {
 	Version string
 }
 
+const maxInt32 = 1<<31 - 1
+
 // SetDefaultOpts sets default options for unset values on the passed in option struct.
 // Fields tht are already set will not be modified.
 func SetDefaultOpts(c *Opts) error {
@@ -128,6 +131,10 @@ func SetDefaultOpts(c *Opts) error {
 			if err != nil {
 				return errors.Wrap(err, "error parsing KUBELET_PORT environment variable")
 			}
+			if p > maxInt32 {
+				return fmt.Errorf("KUBELET_PORT environment variable is too large")
+			}
+			/* #nosec */
 			c.ListenPort = int32(p)
 		} else {
 			c.ListenPort = DefaultListenPort

cmd/virtual-kubelet/internal/commands/root/root.go

@@ -16,8 +16,10 @@ package root
 
 import (
 	"context"
+	"crypto/tls"
+	"net/http"
 	"os"
-	"path"
+	"runtime"
 
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
@@ -26,14 +28,10 @@ import (
 	"github.com/virtual-kubelet/virtual-kubelet/internal/manager"
 	"github.com/virtual-kubelet/virtual-kubelet/log"
 	"github.com/virtual-kubelet/virtual-kubelet/node"
+	"github.com/virtual-kubelet/virtual-kubelet/node/api"
 	"github.com/virtual-kubelet/virtual-kubelet/node/nodeutil"
 	corev1 "k8s.io/api/core/v1"
-	k8serrors "k8s.io/apimachinery/pkg/api/errors"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	kubeinformers "k8s.io/client-go/informers"
-	"k8s.io/client-go/kubernetes/scheme"
-	corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
-	"k8s.io/client-go/tools/record"
+	"k8s.io/apiserver/pkg/server/dynamiccertificates"
 )
 
 // NewCommand creates a new top-level command.
@@ -75,30 +73,33 @@ func runRootCommand(ctx context.Context, s *provider.Store, c Opts) error {
 		}
 	}
 
-	client, err := nodeutil.ClientsetFromEnv(c.KubeConfigPath)
-	if err != nil {
-		return err
-	}
+	mux := http.NewServeMux()
+	newProvider := func(cfg nodeutil.ProviderConfig) (nodeutil.Provider, node.NodeProvider, error) {
+		rm, err := manager.NewResourceManager(cfg.Pods, cfg.Secrets, cfg.ConfigMaps, cfg.Services)
+		if err != nil {
+			return nil, nil, errors.Wrap(err, "could not create resource manager")
+		}
+		initConfig := provider.InitConfig{
+			ConfigPath:        c.ProviderConfigPath,
+			NodeName:          c.NodeName,
+			OperatingSystem:   c.OperatingSystem,
+			ResourceManager:   rm,
+			DaemonPort:        c.ListenPort,
+			InternalIP:        os.Getenv("VKUBELET_POD_IP"),
+			KubeClusterDomain: c.KubeClusterDomain,
+		}
+		pInit := s.Get(c.Provider)
+		if pInit == nil {
+			return nil, nil, errors.Errorf("provider %q not found", c.Provider)
+		}
 
-	// Create a shared informer factory for Kubernetes pods in the current namespace (if specified) and scheduled to the current node.
-	podInformerFactory := kubeinformers.NewSharedInformerFactoryWithOptions(
-		client,
-		c.InformerResyncPeriod,
-		kubeinformers.WithNamespace(c.KubeNamespace),
-		nodeutil.PodInformerFilter(c.NodeName),
-	)
-	podInformer := podInformerFactory.Core().V1().Pods()
-
-	// Create another shared informer factory for Kubernetes secrets and configmaps (not subject to any selectors).
-	scmInformerFactory := kubeinformers.NewSharedInformerFactoryWithOptions(client, c.InformerResyncPeriod)
-	// Create a secret informer and a config map informer so we can pass their listers to the resource manager.
-	secretInformer := scmInformerFactory.Core().V1().Secrets()
-	configMapInformer := scmInformerFactory.Core().V1().ConfigMaps()
-	serviceInformer := scmInformerFactory.Core().V1().Services()
-
-	rm, err := manager.NewResourceManager(podInformer.Lister(), secretInformer.Lister(), configMapInformer.Lister(), serviceInformer.Lister())
-	if err != nil {
-		return errors.Wrap(err, "could not create resource manager")
+		p, err := pInit(initConfig)
+		if err != nil {
+			return nil, nil, errors.Wrapf(err, "error initializing provider %s", c.Provider)
+		}
+		p.ConfigureNode(ctx, cfg.Node)
+		cfg.Node.Status.NodeInfo.KubeletVersion = c.Version
+		return p, nil, nil
 	}
 
 	apiConfig, err := getAPIConfig(c)
@@ -106,28 +107,39 @@ func runRootCommand(ctx context.Context, s *provider.Store, c Opts) error {
 		return err
 	}
 
-	if err := setupTracing(ctx, c); err != nil {
+	cm, err := nodeutil.NewNode(c.NodeName, newProvider, func(cfg *nodeutil.NodeConfig) error {
+		cfg.KubeconfigPath = c.KubeConfigPath
+		cfg.Handler = mux
+		cfg.InformerResyncPeriod = c.InformerResyncPeriod
+
+		if taint != nil {
+			cfg.NodeSpec.Spec.Taints = append(cfg.NodeSpec.Spec.Taints, *taint)
+		}
+		cfg.NodeSpec.Status.NodeInfo.Architecture = runtime.GOARCH
+		cfg.NodeSpec.Status.NodeInfo.OperatingSystem = c.OperatingSystem
+
+		cfg.HTTPListenAddr = apiConfig.Addr
+		cfg.StreamCreationTimeout = apiConfig.StreamCreationTimeout
+		cfg.StreamIdleTimeout = apiConfig.StreamIdleTimeout
+		cfg.DebugHTTP = true
+
+		cfg.NumWorkers = c.PodSyncWorkers
+
+		return nil
+	},
+		setAuth(c.NodeName, apiConfig),
+		nodeutil.WithTLSConfig(
+			nodeutil.WithKeyPairFromPath(apiConfig.CertPath, apiConfig.KeyPath),
+			maybeCA(apiConfig.CACertPath),
+		),
+		nodeutil.AttachProviderRoutes(mux),
+	)
+	if err != nil {
 		return err
 	}
 
-	initConfig := provider.InitConfig{
-		ConfigPath:        c.ProviderConfigPath,
-		NodeName:          c.NodeName,
-		OperatingSystem:   c.OperatingSystem,
-		ResourceManager:   rm,
-		DaemonPort:        c.ListenPort,
-		InternalIP:        os.Getenv("VKUBELET_POD_IP"),
-		KubeClusterDomain: c.KubeClusterDomain,
-	}
-
-	pInit := s.Get(c.Provider)
-	if pInit == nil {
-		return errors.Errorf("provider %q not found", c.Provider)
-	}
-
-	p, err := pInit(initConfig)
-	if err != nil {
-		return errors.Wrapf(err, "error initializing provider %s", c.Provider)
+	if err := setupTracing(ctx, c); err != nil {
+		return err
 	}
 
 	ctx = log.WithLogger(ctx, log.G(ctx).WithFields(log.Fields{
@@ -137,117 +149,54 @@ func runRootCommand(ctx context.Context, s *provider.Store, c Opts) error {
 		"watchedNamespace": c.KubeNamespace,
 	}))
 
-	pNode := NodeFromProvider(ctx, c.NodeName, taint, p, c.Version)
-	np := node.NewNaiveNodeProvider()
-	additionalOptions := []node.NodeControllerOpt{
-		node.WithNodeStatusUpdateErrorHandler(func(ctx context.Context, err error) error {
-			if !k8serrors.IsNotFound(err) {
-				return err
-			}
+	go cm.Run(ctx) //nolint:errcheck
 
-			log.G(ctx).Debug("node not found")
-			newNode := pNode.DeepCopy()
-			newNode.ResourceVersion = ""
-			_, err = client.CoreV1().Nodes().Create(ctx, newNode, metav1.CreateOptions{})
-			if err != nil {
-				return err
-			}
-			log.G(ctx).Debug("created new node")
-			return nil
-		}),
-	}
-	if c.EnableNodeLease {
-		leaseClient := nodeutil.NodeLeaseV1Client(client)
-		// 40 seconds is the default lease time in upstream kubelet
-		additionalOptions = append(additionalOptions, node.WithNodeEnableLeaseV1(leaseClient, 40))
-	}
-	nodeRunner, err := node.NewNodeController(
-		np,
-		pNode,
-		client.CoreV1().Nodes(),
-		additionalOptions...,
-	)
-	if err != nil {
-		log.G(ctx).Fatal(err)
-	}
+	defer func() {
+		log.G(ctx).Debug("Waiting for controllers to be done")
+		cancel()
+		<-cm.Done()
+	}()
 
-	eb := record.NewBroadcaster()
-	eb.StartLogging(log.G(ctx).Infof)
-	eb.StartRecordingToSink(&corev1client.EventSinkImpl{Interface: client.CoreV1().Events(c.KubeNamespace)})
-
-	pc, err := node.NewPodController(node.PodControllerConfig{
-		PodClient:         client.CoreV1(),
-		PodInformer:       podInformer,
-		EventRecorder:     eb.NewRecorder(scheme.Scheme, corev1.EventSource{Component: path.Join(pNode.Name, "pod-controller")}),
-		Provider:          p,
-		SecretInformer:    secretInformer,
-		ConfigMapInformer: configMapInformer,
-		ServiceInformer:   serviceInformer,
-	})
-	if err != nil {
-		return errors.Wrap(err, "error setting up pod controller")
-	}
-
-	go podInformerFactory.Start(ctx.Done())
-	go scmInformerFactory.Start(ctx.Done())
-
-	cancelHTTP, err := setupHTTPServer(ctx, p, apiConfig, func(context.Context) ([]*corev1.Pod, error) {
-		return rm.GetPods(), nil
-	})
-	if err != nil {
+	log.G(ctx).Info("Waiting for controller to be ready")
+	if err := cm.WaitReady(ctx, c.StartupTimeout); err != nil {
 		return err
 	}
-	defer cancelHTTP()
 
-	go func() {
-		if err := pc.Run(ctx, c.PodSyncWorkers); err != nil && errors.Cause(err) != context.Canceled {
-			log.G(ctx).Fatal(err)
-		}
-	}()
+	log.G(ctx).Info("Ready")
 
-	if c.StartupTimeout > 0 {
-		ctx, cancel := context.WithTimeout(ctx, c.StartupTimeout)
-		log.G(ctx).Info("Waiting for pod controller / VK to be ready")
-		select {
-		case <-ctx.Done():
-			cancel()
-			return ctx.Err()
-		case <-pc.Ready():
-		}
-		cancel()
-		if err := pc.Err(); err != nil {
-			return err
-		}
+	select {
+	case <-ctx.Done():
+	case <-cm.Done():
+		return cm.Err()
 	}
-
-	go func() {
-		if err := nodeRunner.Run(ctx); err != nil {
-			log.G(ctx).Fatal(err)
-		}
-	}()
-
-	setNodeReady(pNode)
-	if err := np.UpdateStatus(ctx, pNode); err != nil {
-		return errors.Wrap(err, "error marking the node as ready")
-	}
-	log.G(ctx).Info("Initialized")
-
-	<-ctx.Done()
 	return nil
 }
 
-func setNodeReady(n *corev1.Node) {
-	for i, c := range n.Status.Conditions {
-		if c.Type != "Ready" {
-			continue
+func setAuth(node string, apiCfg *apiServerConfig) nodeutil.NodeOpt {
+	if apiCfg.CACertPath == "" {
+		return func(cfg *nodeutil.NodeConfig) error {
+			cfg.Handler = api.InstrumentHandler(nodeutil.WithAuth(nodeutil.NoAuth(), cfg.Handler))
+			return nil
 		}
+	}
 
-		c.Message = "Kubelet is ready"
-		c.Reason = "KubeletReady"
-		c.Status = corev1.ConditionTrue
-		c.LastHeartbeatTime = metav1.Now()
-		c.LastTransitionTime = metav1.Now()
-		n.Status.Conditions[i] = c
-		return
+	return func(cfg *nodeutil.NodeConfig) error {
+		auth, err := nodeutil.WebhookAuth(cfg.Client, node, func(cfg *nodeutil.WebhookAuthConfig) error {
+			var err error
+			cfg.AuthnConfig.ClientCertificateCAContentProvider, err = dynamiccertificates.NewDynamicCAContentFromFile("ca-cert-bundle", apiCfg.CACertPath)
+			return err
+		})
+		if err != nil {
+			return err
+		}
+		cfg.Handler = api.InstrumentHandler(nodeutil.WithAuth(auth, cfg.Handler))
+		return nil
 	}
 }
+
+func maybeCA(p string) func(*tls.Config) error {
+	if p == "" {
+		return func(*tls.Config) error { return nil }
+	}
+	return nodeutil.WithCAFromPath(p)
+}

cmd/virtual-kubelet/internal/commands/root/tracing.go

@@ -21,6 +21,7 @@ import (
 	"os"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/pkg/errors"
 	"github.com/virtual-kubelet/virtual-kubelet/errdefs"
@@ -105,7 +106,8 @@ func setupZpages(ctx context.Context) {
 	zpages.Handle(mux, "/debug")
 	go func() {
 		// This should never terminate, if it does, it will always terminate with an error
-		e := http.Serve(listener, mux)
+		srv := &http.Server{Handler: mux, ReadHeaderTimeout: 30 * time.Second}
+		e := srv.Serve(listener)
 		if e == http.ErrServerClosed {
 			return
 		}

cmd/virtual-kubelet/internal/commands/root/tracing_register.go

@@ -19,7 +19,8 @@ import (
 	"go.opencensus.io/trace"
 )
 
-type TracingExporterOptions struct { // nolint: golint
+// TracingExporterOptions is the options passed to the tracing exporter init function.
+type TracingExporterOptions struct { //nolint: golint
 	Tags        map[string]string
 	ServiceName string
 }

cmd/virtual-kubelet/internal/commands/root/tracing_register_jaeger.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build !no_jaeger_exporter
 // +build !no_jaeger_exporter
 
 package root
@@ -31,17 +32,17 @@ func init() {
 // NewJaegerExporter creates a new opencensus tracing exporter.
 func NewJaegerExporter(opts TracingExporterOptions) (trace.Exporter, error) {
 	jOpts := jaeger.Options{
-		Endpoint:      os.Getenv("JAEGER_ENDPOINT"),
-		AgentEndpoint: os.Getenv("JAEGER_AGENT_ENDPOINT"),
-		Username:      os.Getenv("JAEGER_USER"),
-		Password:      os.Getenv("JAEGER_PASSWORD"),
+		CollectorEndpoint: os.Getenv("JAEGER_COLLECTOR_ENDPOINT"),
+		AgentEndpoint:     os.Getenv("JAEGER_AGENT_ENDPOINT"),
+		Username:          os.Getenv("JAEGER_USER"),
+		Password:          os.Getenv("JAEGER_PASSWORD"),
 		Process: jaeger.Process{
 			ServiceName: opts.ServiceName,
 		},
 	}
 
-	if jOpts.Endpoint == "" && jOpts.AgentEndpoint == "" { // nolint:staticcheck
-		return nil, errors.New("Must specify either JAEGER_ENDPOINT or JAEGER_AGENT_ENDPOINT")
+	if jOpts.CollectorEndpoint == "" && jOpts.AgentEndpoint == "" { // nolintlint:staticcheck
+		return nil, errors.New("must specify either JAEGER_COLLECTOR_ENDPOINT or JAEGER_AGENT_ENDPOINT")
 	}
 
 	for k, v := range opts.Tags {

cmd/virtual-kubelet/internal/commands/root/tracing_register_ocagent.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build !no_ocagent_exporter
 // +build !no_ocagent_exporter
 
 package root

cmd/virtual-kubelet/internal/provider/mock/mock.go

@@ -5,8 +5,8 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"math/rand"
+	"os"
 	"strings"
 	"time"
 
@@ -17,7 +17,7 @@ import (
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	stats "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
+	stats "k8s.io/kubelet/pkg/apis/stats/v1alpha1"
 )
 
 const (
@@ -42,7 +42,7 @@ var (
 */
 
 // MockProvider implements the virtual-kubelet provider interface and stores pods in memory.
-type MockProvider struct { // nolint:golint
+type MockProvider struct { //nolint:golint
 	nodeName           string
 	operatingSystem    string
 	internalIP         string
@@ -54,7 +54,7 @@ type MockProvider struct { // nolint:golint
 }
 
 // MockConfig contains a mock virtual-kubelet's configurable parameters.
-type MockConfig struct { // nolint:golint
+type MockConfig struct { //nolint:golint
 	CPU    string `json:"cpu,omitempty"`
 	Memory string `json:"memory,omitempty"`
 	Pods   string `json:"pods,omitempty"`
@@ -97,7 +97,7 @@ func NewMockProvider(providerConfig, nodeName, operatingSystem string, internalI
 
 // loadConfig loads the given json configuration files.
 func loadConfig(providerConfig, nodeName string) (config MockConfig, err error) {
-	data, err := ioutil.ReadFile(providerConfig)
+	data, err := os.ReadFile(providerConfig)
 	if err != nil {
 		return config, err
 	}
@@ -283,7 +283,7 @@ func (p *MockProvider) GetContainerLogs(ctx context.Context, namespace, podName,
 	ctx = addAttributes(ctx, span, namespaceKey, namespace, nameKey, podName, containerNameKey, containerName)
 
 	log.G(ctx).Infof("receive GetContainerLogs %q", podName)
-	return ioutil.NopCloser(strings.NewReader("")), nil
+	return io.NopCloser(strings.NewReader("")), nil
 }
 
 // RunInContainer executes a command in a container in the pod, copying data
@@ -328,8 +328,8 @@ func (p *MockProvider) GetPods(ctx context.Context) ([]*v1.Pod, error) {
 	return pods, nil
 }
 
-func (p *MockProvider) ConfigureNode(ctx context.Context, n *v1.Node) { // nolint:golint
-	ctx, span := trace.StartSpan(ctx, "mock.ConfigureNode") // nolint:staticcheck,ineffassign
+func (p *MockProvider) ConfigureNode(ctx context.Context, n *v1.Node) { //nolint:golint
+	ctx, span := trace.StartSpan(ctx, "mock.ConfigureNode") //nolint:staticcheck,ineffassign
 	defer span.End()
 
 	n.Status.Capacity = p.capacity()
@@ -339,7 +339,7 @@ func (p *MockProvider) ConfigureNode(ctx context.Context, n *v1.Node) { // nolin
 	n.Status.DaemonEndpoints = p.nodeDaemonEndpoints()
 	os := p.operatingSystem
 	if os == "" {
-		os = "Linux"
+		os = "linux"
 	}
 	n.Status.NodeInfo.OperatingSystem = os
 	n.Status.NodeInfo.Architecture = "amd64"
@@ -467,10 +467,14 @@ func (p *MockProvider) GetStatsSummary(ctx context.Context) (*stats.Summary, err
 		for _, container := range pod.Spec.Containers {
 			// Grab a dummy value to be used as the total CPU usage.
 			// The value should fit a uint32 in order to avoid overflows later on when computing pod stats.
+
+			/* #nosec */
 			dummyUsageNanoCores := uint64(rand.Uint32())
 			totalUsageNanoCores += dummyUsageNanoCores
 			// Create a dummy value to be used as the total RAM usage.
 			// The value should fit a uint32 in order to avoid overflows later on when computing pod stats.
+
+			/* #nosec */
 			dummyUsageBytes := uint64(rand.Uint32())
 			totalUsageBytes += dummyUsageBytes
 			// Append a ContainerStats object containing the dummy stats to the PodStats object.

cmd/virtual-kubelet/internal/provider/provider.go

@@ -2,35 +2,15 @@ package provider
 
 import (
 	"context"
-	"io"
 
-	"github.com/virtual-kubelet/virtual-kubelet/node"
-	"github.com/virtual-kubelet/virtual-kubelet/node/api"
+	"github.com/virtual-kubelet/virtual-kubelet/node/nodeutil"
 	v1 "k8s.io/api/core/v1"
-	stats "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
 )
 
-// Provider contains the methods required to implement a virtual-kubelet provider.
-//
-// Errors produced by these methods should implement an interface from
-// github.com/virtual-kubelet/virtual-kubelet/errdefs package in order for the
-// core logic to be able to understand the type of failure.
+// Provider wraps the core provider type with an extra function needed to bootstrap the node
 type Provider interface {
-	node.PodLifecycleHandler
-
-	// GetContainerLogs retrieves the logs of a container by name from the provider.
-	GetContainerLogs(ctx context.Context, namespace, podName, containerName string, opts api.ContainerLogOpts) (io.ReadCloser, error)
-
-	// RunInContainer executes a command in a container in the pod, copying data
-	// between in/out/err and the container's stdin/stdout/stderr.
-	RunInContainer(ctx context.Context, namespace, podName, containerName string, cmd []string, attach api.AttachIO) error
-
+	nodeutil.Provider
 	// ConfigureNode enables a provider to configure the node object that
 	// will be used for Kubernetes.
 	ConfigureNode(context.Context, *v1.Node)
 }
-
-// PodMetricsProvider is an optional interface that providers can implement to expose pod stats
-type PodMetricsProvider interface {
-	GetStatsSummary(context.Context) (*stats.Summary, error)
-}

cmd/virtual-kubelet/internal/provider/store.go

@@ -13,7 +13,7 @@ type Store struct {
 	ls map[string]InitFunc
 }
 
-func NewStore() *Store { // nolint:golint
+func NewStore() *Store { //nolint:golint
 	return &Store{
 		ls: make(map[string]InitFunc),
 	}
@@ -71,4 +71,4 @@ type InitConfig struct {
 	ResourceManager   *manager.ResourceManager
 }
 
-type InitFunc func(InitConfig) (Provider, error) // nolint:golint
+type InitFunc func(InitConfig) (Provider, error) //nolint:golint

cmd/virtual-kubelet/internal/provider/types.go

@@ -2,12 +2,12 @@ package provider
 
 const (
 	// OperatingSystemLinux is the configuration value for defining Linux.
-	OperatingSystemLinux = "Linux"
+	OperatingSystemLinux = "linux"
 	// OperatingSystemWindows is the configuration value for defining Windows.
-	OperatingSystemWindows = "Windows"
+	OperatingSystemWindows = "windows"
 )
 
-type OperatingSystems map[string]bool // nolint:golint
+type OperatingSystems map[string]bool //nolint:golint
 
 var (
 	// ValidOperatingSystems defines the group of operating systems
@@ -18,7 +18,7 @@ var (
 	}
 )
 
-func (o OperatingSystems) Names() []string { // nolint:golint
+func (o OperatingSystems) Names() []string { //nolint:golint
 	keys := make([]string, 0, len(o))
 	for k := range o {
 		keys = append(keys, k)

cmd/virtual-kubelet/register.go

@@ -6,6 +6,7 @@ import (
 )
 
 func registerMock(s *provider.Store) {
+	/* #nosec */
 	s.Register("mock", func(cfg provider.InitConfig) (provider.Provider, error) { //nolint:errcheck
 		return mock.NewMockProvider(
 			cfg.ConfigPath,

go.mod

@@ -1,77 +1,100 @@
 module github.com/virtual-kubelet/virtual-kubelet
 
-go 1.15
+go 1.19
 
 require (
-	contrib.go.opencensus.io/exporter/jaeger v0.1.0
-	contrib.go.opencensus.io/exporter/ocagent v0.4.12
-	github.com/bombsimon/logrusr v1.0.0
-	github.com/docker/spdystream v0.0.0-20170912183627-bc6354cbbc29 // indirect
-	github.com/elazarl/goproxy v0.0.0-20190421051319-9d40249d3c2f // indirect
-	github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2 // indirect
-	github.com/google/go-cmp v0.4.0
-	github.com/gorilla/mux v1.7.0
+	contrib.go.opencensus.io/exporter/jaeger v0.2.1
+	contrib.go.opencensus.io/exporter/ocagent v0.7.0
+	github.com/bombsimon/logrusr/v3 v3.1.0
+	github.com/google/go-cmp v0.5.9
+	github.com/gorilla/mux v1.8.0
 	github.com/mitchellh/go-homedir v1.1.0
-	github.com/pkg/errors v0.8.1
-	github.com/prometheus/client_golang v1.0.0
-	github.com/sirupsen/logrus v1.4.2
-	github.com/spf13/cobra v0.0.5
+	github.com/pkg/errors v0.9.1
+	github.com/sirupsen/logrus v1.9.0
+	github.com/spf13/cobra v1.6.1
 	github.com/spf13/pflag v1.0.5
-	go.opencensus.io v0.21.0
-	go.uber.org/goleak v1.1.10
-	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e
-	golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd
-	golang.org/x/time v0.0.0-20190308202827-9d24e82272b4
+	go.opencensus.io v0.24.0
+	go.opentelemetry.io/otel v1.12.0
+	go.opentelemetry.io/otel/sdk v1.12.0
+	go.opentelemetry.io/otel/trace v1.12.0
+	golang.org/x/sync v0.1.0
+	golang.org/x/time v0.3.0
 	gotest.tools v2.2.0+incompatible
-	k8s.io/api v0.18.6
-	k8s.io/apimachinery v0.18.6
-	k8s.io/apiserver v0.18.4
-	k8s.io/client-go v0.18.6
-	k8s.io/klog v1.0.0
-	k8s.io/klog/v2 v2.0.0
-	k8s.io/kubernetes v1.18.4
-	k8s.io/utils v0.0.0-20200603063816-c1c6865ac451
-	sigs.k8s.io/controller-runtime v0.6.3
+	k8s.io/api v0.26.1
+	k8s.io/apimachinery v0.26.1
+	k8s.io/apiserver v0.26.1
+	k8s.io/client-go v0.26.1
+	k8s.io/klog/v2 v2.90.0
+	k8s.io/kubelet v0.26.1
+	k8s.io/utils v0.0.0-20230202215443-34013725500c
+	sigs.k8s.io/controller-runtime v0.14.4
 )
 
-replace k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.18.4
-
-replace k8s.io/cloud-provider => k8s.io/cloud-provider v0.18.4
-
-replace k8s.io/cli-runtime => k8s.io/cli-runtime v0.18.4
-
-replace k8s.io/apiserver => k8s.io/apiserver v0.18.4
-
-replace k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.18.4
-
-replace k8s.io/cri-api => k8s.io/cri-api v0.18.4
-
-replace k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.18.4
-
-replace k8s.io/kubelet => k8s.io/kubelet v0.18.4
-
-replace k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.18.4
-
-replace k8s.io/apimachinery => k8s.io/apimachinery v0.18.4
-
-replace k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.18.4
-
-replace k8s.io/kube-proxy => k8s.io/kube-proxy v0.18.4
-
-replace k8s.io/component-base => k8s.io/component-base v0.18.4
-
-replace k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.18.4
-
-replace k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.18.4
-
-replace k8s.io/metrics => k8s.io/metrics v0.18.4
-
-replace k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.18.4
-
-replace k8s.io/code-generator => k8s.io/code-generator v0.18.4
-
-replace k8s.io/client-go => k8s.io/client-go v0.18.4
-
-replace k8s.io/kubectl => k8s.io/kubectl v0.18.4
-
-replace k8s.io/api => k8s.io/api v0.18.4
+require (
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/blang/semver/v4 v4.0.0 // indirect
+	github.com/cenkalti/backoff/v4 v4.1.3 // indirect
+	github.com/census-instrumentation/opencensus-proto v0.2.1 // indirect
+	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/elazarl/goproxy v0.0.0-20190421051319-9d40249d3c2f // indirect
+	github.com/emicklei/go-restful/v3 v3.9.0 // indirect
+	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
+	github.com/evanphx/json-patch/v5 v5.6.0 // indirect
+	github.com/felixge/httpsnoop v1.0.3 // indirect
+	github.com/fsnotify/fsnotify v1.6.0 // indirect
+	github.com/go-logr/logr v1.2.3 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-openapi/jsonpointer v0.19.5 // indirect
+	github.com/go-openapi/jsonreference v0.20.0 // indirect
+	github.com/go-openapi/swag v0.19.14 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/google/gnostic v0.5.7-v3refs // indirect
+	github.com/google/gofuzz v1.1.0 // indirect
+	github.com/google/uuid v1.1.2 // indirect
+	github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect
+	github.com/imdario/mergo v0.3.12 // indirect
+	github.com/inconshreveable/mousetrap v1.0.1 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/mailru/easyjson v0.7.6 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.2 // indirect
+	github.com/moby/spdystream v0.2.0 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/prometheus/client_golang v1.14.0 // indirect
+	github.com/prometheus/client_model v0.3.0 // indirect
+	github.com/prometheus/common v0.37.0 // indirect
+	github.com/prometheus/procfs v0.8.0 // indirect
+	github.com/uber/jaeger-client-go v2.25.0+incompatible // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.35.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.10.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.10.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.10.0 // indirect
+	go.opentelemetry.io/otel/metric v0.31.0 // indirect
+	go.opentelemetry.io/proto/otlp v0.19.0 // indirect
+	golang.org/x/net v0.3.1-0.20221206200815-1e63c2f08a10 // indirect
+	golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b // indirect
+	golang.org/x/sys v0.5.0 // indirect
+	golang.org/x/term v0.3.0 // indirect
+	golang.org/x/text v0.5.0 // indirect
+	google.golang.org/api v0.43.0 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect
+	google.golang.org/grpc v1.49.0 // indirect
+	google.golang.org/protobuf v1.28.1 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	k8s.io/apiextensions-apiserver v0.26.1 // indirect
+	k8s.io/component-base v0.26.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect
+	sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.35 // indirect
+	sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
+	sigs.k8s.io/yaml v1.3.0 // indirect
+)

hack/skaffold/virtual-kubelet/base.yml

@@ -56,6 +56,14 @@ rules:
   verbs:
   - create
   - patch
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - get
+  - create
+  - update
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding

hack/skaffold/virtual-kubelet/pod.yml

@@ -4,6 +4,8 @@ metadata:
   name: vkubelet-mock-0
 spec:
   containers:
+  - name: jaeger-tracing
+    image: jaegertracing/all-in-one:1.22
   - name: vkubelet-mock-0
     image: virtual-kubelet
     # "IfNotPresent" is used to prevent Minikube from trying to pull from the registry (and failing) in the first place.
@@ -23,18 +25,16 @@ spec:
     - --klog.logtostderr
     - --log-level
     - debug
+    - --trace-exporter
+    - jaeger
+    - --trace-sample-rate=always
     env:
+    - name: JAEGER_AGENT_ENDPOINT
+      value: localhost:6831
     - name: KUBELET_PORT
       value: "10250"
     - name: VKUBELET_POD_IP
       valueFrom:
         fieldRef:
           fieldPath: status.podIP
-    ports:
-    - name: metrics
-      containerPort: 10255
-    readinessProbe:
-      httpGet:
-        path: /stats/summary
-        port: metrics
   serviceAccountName: virtual-kubelet

internal/expansion/expand_test.go

@@ -2,24 +2,20 @@ package expansion
 
 import (
 	"testing"
-
-	api "k8s.io/kubernetes/pkg/apis/core"
 )
 
 func TestMapReference(t *testing.T) {
-	envs := []api.EnvVar{
-		{
-			Name:  "FOO",
-			Value: "bar",
-		},
-		{
-			Name:  "ZOO",
-			Value: "$(FOO)-1",
-		},
-		{
-			Name:  "BLU",
-			Value: "$(ZOO)-2",
-		},
+	// We use a struct here instead of a map because we need mappings to happen in order.
+	// Go maps are randomized.
+	type envVar struct {
+		Name  string
+		Value string
+	}
+
+	envs := []envVar{
+		{"FOO", "bar"},
+		{"ZOO", "$(FOO)-1"},
+		{"BLU", "$(ZOO)-2"},
 	}
 
 	declaredEnv := map[string]string{

internal/lockdeps/lockdeps.go

@@ -1,9 +0,0 @@
-package lockdeps
-
-import (
-	// TODO(Sargun): Remove in Go1.13
-	// This is a dep that `go mod tidy` keeps removing, because it's a transitive dep that's pulled in via a test
-	// See: https://github.com/golang/go/issues/29702
-	_ "github.com/prometheus/client_golang/prometheus"
-	_ "golang.org/x/sys/unix"
-)

internal/manager/resource_test.go

@@ -123,7 +123,7 @@ func TestGetConfigMap(t *testing.T) {
 	}
 	value := configMap.Data["key-0"]
 	if value != "val-0" {
-		t.Fatal("got unexpected value", string(value))
+		t.Fatal("got unexpected value", value)
 	}
 
 	// Try to get a configmap that does not exist, and make sure we've got a "not found" error as a response.

internal/podutils/README.md

@@ -0,0 +1,15 @@
+Much of this is copied from k8s.io/kubernetes, even if it isn't a 1-1 copy of a
+file.  This exists so we do not have to import from k8s.io/kubernetes which is
+currently problematic.  Ideally most or all of this will go away and an upstream
+solution is found so that we can share an implementation with Kubelet without
+importing from k8s.io/kubernetes
+
+
+| filename | upstream location |
+|----------|-------------------|
+| envvars.go | https://github.com/kubernetes/kubernetes/blob/98d5dc5d36d34a7ee13368a7893dcb400ec4e566/pkg/kubelet/envvars/envvars.go#L32 |
+| helper.go#ConvertDownwardAPIFieldLabel | https://github.com/kubernetes/kubernetes/blob/98d5dc5d36d34a7ee13368a7893dcb400ec4e566/pkg/apis/core/pods/helpers.go#L65 |
+| helper.go#ExtractFieldPathAsString | https://github.com/kubernetes/kubernetes/blob/98d5dc5d36d34a7ee13368a7893dcb400ec4e566/pkg/fieldpath/fieldpath.go#L46 |
+| helper.go#SplitMaybeSubscriptedPath | https://github.com/kubernetes/kubernetes/blob/98d5dc5d36d34a7ee13368a7893dcb400ec4e566/pkg/fieldpath/fieldpath.go#L96 |
+| helper.go#FormatMap | https://github.com/kubernetes/kubernetes/blob/ea0764452222146c47ec826977f49d7001b0ea8c/pkg/fieldpath/fieldpath.go#L29 |
+| helper.go#IsServiceIPSet | https://github.com/kubernetes/kubernetes/blob/ea0764452222146c47ec826977f49d7001b0ea8c/pkg/apis/core/v1/helper/helpers.go#L139 |

internal/podutils/env.go

@@ -29,10 +29,6 @@ import (
 	"k8s.io/apimachinery/pkg/util/sets"
 	apivalidation "k8s.io/apimachinery/pkg/util/validation"
 	"k8s.io/client-go/tools/record"
-	podshelper "k8s.io/kubernetes/pkg/apis/core/pods"
-	v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper"
-	fieldpath "k8s.io/kubernetes/pkg/fieldpath"
-	"k8s.io/kubernetes/pkg/kubelet/envvars"
 	"k8s.io/utils/pointer"
 )
 
@@ -139,7 +135,7 @@ func getServiceEnvVarMap(rm *manager.ResourceManager, ns string, enableServiceLi
 	for i := range services {
 		service := services[i]
 		// ignore services where ClusterIP is "None" or empty
-		if !v1helper.IsServiceIPSet(service) {
+		if !IsServiceIPSet(service) {
 			continue
 		}
 		serviceName := service.Name
@@ -162,7 +158,7 @@ func getServiceEnvVarMap(rm *manager.ResourceManager, ns string, enableServiceLi
 		mappedServices = append(mappedServices, serviceMap[key])
 	}
 
-	for _, e := range envvars.FromServices(mappedServices) {
+	for _, e := range FromServices(mappedServices) {
 		m[e.Name] = e.Value
 	}
 	return m, nil
@@ -486,7 +482,7 @@ func getEnvironmentVariableValueWithValueFromFieldRef(ctx context.Context, env *
 // podFieldSelectorRuntimeValue returns the runtime value of the given
 // selector for a pod.
 func podFieldSelectorRuntimeValue(fs *corev1.ObjectFieldSelector, pod *corev1.Pod) (string, error) {
-	internalFieldPath, _, err := podshelper.ConvertDownwardAPIFieldLabel(fs.APIVersion, fs.FieldPath, "")
+	internalFieldPath, _, err := ConvertDownwardAPIFieldLabel(fs.APIVersion, fs.FieldPath, "")
 	if err != nil {
 		return "", err
 	}
@@ -497,5 +493,5 @@ func podFieldSelectorRuntimeValue(fs *corev1.ObjectFieldSelector, pod *corev1.Po
 		return pod.Spec.ServiceAccountName, nil
 
 	}
-	return fieldpath.ExtractFieldPathAsString(pod, internalFieldPath)
+	return ExtractFieldPathAsString(pod, internalFieldPath)
 }

internal/podutils/envvars.go

@@ -0,0 +1,112 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package podutils
+
+import (
+	"fmt"
+	"net"
+	"strconv"
+	"strings"
+
+	v1 "k8s.io/api/core/v1"
+)
+
+// FromServices builds environment variables that a container is started with,
+// which tell the container where to find the services it may need, which are
+// provided as an argument.
+func FromServices(services []*v1.Service) []v1.EnvVar {
+	var result []v1.EnvVar
+	for i := range services {
+		service := services[i]
+
+		// ignore services where ClusterIP is "None" or empty
+		// the services passed to this method should be pre-filtered
+		// only services that have the cluster IP set should be included here
+		if !IsServiceIPSet(service) {
+			continue
+		}
+
+		// Host
+		name := makeEnvVariableName(service.Name) + "_SERVICE_HOST"
+		result = append(result, v1.EnvVar{Name: name, Value: service.Spec.ClusterIP})
+		// First port - give it the backwards-compatible name
+		name = makeEnvVariableName(service.Name) + "_SERVICE_PORT"
+		result = append(result, v1.EnvVar{Name: name, Value: strconv.Itoa(int(service.Spec.Ports[0].Port))})
+		// All named ports (only the first may be unnamed, checked in validation)
+		for i := range service.Spec.Ports {
+			sp := &service.Spec.Ports[i]
+			if sp.Name != "" {
+				pn := name + "_" + makeEnvVariableName(sp.Name)
+				result = append(result, v1.EnvVar{Name: pn, Value: strconv.Itoa(int(sp.Port))})
+			}
+		}
+		// Docker-compatible vars.
+		result = append(result, makeLinkVariables(service)...)
+	}
+	return result
+}
+
+func makeEnvVariableName(str string) string {
+	// TODO: If we simplify to "all names are DNS1123Subdomains" this
+	// will need two tweaks:
+	//   1) Handle leading digits
+	//   2) Handle dots
+	return strings.ToUpper(strings.Replace(str, "-", "_", -1))
+}
+
+func makeLinkVariables(service *v1.Service) []v1.EnvVar {
+	prefix := makeEnvVariableName(service.Name)
+	all := []v1.EnvVar{}
+	for i := range service.Spec.Ports {
+		sp := &service.Spec.Ports[i]
+
+		protocol := string(v1.ProtocolTCP)
+		if sp.Protocol != "" {
+			protocol = string(sp.Protocol)
+		}
+
+		hostPort := net.JoinHostPort(service.Spec.ClusterIP, strconv.Itoa(int(sp.Port)))
+
+		if i == 0 {
+			// Docker special-cases the first port.
+			all = append(all, v1.EnvVar{
+				Name:  prefix + "_PORT",
+				Value: fmt.Sprintf("%s://%s", strings.ToLower(protocol), hostPort),
+			})
+		}
+		portPrefix := fmt.Sprintf("%s_PORT_%d_%s", prefix, sp.Port, strings.ToUpper(protocol))
+		all = append(all, []v1.EnvVar{
+			{
+				Name:  portPrefix,
+				Value: fmt.Sprintf("%s://%s", strings.ToLower(protocol), hostPort),
+			},
+			{
+				Name:  portPrefix + "_PROTO",
+				Value: strings.ToLower(protocol),
+			},
+			{
+				Name:  portPrefix + "_PORT",
+				Value: strconv.Itoa(int(sp.Port)),
+			},
+			{
+				Name:  portPrefix + "_ADDR",
+				Value: service.Spec.ClusterIP,
+			},
+		}...)
+	}
+	return all
+}

internal/podutils/helper.go

@@ -0,0 +1,156 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package podutils
+
+import (
+	"fmt"
+	"strings"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/meta"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apimachinery/pkg/util/validation"
+)
+
+// ConvertDownwardAPIFieldLabel converts the specified downward API field label
+// and its value in the pod of the specified version to the internal version,
+// and returns the converted label and value. This function returns an error if
+// the conversion fails.
+func ConvertDownwardAPIFieldLabel(version, label, value string) (string, string, error) {
+	if version != "v1" {
+		return "", "", fmt.Errorf("unsupported pod version: %s", version)
+	}
+
+	if path, _, ok := SplitMaybeSubscriptedPath(label); ok {
+		switch path {
+		case "metadata.annotations", "metadata.labels":
+			return label, value, nil
+		default:
+			return "", "", fmt.Errorf("field label does not support subscript: %s", label)
+		}
+	}
+
+	switch label {
+	case "metadata.annotations",
+		"metadata.labels",
+		"metadata.name",
+		"metadata.namespace",
+		"metadata.uid",
+		"spec.nodeName",
+		"spec.restartPolicy",
+		"spec.serviceAccountName",
+		"spec.schedulerName",
+		"status.phase",
+		"status.hostIP",
+		"status.podIP",
+		"status.podIPs":
+		return label, value, nil
+	// This is for backwards compatibility with old v1 clients which send spec.host
+	case "spec.host":
+		return "spec.nodeName", value, nil
+	default:
+		return "", "", fmt.Errorf("field label not supported: %s", label)
+	}
+}
+
+// ExtractFieldPathAsString extracts the field from the given object
+// and returns it as a string.  The object must be a pointer to an
+// API type.
+func ExtractFieldPathAsString(obj interface{}, fieldPath string) (string, error) {
+	accessor, err := meta.Accessor(obj)
+	if err != nil {
+		return "", err
+	}
+
+	if path, subscript, ok := SplitMaybeSubscriptedPath(fieldPath); ok {
+		switch path {
+		case "metadata.annotations":
+			if errs := validation.IsQualifiedName(strings.ToLower(subscript)); len(errs) != 0 {
+				return "", fmt.Errorf("invalid key subscript in %s: %s", fieldPath, strings.Join(errs, ";"))
+			}
+			return accessor.GetAnnotations()[subscript], nil
+		case "metadata.labels":
+			if errs := validation.IsQualifiedName(subscript); len(errs) != 0 {
+				return "", fmt.Errorf("invalid key subscript in %s: %s", fieldPath, strings.Join(errs, ";"))
+			}
+			return accessor.GetLabels()[subscript], nil
+		default:
+			return "", fmt.Errorf("fieldPath %q does not support subscript", fieldPath)
+		}
+	}
+
+	switch fieldPath {
+	case "metadata.annotations":
+		return FormatMap(accessor.GetAnnotations()), nil
+	case "metadata.labels":
+		return FormatMap(accessor.GetLabels()), nil
+	case "metadata.name":
+		return accessor.GetName(), nil
+	case "metadata.namespace":
+		return accessor.GetNamespace(), nil
+	case "metadata.uid":
+		return string(accessor.GetUID()), nil
+	}
+
+	return "", fmt.Errorf("unsupported fieldPath: %v", fieldPath)
+}
+
+// SplitMaybeSubscriptedPath checks whether the specified fieldPath is
+// subscripted, and
+//  - if yes, this function splits the fieldPath into path and subscript, and
+//    returns (path, subscript, true).
+//  - if no, this function returns (fieldPath, "", false).
+//
+// Example inputs and outputs:
+//  - "metadata.annotations['myKey']" --> ("metadata.annotations", "myKey", true)
+//  - "metadata.annotations['a[b]c']" --> ("metadata.annotations", "a[b]c", true)
+//  - "metadata.labels['']"           --> ("metadata.labels", "", true)
+//  - "metadata.labels"               --> ("metadata.labels", "", false)
+func SplitMaybeSubscriptedPath(fieldPath string) (string, string, bool) {
+	if !strings.HasSuffix(fieldPath, "']") {
+		return fieldPath, "", false
+	}
+	s := strings.TrimSuffix(fieldPath, "']")
+	parts := strings.SplitN(s, "['", 2)
+	if len(parts) < 2 {
+		return fieldPath, "", false
+	}
+	if len(parts[0]) == 0 {
+		return fieldPath, "", false
+	}
+	return parts[0], parts[1], true
+}
+
+// FormatMap formats map[string]string to a string.
+func FormatMap(m map[string]string) (fmtStr string) {
+	// output with keys in sorted order to provide stable output
+	keys := sets.NewString()
+	for key := range m {
+		keys.Insert(key)
+	}
+	for _, key := range keys.List() {
+		fmtStr += fmt.Sprintf("%v=%q\n", key, m[key])
+	}
+	fmtStr = strings.TrimSuffix(fmtStr, "\n")
+
+	return
+}
+
+// IsServiceIPSet aims to check if the service's ClusterIP is set or not the objective is not to perform validation here
+func IsServiceIPSet(service *corev1.Service) bool {
+	return service.Spec.ClusterIP != corev1.ClusterIPNone && service.Spec.ClusterIP != ""
+}

internal/queue/queue.go

@@ -15,6 +15,7 @@
 package queue
 
 import (
+	"container/list"
 	"context"
 	"fmt"
 	"sync"
@@ -23,8 +24,10 @@ import (
 	pkgerrors "github.com/pkg/errors"
 	"github.com/virtual-kubelet/virtual-kubelet/log"
 	"github.com/virtual-kubelet/virtual-kubelet/trace"
+	"golang.org/x/sync/semaphore"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/util/workqueue"
+	"k8s.io/utils/clock"
 )
 
 const (
@@ -32,58 +35,257 @@ const (
 	MaxRetries = 20
 )
 
+// ShouldRetryFunc is a mechanism to have a custom retry policy
+type ShouldRetryFunc func(ctx context.Context, key string, timesTried int, originallyAdded time.Time, err error) (*time.Duration, error)
+
 // ItemHandler is a callback that handles a single key on the Queue
 type ItemHandler func(ctx context.Context, key string) error
 
 // Queue implements a wrapper around workqueue with native VK instrumentation
 type Queue struct {
-	lock      sync.Mutex
-	running   bool
-	name      string
-	workqueue workqueue.RateLimitingInterface
-	handler   ItemHandler
+	// clock is used for testing
+	clock clock.Clock
+	// lock protects running, and the items list / map
+	lock    sync.Mutex
+	running bool
+	name    string
+	handler ItemHandler
+
+	ratelimiter workqueue.RateLimiter
+	// items are items that are marked dirty waiting for processing.
+	items *list.List
+	// itemInQueue is a map of (string) key -> item while it is in the items list
+	itemsInQueue map[string]*list.Element
+	// itemsBeingProcessed is a map of (string) key -> item once it has been moved
+	itemsBeingProcessed map[string]*queueItem
+	// Wait for next semaphore is an exclusive (1 item) lock that is taken every time items is checked to see if there
+	// is an item in queue for work
+	waitForNextItemSemaphore *semaphore.Weighted
+
+	// wakeup
+	wakeupCh chan struct{}
+
+	retryFunc ShouldRetryFunc
+}
+
+type queueItem struct {
+	key                    string
+	plannedToStartWorkAt   time.Time
+	redirtiedAt            time.Time
+	redirtiedWithRatelimit bool
+	forget                 bool
+	requeues               int
+
+	// Debugging information only
+	originallyAdded     time.Time
+	addedViaRedirty     bool
+	delayedViaRateLimit *time.Duration
+}
+
+func (item *queueItem) String() string {
+	return fmt.Sprintf("<plannedToStartWorkAt:%s key: %s>", item.plannedToStartWorkAt.String(), item.key)
 }
 
 // New creates a queue
 //
-// It expects to get a item rate limiter, and a friendly name which is used in logs, and
-// in the internal kubernetes metrics.
-func New(ratelimiter workqueue.RateLimiter, name string, handler ItemHandler) *Queue {
+// It expects to get a item rate limiter, and a friendly name which is used in logs, and in the internal kubernetes
+// metrics. If retryFunc is nil, the default retry function.
+func New(ratelimiter workqueue.RateLimiter, name string, handler ItemHandler, retryFunc ShouldRetryFunc) *Queue {
+	if retryFunc == nil {
+		retryFunc = DefaultRetryFunc
+	}
 	return &Queue{
-		name:      name,
-		workqueue: workqueue.NewNamedRateLimitingQueue(ratelimiter, name),
-		handler:   handler,
+		clock:                    clock.RealClock{},
+		name:                     name,
+		ratelimiter:              ratelimiter,
+		items:                    list.New(),
+		itemsBeingProcessed:      make(map[string]*queueItem),
+		itemsInQueue:             make(map[string]*list.Element),
+		handler:                  handler,
+		wakeupCh:                 make(chan struct{}, 1),
+		waitForNextItemSemaphore: semaphore.NewWeighted(1),
+		retryFunc:                retryFunc,
 	}
 }
 
 // Enqueue enqueues the key in a rate limited fashion
-func (q *Queue) Enqueue(key string) {
-	q.workqueue.AddRateLimited(key)
+func (q *Queue) Enqueue(ctx context.Context, key string) {
+	q.lock.Lock()
+	defer q.lock.Unlock()
+
+	q.insert(ctx, key, true, nil)
 }
 
 // EnqueueWithoutRateLimit enqueues the key without a rate limit
-func (q *Queue) EnqueueWithoutRateLimit(key string) {
-	q.workqueue.Add(key)
+func (q *Queue) EnqueueWithoutRateLimit(ctx context.Context, key string) {
+	q.lock.Lock()
+	defer q.lock.Unlock()
+
+	q.insert(ctx, key, false, nil)
 }
 
 // Forget forgets the key
-func (q *Queue) Forget(key string) {
-	q.workqueue.Forget(key)
+func (q *Queue) Forget(ctx context.Context, key string) {
+	q.lock.Lock()
+	defer q.lock.Unlock()
+	ctx, span := trace.StartSpan(ctx, "Forget")
+	defer span.End()
+
+	ctx = span.WithFields(ctx, map[string]interface{}{
+		"queue": q.name,
+		"key":   key,
+	})
+
+	if item, ok := q.itemsInQueue[key]; ok {
+		span.WithField(ctx, "status", "itemInQueue")
+		delete(q.itemsInQueue, key)
+		q.items.Remove(item)
+		return
+	}
+
+	if qi, ok := q.itemsBeingProcessed[key]; ok {
+		span.WithField(ctx, "status", "itemBeingProcessed")
+		qi.forget = true
+		return
+	}
+	span.WithField(ctx, "status", "notfound")
 }
 
-// EnqueueAfter enqueues the item after this period
-//
-// Since it wrap workqueue semantics, if an item has been enqueued  after, and it is immediately scheduled for work,
-// it will process the immediate item, and then upon the latter delayed processing it will be processed again
-func (q *Queue) EnqueueAfter(key string, after time.Duration) {
-	q.workqueue.AddAfter(key, after)
+func durationDeref(duration *time.Duration, def time.Duration) time.Duration {
+	if duration == nil {
+		return def
+	}
+
+	return *duration
+}
+
+// insert inserts a new item to be processed at time time. It will not further delay items if when is later than the
+// original time the item was scheduled to be processed. If when is earlier, it will "bring it forward"
+// If ratelimit is specified, and delay is nil, then the ratelimiter's delay (return from When function) will be used
+// If ratelimit is specified, and the delay is non-nil, then the delay value will be used
+// If ratelimit is false, then only delay is used to schedule the work. If delay is nil, it will be considered 0.
+func (q *Queue) insert(ctx context.Context, key string, ratelimit bool, delay *time.Duration) *queueItem {
+	ctx, span := trace.StartSpan(ctx, "insert")
+	defer span.End()
+
+	ctx = span.WithFields(ctx, map[string]interface{}{
+		"queue":     q.name,
+		"key":       key,
+		"ratelimit": ratelimit,
+	})
+	if delay == nil {
+		ctx = span.WithField(ctx, "delay", "nil")
+	} else {
+		ctx = span.WithField(ctx, "delay", delay.String())
+	}
+
+	defer func() {
+		select {
+		case q.wakeupCh <- struct{}{}:
+		default:
+		}
+	}()
+
+	// First see if the item is already being processed
+	if item, ok := q.itemsBeingProcessed[key]; ok {
+		span.WithField(ctx, "status", "itemsBeingProcessed")
+		when := q.clock.Now().Add(durationDeref(delay, 0))
+		// Is the item already been redirtied?
+		if item.redirtiedAt.IsZero() {
+			item.redirtiedAt = when
+			item.redirtiedWithRatelimit = ratelimit
+		} else if when.Before(item.redirtiedAt) {
+			item.redirtiedAt = when
+			item.redirtiedWithRatelimit = ratelimit
+		}
+		item.forget = false
+		return item
+	}
+
+	// Is the item already in the queue?
+	if item, ok := q.itemsInQueue[key]; ok {
+		span.WithField(ctx, "status", "itemsInQueue")
+		qi := item.Value.(*queueItem)
+		when := q.clock.Now().Add(durationDeref(delay, 0))
+		q.adjustPosition(qi, item, when)
+		return qi
+	}
+
+	span.WithField(ctx, "status", "added")
+	now := q.clock.Now()
+	val := &queueItem{
+		key:                  key,
+		plannedToStartWorkAt: now,
+		originallyAdded:      now,
+	}
+
+	if ratelimit {
+		actualDelay := q.ratelimiter.When(key)
+		// Check if delay is overridden
+		if delay != nil {
+			actualDelay = *delay
+		}
+		span.WithField(ctx, "delay", actualDelay.String())
+		val.plannedToStartWorkAt = val.plannedToStartWorkAt.Add(actualDelay)
+		val.delayedViaRateLimit = &actualDelay
+	} else {
+		val.plannedToStartWorkAt = val.plannedToStartWorkAt.Add(durationDeref(delay, 0))
+	}
+
+	for item := q.items.Back(); item != nil; item = item.Prev() {
+		qi := item.Value.(*queueItem)
+		if qi.plannedToStartWorkAt.Before(val.plannedToStartWorkAt) {
+			q.itemsInQueue[key] = q.items.InsertAfter(val, item)
+			return val
+		}
+	}
+
+	q.itemsInQueue[key] = q.items.PushFront(val)
+	return val
+}
+
+func (q *Queue) adjustPosition(qi *queueItem, element *list.Element, when time.Time) {
+	if when.After(qi.plannedToStartWorkAt) {
+		// The item has already been delayed appropriately
+		return
+	}
+
+	qi.plannedToStartWorkAt = when
+	for prev := element.Prev(); prev != nil; prev = prev.Prev() {
+		item := prev.Value.(*queueItem)
+		// does this item plan to start work *before* the new time? If so add it
+		if item.plannedToStartWorkAt.Before(when) {
+			q.items.MoveAfter(element, prev)
+			return
+		}
+	}
+
+	q.items.MoveToFront(element)
+}
+
+// EnqueueWithoutRateLimitWithDelay enqueues without rate limiting, but work will not start for this given delay period
+func (q *Queue) EnqueueWithoutRateLimitWithDelay(ctx context.Context, key string, after time.Duration) {
+	q.lock.Lock()
+	defer q.lock.Unlock()
+	q.insert(ctx, key, false, &after)
 }
 
 // Empty returns if the queue has no items in it
 //
-// It should only be used for debugging, as delayed items are not counted, leading to confusion
+// It should only be used for debugging.
 func (q *Queue) Empty() bool {
-	return q.workqueue.Len() == 0
+	return q.Len() == 0
+}
+
+// Len includes items that are in the queue, and are being processed
+func (q *Queue) Len() int {
+	q.lock.Lock()
+	defer q.lock.Unlock()
+	if q.items.Len() != len(q.itemsInQueue) {
+		panic("Internally inconsistent state")
+	}
+
+	return q.items.Len() + len(q.itemsBeingProcessed)
 }
 
 // Run starts the workers
@@ -112,13 +314,14 @@ func (q *Queue) Run(ctx context.Context, workers int) {
 
 	group := &wait.Group{}
 	for i := 0; i < workers; i++ {
+		// This is required because i is referencing a mutable variable and that's running in a separate goroutine
+		idx := i
 		group.StartWithContext(ctx, func(ctx context.Context) {
-			q.worker(ctx, i)
+			q.worker(ctx, idx)
 		})
 	}
 	defer group.Wait()
 	<-ctx.Done()
-	q.workqueue.ShutDown()
 }
 
 func (q *Queue) worker(ctx context.Context, i int) {
@@ -130,6 +333,54 @@ func (q *Queue) worker(ctx context.Context, i int) {
 	}
 }
 
+func (q *Queue) getNextItem(ctx context.Context) (*queueItem, error) {
+	if err := q.waitForNextItemSemaphore.Acquire(ctx, 1); err != nil {
+		return nil, err
+	}
+	defer q.waitForNextItemSemaphore.Release(1)
+
+	for {
+		q.lock.Lock()
+		element := q.items.Front()
+		if element == nil {
+			// Wait for the next item
+			q.lock.Unlock()
+			select {
+			case <-ctx.Done():
+				return nil, ctx.Err()
+			case <-q.wakeupCh:
+			}
+		} else {
+			qi := element.Value.(*queueItem)
+			timeUntilProcessing := time.Until(qi.plannedToStartWorkAt)
+
+			// Do we need to sleep? If not, let's party.
+			if timeUntilProcessing <= 0 {
+				q.itemsBeingProcessed[qi.key] = qi
+				q.items.Remove(element)
+				delete(q.itemsInQueue, qi.key)
+				q.lock.Unlock()
+				return qi, nil
+			}
+
+			q.lock.Unlock()
+			if err := func() error {
+				timer := q.clock.NewTimer(timeUntilProcessing)
+				defer timer.Stop()
+				select {
+				case <-timer.C():
+				case <-ctx.Done():
+					return ctx.Err()
+				case <-q.wakeupCh:
+				}
+				return nil
+			}(); err != nil {
+				return nil, err
+			}
+		}
+	}
+}
+
 // handleQueueItem handles a single item
 //
 // A return value of "false" indicates that further processing should be stopped.
@@ -137,8 +388,9 @@ func (q *Queue) handleQueueItem(ctx context.Context) bool {
 	ctx, span := trace.StartSpan(ctx, "handleQueueItem")
 	defer span.End()
 
-	obj, shutdown := q.workqueue.Get()
-	if shutdown {
+	qi, err := q.getNextItem(ctx)
+	if err != nil {
+		span.SetStatus(err)
 		return false
 	}
 
@@ -146,11 +398,10 @@ func (q *Queue) handleQueueItem(ctx context.Context) bool {
 	// These are of the form namespace/name.
 	// We do this as the delayed nature of the work Queue means the items in the informer cache may actually be more u
 	// to date that when the item was initially put onto the workqueue.
-	key := obj.(string)
-	ctx = span.WithField(ctx, "key", key)
+	ctx = span.WithField(ctx, "key", qi.key)
 	log.G(ctx).Debug("Got Queue object")
 
-	err := q.handleQueueItemObject(ctx, key)
+	err = q.handleQueueItemObject(ctx, qi)
 	if err != nil {
 		// We've actually hit an error, so we set the span's status based on the error.
 		span.SetStatus(err)
@@ -162,35 +413,90 @@ func (q *Queue) handleQueueItem(ctx context.Context) bool {
 	return true
 }
 
-func (q *Queue) handleQueueItemObject(ctx context.Context, key string) error {
+func (q *Queue) handleQueueItemObject(ctx context.Context, qi *queueItem) error {
 	// This is a separate function / span, because the handleQueueItem span is the time spent waiting for the object
 	// plus the time spend handling the object. Instead, this function / span is scoped to a single object.
 	ctx, span := trace.StartSpan(ctx, "handleQueueItemObject")
 	defer span.End()
-	ctx = span.WithField(ctx, "key", key)
 
-	// We call Done here so the work Queue knows we have finished processing this item.
-	// We also must remember to call Forget if we do not want this work item being re-queued.
-	// For example, we do not call Forget if a transient error occurs.
-	// Instead, the item is put back on the work Queue and attempted again after a back-off period.
-	defer q.workqueue.Done(key)
+	ctx = span.WithFields(ctx, map[string]interface{}{
+		"requeues":        qi.requeues,
+		"originallyAdded": qi.originallyAdded.String(),
+		"addedViaRedirty": qi.addedViaRedirty,
+		"plannedForWork":  qi.plannedToStartWorkAt.String(),
+	})
+
+	if qi.delayedViaRateLimit != nil {
+		ctx = span.WithField(ctx, "delayedViaRateLimit", qi.delayedViaRateLimit.String())
+	}
 
 	// Add the current key as an attribute to the current span.
-	ctx = span.WithField(ctx, "key", key)
+	ctx = span.WithField(ctx, "key", qi.key)
 	// Run the syncHandler, passing it the namespace/name string of the Pod resource to be synced.
-	if err := q.handler(ctx, key); err != nil {
-		if q.workqueue.NumRequeues(key) < MaxRetries {
+	err := q.handler(ctx, qi.key)
+
+	q.lock.Lock()
+	defer q.lock.Unlock()
+
+	delete(q.itemsBeingProcessed, qi.key)
+	if qi.forget {
+		q.ratelimiter.Forget(qi.key)
+		log.G(ctx).WithError(err).Warnf("forgetting %q as told to forget while in progress", qi.key)
+		return nil
+	}
+
+	if err != nil {
+		ctx = span.WithField(ctx, "error", err.Error())
+		var delay *time.Duration
+
+		// Stash the original error for logging below
+		originalError := err
+		delay, err = q.retryFunc(ctx, qi.key, qi.requeues+1, qi.originallyAdded, err)
+		if err == nil {
 			// Put the item back on the work Queue to handle any transient errors.
-			log.G(ctx).WithError(err).Warnf("requeuing %q due to failed sync", key)
-			q.workqueue.AddRateLimited(key)
+			log.G(ctx).WithError(originalError).Warnf("requeuing %q due to failed sync", qi.key)
+			newQI := q.insert(ctx, qi.key, true, delay)
+			newQI.requeues = qi.requeues + 1
+			newQI.originallyAdded = qi.originallyAdded
+
 			return nil
 		}
-		// We've exceeded the maximum retries, so we must Forget the key.
-		q.workqueue.Forget(key)
-		return pkgerrors.Wrapf(err, "forgetting %q due to maximum retries reached", key)
+		if !qi.redirtiedAt.IsZero() {
+			err = fmt.Errorf("temporarily (requeued) forgetting %q due to: %w", qi.key, err)
+		} else {
+			err = fmt.Errorf("forgetting %q due to: %w", qi.key, err)
+		}
 	}
-	// Finally, if no error occurs we Forget this item so it does not get queued again until another change happens.
-	q.workqueue.Forget(key)
 
-	return nil
+	// We've exceeded the maximum retries or we were successful.
+	q.ratelimiter.Forget(qi.key)
+	if !qi.redirtiedAt.IsZero() {
+		delay := time.Until(qi.redirtiedAt)
+		newQI := q.insert(ctx, qi.key, qi.redirtiedWithRatelimit, &delay)
+		newQI.addedViaRedirty = true
+	}
+
+	span.SetStatus(err)
+	return err
+}
+
+func (q *Queue) String() string {
+	q.lock.Lock()
+	defer q.lock.Unlock()
+
+	items := make([]string, 0, q.items.Len())
+
+	for next := q.items.Front(); next != nil; next = next.Next() {
+		items = append(items, next.Value.(*queueItem).String())
+	}
+	return fmt.Sprintf("<items:%s>", items)
+}
+
+// DefaultRetryFunc is the default function used for retries by the queue subsystem.
+func DefaultRetryFunc(ctx context.Context, key string, timesTried int, originallyAdded time.Time, err error) (*time.Duration, error) {
+	if timesTried < MaxRetries {
+		return nil, nil
+	}
+
+	return nil, pkgerrors.Wrapf(err, "maximum retries (%d) reached", MaxRetries)
 }

internal/queue/queue_test.go

@@ -5,20 +5,24 @@ import (
 	"errors"
 	"strconv"
 	"sync"
+	"sync/atomic"
 	"testing"
 	"time"
 
 	"github.com/sirupsen/logrus"
 	"github.com/virtual-kubelet/virtual-kubelet/log"
 	logruslogger "github.com/virtual-kubelet/virtual-kubelet/log/logrus"
-	"go.uber.org/goleak"
 	"golang.org/x/time/rate"
 	"gotest.tools/assert"
 	is "gotest.tools/assert/cmp"
-	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/util/workqueue"
+	"k8s.io/utils/clock"
 )
 
+func durationPtr(d time.Duration) *time.Duration {
+	return &d
+}
+
 func TestQueueMaxRetries(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
@@ -35,15 +39,66 @@ func TestQueueMaxRetries(t *testing.T) {
 		// The default upper bound is 1000 seconds. Let's not use that.
 		workqueue.NewItemExponentialFailureRateLimiter(5*time.Millisecond, 10*time.Millisecond),
 		&workqueue.BucketRateLimiter{Limiter: rate.NewLimiter(rate.Limit(10), 100)},
-	), t.Name(), handler)
-	wq.Enqueue("test")
+	), t.Name(), handler, nil)
+	wq.Enqueue(context.TODO(), "test")
 
 	for n < MaxRetries {
 		assert.Assert(t, wq.handleQueueItem(ctx))
 	}
 
 	assert.Assert(t, is.Equal(n, MaxRetries))
-	assert.Assert(t, is.Equal(0, wq.workqueue.Len()))
+	assert.Assert(t, is.Equal(0, wq.Len()))
+}
+
+func TestQueueCustomRetries(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	logger := logrus.New()
+	logger.SetLevel(logrus.DebugLevel)
+	ctx = log.WithLogger(ctx, logruslogger.FromLogrus(logrus.NewEntry(logger)))
+	n := 0
+	errorSeen := 0
+	retryTestError := errors.New("Error should be retried every 10 milliseconds")
+	handler := func(ctx context.Context, key string) error {
+		if key == "retrytest" {
+			n++
+			return retryTestError
+		}
+		return errors.New("Unknown error")
+	}
+
+	shouldRetryFunc := func(ctx context.Context, key string, timesTried int, originallyAdded time.Time, err error) (*time.Duration, error) {
+		var sleepTime *time.Duration
+		if errors.Is(err, retryTestError) {
+			errorSeen++
+			sleepTime = durationPtr(10 * time.Millisecond)
+		}
+		_, retErr := DefaultRetryFunc(ctx, key, timesTried, originallyAdded, err)
+		return sleepTime, retErr
+	}
+
+	wq := New(&workqueue.BucketRateLimiter{Limiter: rate.NewLimiter(rate.Limit(1000), 1000)}, t.Name(), handler, shouldRetryFunc)
+
+	timeTaken := func(key string) time.Duration {
+		start := time.Now()
+		wq.Enqueue(context.TODO(), key)
+		for i := 0; i < MaxRetries; i++ {
+			assert.Assert(t, wq.handleQueueItem(ctx))
+		}
+		return time.Since(start)
+	}
+
+	unknownTime := timeTaken("unknown")
+	assert.Assert(t, n == 0)
+	assert.Assert(t, unknownTime < 10*time.Millisecond)
+
+	retrytestTime := timeTaken("retrytest")
+	assert.Assert(t, is.Equal(n, MaxRetries))
+	assert.Assert(t, is.Equal(errorSeen, MaxRetries))
+
+	assert.Assert(t, is.Equal(0, wq.Len()))
+	assert.Assert(t, retrytestTime > 10*time.Millisecond*time.Duration(n-1))
+	assert.Assert(t, retrytestTime < 2*10*time.Millisecond*time.Duration(n-1))
 }
 
 func TestForget(t *testing.T) {
@@ -51,65 +106,405 @@ func TestForget(t *testing.T) {
 	handler := func(ctx context.Context, key string) error {
 		panic("Should never be called")
 	}
-	wq := New(workqueue.DefaultItemBasedRateLimiter(), t.Name(), handler)
+	wq := New(workqueue.DefaultItemBasedRateLimiter(), t.Name(), handler, nil)
 
-	wq.Forget("val")
-	assert.Assert(t, is.Equal(0, wq.workqueue.Len()))
+	wq.Forget(context.TODO(), "val")
+	assert.Assert(t, is.Equal(0, wq.Len()))
 
 	v := "test"
-	wq.EnqueueWithoutRateLimit(v)
-	assert.Assert(t, is.Equal(1, wq.workqueue.Len()))
-
-	t.Skip("This is broken")
-	// Workqueue docs:
-	// Forget indicates that an item is finished being retried.  Doesn't matter whether it's for perm failing
-	// or for success, we'll stop the rate limiter from tracking it.  This only clears the `rateLimiter`, you
-	// still have to call `Done` on the queue.
-	// Even if you do this, it doesn't work: https://play.golang.com/p/8vfL_RCsFGI
-	assert.Assert(t, is.Equal(0, wq.workqueue.Len()))
-
+	wq.EnqueueWithoutRateLimit(context.TODO(), v)
+	assert.Assert(t, is.Equal(1, wq.Len()))
 }
 
-func TestQueueTerminate(t *testing.T) {
+func TestQueueEmpty(t *testing.T) {
 	t.Parallel()
-	defer goleak.VerifyNone(t,
-		// Ignore existing goroutines
-		goleak.IgnoreCurrent(),
-		// Ignore klog background flushers
-		goleak.IgnoreTopFunction("k8s.io/klog.(*loggingT).flushDaemon"),
-		goleak.IgnoreTopFunction("k8s.io/klog/v2.(*loggingT).flushDaemon"),
-		// Workqueue runs a goroutine in the background to handle background functions. AFAICT, they're unkillable
-		// and are designed to stop after a certain idle window
-		goleak.IgnoreTopFunction("k8s.io/client-go/util/workqueue.(*Type).updateUnfinishedWorkLoop"),
-		goleak.IgnoreTopFunction("k8s.io/client-go/util/workqueue.(*delayingType).waitingLoop"),
-	)
-	ctx, cancel := context.WithCancel(context.Background())
+	ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
 	defer cancel()
 
-	testMap := &sync.Map{}
-	handler := func(ctx context.Context, key string) error {
-		testMap.Store(key, struct{}{})
+	q := New(workqueue.DefaultItemBasedRateLimiter(), t.Name(), func(ctx context.Context, key string) error {
 		return nil
-	}
+	}, nil)
 
-	wq := New(workqueue.DefaultItemBasedRateLimiter(), t.Name(), handler)
-	group := &wait.Group{}
-	group.StartWithContext(ctx, func(ctx context.Context) {
-		wq.Run(ctx, 10)
+	item, err := q.getNextItem(ctx)
+	assert.Error(t, err, context.DeadlineExceeded.Error())
+	assert.Assert(t, is.Nil(item))
+}
+
+func TestQueueItemNoSleep(t *testing.T) {
+	t.Parallel()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 1000*time.Millisecond)
+	defer cancel()
+
+	q := New(workqueue.DefaultItemBasedRateLimiter(), t.Name(), func(ctx context.Context, key string) error {
+		return nil
+	}, nil)
+
+	q.lock.Lock()
+	q.insert(ctx, "foo", false, durationPtr(-1*time.Hour))
+	q.insert(ctx, "bar", false, durationPtr(-1*time.Hour))
+	q.lock.Unlock()
+
+	item, err := q.getNextItem(ctx)
+	assert.NilError(t, err)
+	assert.Assert(t, is.Equal(item.key, "foo"))
+
+	item, err = q.getNextItem(ctx)
+	assert.NilError(t, err)
+	assert.Assert(t, is.Equal(item.key, "bar"))
+}
+
+func TestQueueItemSleep(t *testing.T) {
+	t.Parallel()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 1000*time.Millisecond)
+	defer cancel()
+
+	q := New(workqueue.DefaultItemBasedRateLimiter(), t.Name(), func(ctx context.Context, key string) error {
+		return nil
+	}, nil)
+	q.lock.Lock()
+	q.insert(ctx, "foo", false, durationPtr(100*time.Millisecond))
+	q.insert(ctx, "bar", false, durationPtr(100*time.Millisecond))
+	q.lock.Unlock()
+
+	item, err := q.getNextItem(ctx)
+	assert.NilError(t, err)
+	assert.Assert(t, is.Equal(item.key, "foo"))
+}
+
+func TestQueueBackgroundAdd(t *testing.T) {
+	t.Parallel()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5000*time.Millisecond)
+	defer cancel()
+
+	q := New(workqueue.DefaultItemBasedRateLimiter(), t.Name(), func(ctx context.Context, key string) error {
+		return nil
+	}, nil)
+	start := time.Now()
+	time.AfterFunc(100*time.Millisecond, func() {
+		q.lock.Lock()
+		defer q.lock.Unlock()
+		q.insert(ctx, "foo", false, nil)
 	})
-	for i := 0; i < 1000; i++ {
-		wq.EnqueueWithoutRateLimit(strconv.Itoa(i))
+
+	item, err := q.getNextItem(ctx)
+	assert.NilError(t, err)
+	assert.Assert(t, is.Equal(item.key, "foo"))
+	assert.Assert(t, time.Since(start) > 100*time.Millisecond)
+}
+
+func TestQueueBackgroundAdvance(t *testing.T) {
+	t.Parallel()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5000*time.Millisecond)
+	defer cancel()
+
+	q := New(workqueue.DefaultItemBasedRateLimiter(), t.Name(), func(ctx context.Context, key string) error {
+		return nil
+	}, nil)
+	start := time.Now()
+	q.lock.Lock()
+	q.insert(ctx, "foo", false, durationPtr(10*time.Second))
+	q.lock.Unlock()
+
+	time.AfterFunc(200*time.Millisecond, func() {
+		q.lock.Lock()
+		defer q.lock.Unlock()
+		q.insert(ctx, "foo", false, nil)
+	})
+
+	item, err := q.getNextItem(ctx)
+	assert.NilError(t, err)
+	assert.Assert(t, is.Equal(item.key, "foo"))
+	assert.Assert(t, time.Since(start) > 200*time.Millisecond)
+	assert.Assert(t, time.Since(start) < 5*time.Second)
+}
+
+func TestQueueRedirty(t *testing.T) {
+	t.Parallel()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5000*time.Millisecond)
+	defer cancel()
+
+	var times int64
+	var q *Queue
+	q = New(workqueue.DefaultItemBasedRateLimiter(), t.Name(), func(ctx context.Context, key string) error {
+		assert.Assert(t, is.Equal(key, "foo"))
+		if atomic.AddInt64(&times, 1) == 1 {
+			q.EnqueueWithoutRateLimit(context.TODO(), "foo")
+		} else {
+			cancel()
+		}
+		return nil
+	}, nil)
+
+	q.EnqueueWithoutRateLimit(context.TODO(), "foo")
+	q.Run(ctx, 1)
+	for !q.Empty() {
+		time.Sleep(100 * time.Millisecond)
+	}
+	assert.Assert(t, is.Equal(atomic.LoadInt64(&times), int64(2)))
+}
+
+func TestHeapConcurrency(t *testing.T) {
+	t.Parallel()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5000*time.Millisecond)
+	defer cancel()
+
+	start := time.Now()
+	seen := sync.Map{}
+	q := New(workqueue.DefaultItemBasedRateLimiter(), t.Name(), func(ctx context.Context, key string) error {
+		seen.Store(key, struct{}{})
+		time.Sleep(time.Second)
+		return nil
+	}, nil)
+	for i := 0; i < 20; i++ {
+		q.EnqueueWithoutRateLimit(context.TODO(), strconv.Itoa(i))
 	}
 
-	for wq.workqueue.Len() > 0 {
+	assert.Assert(t, q.Len() == 20)
+
+	go q.Run(ctx, 20)
+	for q.Len() > 0 {
 		time.Sleep(100 * time.Millisecond)
 	}
 
-	for i := 0; i < 1000; i++ {
-		_, ok := testMap.Load(strconv.Itoa(i))
-		assert.Assert(t, ok, "Item %d missing", i)
+	for i := 0; i < 20; i++ {
+		_, ok := seen.Load(strconv.Itoa(i))
+		assert.Assert(t, ok, "Did not observe: %d", i)
+	}
+	assert.Assert(t, time.Since(start) < 5*time.Second)
+}
+
+func checkConsistency(t *testing.T, q *Queue) {
+	q.lock.Lock()
+	defer q.lock.Unlock()
+
+	for next := q.items.Front(); next != nil && next.Next() != nil; next = next.Next() {
+		qi := next.Value.(*queueItem)
+		qiNext := next.Next().Value.(*queueItem)
+		assert.Assert(t, qi.plannedToStartWorkAt.Before(qiNext.plannedToStartWorkAt) || qi.plannedToStartWorkAt.Equal(qiNext.plannedToStartWorkAt))
+	}
+}
+
+func TestHeapOrder(t *testing.T) {
+	q := New(workqueue.DefaultItemBasedRateLimiter(), t.Name(), func(ctx context.Context, key string) error {
+		return nil
+	}, nil)
+	q.clock = nonmovingClock{}
+
+	q.EnqueueWithoutRateLimitWithDelay(context.TODO(), "a", 1000)
+	q.EnqueueWithoutRateLimitWithDelay(context.TODO(), "b", 2000)
+	q.EnqueueWithoutRateLimitWithDelay(context.TODO(), "c", 3000)
+	q.EnqueueWithoutRateLimitWithDelay(context.TODO(), "d", 4000)
+	q.EnqueueWithoutRateLimitWithDelay(context.TODO(), "e", 5000)
+	checkConsistency(t, q)
+	t.Logf("%v", q)
+	q.EnqueueWithoutRateLimitWithDelay(context.TODO(), "d", 1000)
+	checkConsistency(t, q)
+	t.Logf("%v", q)
+	q.EnqueueWithoutRateLimitWithDelay(context.TODO(), "c", 1001)
+	checkConsistency(t, q)
+	t.Logf("%v", q)
+	q.EnqueueWithoutRateLimitWithDelay(context.TODO(), "e", 999)
+	checkConsistency(t, q)
+	t.Logf("%v", q)
+}
+
+type rateLimitWrapper struct {
+	addedMap     sync.Map
+	forgottenMap sync.Map
+	rl           workqueue.RateLimiter
+}
+
+func (r *rateLimitWrapper) When(item interface{}) time.Duration {
+	if _, ok := r.forgottenMap.Load(item); ok {
+		r.forgottenMap.Delete(item)
+		// Reset the added map
+		r.addedMap.Store(item, 1)
+	} else {
+		actual, loaded := r.addedMap.LoadOrStore(item, 1)
+		if loaded {
+			r.addedMap.Store(item, actual.(int)+1)
+		}
 	}
 
-	cancel()
-	group.Wait()
+	return r.rl.When(item)
+}
+
+func (r *rateLimitWrapper) Forget(item interface{}) {
+	r.forgottenMap.Store(item, struct{}{})
+	r.rl.Forget(item)
+}
+
+func (r *rateLimitWrapper) NumRequeues(item interface{}) int {
+	return r.rl.NumRequeues(item)
+}
+
+func TestRateLimiter(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 5000*time.Millisecond)
+	defer cancel()
+
+	syncMap := sync.Map{}
+	syncMap.Store("foo", 0)
+	syncMap.Store("bar", 0)
+	syncMap.Store("baz", 0)
+	syncMap.Store("quux", 0)
+
+	start := time.Now()
+	ratelimiter := &rateLimitWrapper{
+		rl: workqueue.NewItemFastSlowRateLimiter(1*time.Millisecond, 100*time.Millisecond, 1),
+	}
+
+	q := New(ratelimiter, t.Name(), func(ctx context.Context, key string) error {
+		oldValue, _ := syncMap.Load(key)
+		syncMap.Store(key, oldValue.(int)+1)
+		if oldValue.(int) < 9 {
+			return errors.New("test")
+		}
+		return nil
+	}, nil)
+
+	enqueued := 0
+	syncMap.Range(func(key, value interface{}) bool {
+		enqueued++
+		q.Enqueue(context.TODO(), key.(string))
+		return true
+	})
+
+	assert.Assert(t, enqueued == 4)
+	go q.Run(ctx, 10)
+
+	incomplete := true
+	for incomplete {
+		time.Sleep(10 * time.Millisecond)
+		incomplete = false
+		// Wait for all items to finish processing.
+		syncMap.Range(func(key, value interface{}) bool {
+			if value.(int) < 10 {
+				incomplete = true
+			}
+			return true
+		})
+	}
+
+	// Make sure there were ~9 "slow" rate limits per item, and 1 fast
+	assert.Assert(t, time.Since(start) > 9*100*time.Millisecond)
+	// Make sure we didn't go off the deep end.
+	assert.Assert(t, time.Since(start) < 2*9*100*time.Millisecond)
+
+	// Make sure each item was seen. And Forgotten.
+	syncMap.Range(func(key, value interface{}) bool {
+		_, ok := ratelimiter.forgottenMap.Load(key)
+		assert.Assert(t, ok, "%s in forgotten map", key)
+		val, ok := ratelimiter.addedMap.Load(key)
+		assert.Assert(t, ok, "%s in added map", key)
+		assert.Assert(t, val == 10)
+		return true
+	})
+
+	q.lock.Lock()
+	defer q.lock.Unlock()
+	assert.Assert(t, len(q.itemsInQueue) == 0)
+	assert.Assert(t, len(q.itemsBeingProcessed) == 0)
+	assert.Assert(t, q.items.Len() == 0)
+
+}
+
+func TestQueueForgetInProgress(t *testing.T) {
+	t.Parallel()
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	var times int64
+	var q *Queue
+	q = New(workqueue.DefaultItemBasedRateLimiter(), t.Name(), func(ctx context.Context, key string) error {
+		assert.Assert(t, is.Equal(key, "foo"))
+		atomic.AddInt64(&times, 1)
+		q.Forget(context.TODO(), key)
+		return errors.New("test")
+	}, nil)
+
+	q.EnqueueWithoutRateLimit(context.TODO(), "foo")
+	go q.Run(ctx, 1)
+	for !q.Empty() {
+		time.Sleep(100 * time.Millisecond)
+	}
+	assert.Assert(t, is.Equal(atomic.LoadInt64(&times), int64(1)))
+}
+
+func TestQueueForgetBeforeStart(t *testing.T) {
+	t.Parallel()
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	q := New(workqueue.DefaultItemBasedRateLimiter(), t.Name(), func(ctx context.Context, key string) error {
+		panic("shouldn't be called")
+	}, nil)
+
+	q.EnqueueWithoutRateLimit(context.TODO(), "foo")
+	q.Forget(context.TODO(), "foo")
+	go q.Run(ctx, 1)
+	for !q.Empty() {
+		time.Sleep(100 * time.Millisecond)
+	}
+}
+
+func TestQueueMoveItem(t *testing.T) {
+	t.Parallel()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	q := New(workqueue.DefaultItemBasedRateLimiter(), t.Name(), func(ctx context.Context, key string) error {
+		panic("shouldn't be called")
+	}, nil)
+	q.clock = nonmovingClock{}
+
+	q.insert(ctx, "foo", false, durationPtr(3000))
+	q.insert(ctx, "bar", false, durationPtr(2000))
+	q.insert(ctx, "baz", false, durationPtr(1000))
+	checkConsistency(t, q)
+	t.Log(q)
+
+	q.insert(ctx, "foo", false, durationPtr(2000))
+	checkConsistency(t, q)
+	t.Log(q)
+
+	q.insert(ctx, "foo", false, durationPtr(1999))
+	checkConsistency(t, q)
+	t.Log(q)
+
+	q.insert(ctx, "foo", false, durationPtr(999))
+	checkConsistency(t, q)
+	t.Log(q)
+}
+
+type nonmovingClock struct {
+}
+
+func (n nonmovingClock) Now() time.Time {
+	return time.Time{}
+}
+
+func (n nonmovingClock) Since(t time.Time) time.Duration {
+	return n.Now().Sub(t)
+}
+
+func (n nonmovingClock) After(d time.Duration) <-chan time.Time {
+	panic("implement me")
+}
+
+func (n nonmovingClock) NewTimer(d time.Duration) clock.Timer {
+	panic("implement me")
+}
+
+func (n nonmovingClock) Sleep(d time.Duration) {
+	panic("implement me")
+}
+
+func (n nonmovingClock) Tick(d time.Duration) <-chan time.Time {
+	panic("implement me")
 }

internal/test/e2e/framework/pod.go

@@ -6,13 +6,13 @@ import (
 	"strings"
 
 	corev1 "k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/runtime"
 	watchapi "k8s.io/apimachinery/pkg/watch"
 	"k8s.io/client-go/tools/cache"
 	"k8s.io/client-go/tools/watch"
-	podutil "k8s.io/kubernetes/pkg/api/v1/pod"
 )
 
 // CreateDummyPodObjectWithPrefix creates a dujmmy pod object using the specified prefix as the value of .metadata.generateName.
@@ -101,10 +101,20 @@ func (f *Framework) WaitUntilPodCondition(namespace, name string, fn watch.Condi
 func (f *Framework) WaitUntilPodReady(namespace, name string) (*corev1.Pod, error) {
 	return f.WaitUntilPodCondition(namespace, name, func(event watchapi.Event) (bool, error) {
 		pod := event.Object.(*corev1.Pod)
-		return pod.Status.Phase == corev1.PodRunning && podutil.IsPodReady(pod) && pod.Status.PodIP != "", nil
+		return pod.Status.Phase == corev1.PodRunning && IsPodReady(pod) && pod.Status.PodIP != "", nil
 	})
 }
 
+// IsPodReady returns true if a pod is ready.
+func IsPodReady(pod *v1.Pod) bool {
+	for _, cond := range pod.Status.Conditions {
+		if cond.Type == v1.PodReady && cond.Status == v1.ConditionTrue {
+			return true
+		}
+	}
+	return false
+}
+
 // WaitUntilPodDeleted blocks until the pod with the specified name and namespace is deleted from apiserver.
 func (f *Framework) WaitUntilPodDeleted(namespace, name string) (*corev1.Pod, error) {
 	return f.WaitUntilPodCondition(namespace, name, func(event watchapi.Event) (bool, error) {

internal/test/e2e/framework/stats.go

@@ -3,10 +3,9 @@ package framework
 import (
 	"context"
 	"encoding/json"
-	"strconv"
 
 	"k8s.io/apimachinery/pkg/util/net"
-	stats "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
+	stats "k8s.io/kubelet/pkg/apis/stats/v1alpha1"
 )
 
 // GetStatsSummary queries the /stats/summary endpoint of the virtual-kubelet and returns the Summary object obtained as a response.
@@ -18,7 +17,7 @@ func (f *Framework) GetStatsSummary(ctx context.Context) (*stats.Summary, error)
 		Namespace(f.Namespace).
 		Resource("pods").
 		SubResource("proxy").
-		Name(net.JoinSchemeNamePort("http", f.NodeName, strconv.Itoa(10255))).
+		Name(net.JoinSchemeNamePort("https", f.NodeName, "10250")).
 		Suffix("/stats/summary").DoRaw(ctx)
 	if err != nil {
 		return nil, err

internal/token/token_manager.go

@@ -0,0 +1,206 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package token implements a manager of serviceaccount tokens for pods running
+// on the node.
+package token
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"math/rand"
+	"sync"
+	"time"
+
+	authenticationv1 "k8s.io/api/authentication/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/wait"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/klog/v2"
+	"k8s.io/utils/clock"
+)
+
+const (
+	maxTTL    = 24 * time.Hour
+	gcPeriod  = time.Minute
+	maxJitter = 10 * time.Second
+)
+
+// NewManager returns a new token manager.
+func NewManager(c clientset.Interface) *Manager {
+	// check whether the server supports token requests so we can give a more helpful error message
+	supported := false
+	once := &sync.Once{}
+	tokenRequestsSupported := func() bool {
+		once.Do(func() {
+			resources, err := c.Discovery().ServerResourcesForGroupVersion("v1")
+			if err != nil {
+				return
+			}
+			for _, resource := range resources.APIResources {
+				if resource.Name == "serviceaccounts/token" {
+					supported = true
+					return
+				}
+			}
+		})
+		return supported
+	}
+
+	m := &Manager{
+		getToken: func(name, namespace string, tr *authenticationv1.TokenRequest) (*authenticationv1.TokenRequest, error) {
+			if c == nil {
+				return nil, errors.New("cannot use TokenManager when kubelet is in standalone mode")
+			}
+			tokenRequest, err := c.CoreV1().ServiceAccounts(namespace).CreateToken(context.TODO(), name, tr, metav1.CreateOptions{})
+			if apierrors.IsNotFound(err) && !tokenRequestsSupported() {
+				return nil, fmt.Errorf("the API server does not have TokenRequest endpoints enabled")
+			}
+			return tokenRequest, err
+		},
+		cache: make(map[string]*authenticationv1.TokenRequest),
+		clock: clock.RealClock{},
+	}
+	go wait.Forever(m.cleanup, gcPeriod)
+	return m
+}
+
+// Manager manages service account tokens for pods.
+type Manager struct {
+
+	// cacheMutex guards the cache
+	cacheMutex sync.RWMutex
+	cache      map[string]*authenticationv1.TokenRequest
+
+	// mocked for testing
+	getToken func(name, namespace string, tr *authenticationv1.TokenRequest) (*authenticationv1.TokenRequest, error)
+	clock    clock.Clock
+}
+
+// GetServiceAccountToken gets a service account token for a pod from cache or
+// from the TokenRequest API. This process is as follows:
+// * Check the cache for the current token request.
+// * If the token exists and does not require a refresh, return the current token.
+// * Attempt to refresh the token.
+// * If the token is refreshed successfully, save it in the cache and return the token.
+// * If refresh fails and the old token is still valid, log an error and return the old token.
+// * If refresh fails and the old token is no longer valid, return an error
+func (m *Manager) GetServiceAccountToken(namespace, name string, tr *authenticationv1.TokenRequest) (*authenticationv1.TokenRequest, error) {
+	key := keyFunc(name, namespace, tr)
+
+	ctr, ok := m.get(key)
+
+	if ok && !m.requiresRefresh(ctr) {
+		return ctr, nil
+	}
+
+	tr, err := m.getToken(name, namespace, tr)
+	if err != nil {
+		switch {
+		case !ok:
+			return nil, fmt.Errorf("failed to fetch token: %v", err)
+		case m.expired(ctr):
+			return nil, fmt.Errorf("token %s expired and refresh failed: %v", key, err)
+		default:
+			klog.ErrorS(err, "Couldn't update token", "cacheKey", key)
+			return ctr, nil
+		}
+	}
+
+	m.set(key, tr)
+	return tr, nil
+}
+
+// DeleteServiceAccountToken should be invoked when pod got deleted. It simply
+// clean token manager cache.
+func (m *Manager) DeleteServiceAccountToken(podUID types.UID) {
+	m.cacheMutex.Lock()
+	defer m.cacheMutex.Unlock()
+	for k, tr := range m.cache {
+		if tr.Spec.BoundObjectRef.UID == podUID {
+			delete(m.cache, k)
+		}
+	}
+}
+
+func (m *Manager) cleanup() {
+	m.cacheMutex.Lock()
+	defer m.cacheMutex.Unlock()
+	for k, tr := range m.cache {
+		if m.expired(tr) {
+			delete(m.cache, k)
+		}
+	}
+}
+
+func (m *Manager) get(key string) (*authenticationv1.TokenRequest, bool) {
+	m.cacheMutex.RLock()
+	defer m.cacheMutex.RUnlock()
+	ctr, ok := m.cache[key]
+	return ctr, ok
+}
+
+func (m *Manager) set(key string, tr *authenticationv1.TokenRequest) {
+	m.cacheMutex.Lock()
+	defer m.cacheMutex.Unlock()
+	m.cache[key] = tr
+}
+
+func (m *Manager) expired(t *authenticationv1.TokenRequest) bool {
+	return m.clock.Now().After(t.Status.ExpirationTimestamp.Time)
+}
+
+// requiresRefresh returns true if the token is older than 80% of its total
+// ttl, or if the token is older than 24 hours.
+func (m *Manager) requiresRefresh(tr *authenticationv1.TokenRequest) bool {
+	if tr.Spec.ExpirationSeconds == nil {
+		cpy := tr.DeepCopy()
+		cpy.Status.Token = ""
+		klog.ErrorS(nil, "Expiration seconds was nil for token request", "tokenRequest", cpy)
+		return false
+	}
+	now := m.clock.Now()
+	exp := tr.Status.ExpirationTimestamp.Time
+	iat := exp.Add(-1 * time.Duration(*tr.Spec.ExpirationSeconds) * time.Second)
+
+	jitter := time.Duration(rand.Float64()*maxJitter.Seconds()) * time.Second
+	if now.After(iat.Add(maxTTL - jitter)) {
+		return true
+	}
+	// Require a refresh if within 20% of the TTL plus a jitter from the expiration time.
+	if now.After(exp.Add(-1*time.Duration((*tr.Spec.ExpirationSeconds*20)/100)*time.Second - jitter)) {
+		return true
+	}
+	return false
+}
+
+// keys should be nonconfidential and safe to log
+func keyFunc(name, namespace string, tr *authenticationv1.TokenRequest) string {
+	var exp int64
+	if tr.Spec.ExpirationSeconds != nil {
+		exp = *tr.Spec.ExpirationSeconds
+	}
+
+	var ref authenticationv1.BoundObjectReference
+	if tr.Spec.BoundObjectRef != nil {
+		ref = *tr.Spec.BoundObjectRef
+	}
+
+	return fmt.Sprintf("%q/%q/%#v/%#v/%#v", name, namespace, tr.Spec.Audiences, exp, ref)
+}

internal/token/token_manager_test.go

@@ -0,0 +1,606 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package token
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	authenticationv1 "k8s.io/api/authentication/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	testingclock "k8s.io/utils/clock/testing"
+)
+
+func TestTokenCachingAndExpiration(t *testing.T) {
+	type suite struct {
+		clock *testingclock.FakeClock
+		tg    *fakeTokenGetter
+		mgr   *Manager
+	}
+
+	cases := []struct {
+		name string
+		exp  time.Duration
+		f    func(t *testing.T, s *suite)
+	}{
+		{
+			name: "rotate hour token expires in the last 12 minutes",
+			exp:  time.Hour,
+			f: func(t *testing.T, s *suite) {
+				s.clock.SetTime(s.clock.Now().Add(50 * time.Minute))
+				if _, err := s.mgr.GetServiceAccountToken("a", "b", getTokenRequest()); err != nil {
+					t.Fatalf("unexpected error: %v", err)
+				}
+				if s.tg.count != 2 {
+					t.Fatalf("expected token to be refreshed: call count was %d", s.tg.count)
+				}
+			},
+		},
+		{
+			name: "rotate 24 hour token that expires in 40 hours",
+			exp:  40 * time.Hour,
+			f: func(t *testing.T, s *suite) {
+				s.clock.SetTime(s.clock.Now().Add(25 * time.Hour))
+				if _, err := s.mgr.GetServiceAccountToken("a", "b", getTokenRequest()); err != nil {
+					t.Fatalf("unexpected error: %v", err)
+				}
+				if s.tg.count != 2 {
+					t.Fatalf("expected token to be refreshed: call count was %d", s.tg.count)
+				}
+			},
+		},
+		{
+			name: "rotate hour token fails, old token is still valid, doesn't error",
+			exp:  time.Hour,
+			f: func(t *testing.T, s *suite) {
+				s.clock.SetTime(s.clock.Now().Add(50 * time.Minute))
+				tg := &fakeTokenGetter{
+					err: fmt.Errorf("err"),
+				}
+				s.mgr.getToken = tg.getToken
+				tr, err := s.mgr.GetServiceAccountToken("a", "b", getTokenRequest())
+				if err != nil {
+					t.Fatalf("unexpected error: %v", err)
+				}
+				if tr.Status.Token != "foo" {
+					t.Fatalf("unexpected token: %v", tr.Status.Token)
+				}
+			},
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			clock := testingclock.NewFakeClock(time.Time{}.Add(30 * 24 * time.Hour))
+			expSecs := int64(c.exp.Seconds())
+			s := &suite{
+				clock: clock,
+				mgr:   NewManager(nil),
+				tg: &fakeTokenGetter{
+					tr: &authenticationv1.TokenRequest{
+						Spec: authenticationv1.TokenRequestSpec{
+							ExpirationSeconds: &expSecs,
+						},
+						Status: authenticationv1.TokenRequestStatus{
+							Token:               "foo",
+							ExpirationTimestamp: metav1.Time{Time: clock.Now().Add(c.exp)},
+						},
+					},
+				},
+			}
+			s.mgr.getToken = s.tg.getToken
+			s.mgr.clock = s.clock
+			if _, err := s.mgr.GetServiceAccountToken("a", "b", getTokenRequest()); err != nil {
+				t.Fatalf("unexpected error: %v", err)
+			}
+			if s.tg.count != 1 {
+				t.Fatalf("unexpected client call, got: %d, want: 1", s.tg.count)
+			}
+
+			if _, err := s.mgr.GetServiceAccountToken("a", "b", getTokenRequest()); err != nil {
+				t.Fatalf("unexpected error: %v", err)
+			}
+			if s.tg.count != 1 {
+				t.Fatalf("expected token to be served from cache: saw %d", s.tg.count)
+			}
+
+			c.f(t, s)
+		})
+	}
+}
+
+func TestRequiresRefresh(t *testing.T) {
+	start := time.Now()
+	cases := []struct {
+		now, exp      time.Time
+		expectRefresh bool
+		requestTweaks func(*authenticationv1.TokenRequest)
+	}{
+		{
+			now:           start.Add(10 * time.Minute),
+			exp:           start.Add(60 * time.Minute),
+			expectRefresh: false,
+		},
+		{
+			now:           start.Add(50 * time.Minute),
+			exp:           start.Add(60 * time.Minute),
+			expectRefresh: true,
+		},
+		{
+			now:           start.Add(25 * time.Hour),
+			exp:           start.Add(60 * time.Hour),
+			expectRefresh: true,
+		},
+		{
+			now:           start.Add(70 * time.Minute),
+			exp:           start.Add(60 * time.Minute),
+			expectRefresh: true,
+		},
+		{
+			// expiry will be overwritten by the tweak below.
+			now:           start.Add(0 * time.Minute),
+			exp:           start.Add(60 * time.Minute),
+			expectRefresh: false,
+			requestTweaks: func(tr *authenticationv1.TokenRequest) {
+				tr.Spec.ExpirationSeconds = nil
+			},
+		},
+	}
+
+	for i, c := range cases {
+		t.Run(fmt.Sprint(i), func(t *testing.T) {
+			clock := testingclock.NewFakeClock(c.now)
+			secs := int64(c.exp.Sub(start).Seconds())
+			tr := &authenticationv1.TokenRequest{
+				Spec: authenticationv1.TokenRequestSpec{
+					ExpirationSeconds: &secs,
+				},
+				Status: authenticationv1.TokenRequestStatus{
+					ExpirationTimestamp: metav1.Time{Time: c.exp},
+				},
+			}
+
+			if c.requestTweaks != nil {
+				c.requestTweaks(tr)
+			}
+
+			mgr := NewManager(nil)
+			mgr.clock = clock
+
+			rr := mgr.requiresRefresh(tr)
+			if rr != c.expectRefresh {
+				t.Fatalf("unexpected requiresRefresh result, got: %v, want: %v", rr, c.expectRefresh)
+			}
+		})
+	}
+}
+
+func TestDeleteServiceAccountToken(t *testing.T) {
+	type request struct {
+		name, namespace string
+		tr              authenticationv1.TokenRequest
+		shouldFail      bool
+	}
+
+	cases := []struct {
+		name         string
+		requestIndex []int
+		deletePodUID []types.UID
+		expLeftIndex []int
+	}{
+		{
+			name:         "delete none with all success requests",
+			requestIndex: []int{0, 1, 2},
+			expLeftIndex: []int{0, 1, 2},
+		},
+		{
+			name:         "delete one with all success requests",
+			requestIndex: []int{0, 1, 2},
+			deletePodUID: []types.UID{"fake-uid-1"},
+			expLeftIndex: []int{1, 2},
+		},
+		{
+			name:         "delete two with all success requests",
+			requestIndex: []int{0, 1, 2},
+			deletePodUID: []types.UID{"fake-uid-1", "fake-uid-3"},
+			expLeftIndex: []int{1},
+		},
+		{
+			name:         "delete all with all success requests",
+			requestIndex: []int{0, 1, 2},
+			deletePodUID: []types.UID{"fake-uid-1", "fake-uid-2", "fake-uid-3"},
+		},
+		{
+			name:         "delete no pod with failed requests",
+			requestIndex: []int{0, 1, 2, 3},
+			deletePodUID: []types.UID{},
+			expLeftIndex: []int{0, 1, 2},
+		},
+		{
+			name:         "delete other pod with failed requests",
+			requestIndex: []int{0, 1, 2, 3},
+			deletePodUID: []types.UID{"fake-uid-2"},
+			expLeftIndex: []int{0, 2},
+		},
+		{
+			name:         "delete no pod with request which success after failure",
+			requestIndex: []int{0, 1, 2, 3, 4},
+			deletePodUID: []types.UID{},
+			expLeftIndex: []int{0, 1, 2, 4},
+		},
+		{
+			name:         "delete the pod which success after failure",
+			requestIndex: []int{0, 1, 2, 3, 4},
+			deletePodUID: []types.UID{"fake-uid-4"},
+			expLeftIndex: []int{0, 1, 2},
+		},
+		{
+			name:         "delete other pod with request which success after failure",
+			requestIndex: []int{0, 1, 2, 3, 4},
+			deletePodUID: []types.UID{"fake-uid-1"},
+			expLeftIndex: []int{1, 2, 4},
+		},
+		{
+			name:         "delete some pod not in the set",
+			requestIndex: []int{0, 1, 2},
+			deletePodUID: []types.UID{"fake-uid-100", "fake-uid-200"},
+			expLeftIndex: []int{0, 1, 2},
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			requests := []request{
+				{
+					name:      "fake-name-1",
+					namespace: "fake-namespace-1",
+					tr: authenticationv1.TokenRequest{
+						Spec: authenticationv1.TokenRequestSpec{
+							BoundObjectRef: &authenticationv1.BoundObjectReference{
+								UID:  "fake-uid-1",
+								Name: "fake-name-1",
+							},
+						},
+					},
+					shouldFail: false,
+				},
+				{
+					name:      "fake-name-2",
+					namespace: "fake-namespace-2",
+					tr: authenticationv1.TokenRequest{
+						Spec: authenticationv1.TokenRequestSpec{
+							BoundObjectRef: &authenticationv1.BoundObjectReference{
+								UID:  "fake-uid-2",
+								Name: "fake-name-2",
+							},
+						},
+					},
+					shouldFail: false,
+				},
+				{
+					name:      "fake-name-3",
+					namespace: "fake-namespace-3",
+					tr: authenticationv1.TokenRequest{
+						Spec: authenticationv1.TokenRequestSpec{
+							BoundObjectRef: &authenticationv1.BoundObjectReference{
+								UID:  "fake-uid-3",
+								Name: "fake-name-3",
+							},
+						},
+					},
+					shouldFail: false,
+				},
+				{
+					name:      "fake-name-4",
+					namespace: "fake-namespace-4",
+					tr: authenticationv1.TokenRequest{
+						Spec: authenticationv1.TokenRequestSpec{
+							BoundObjectRef: &authenticationv1.BoundObjectReference{
+								UID:  "fake-uid-4",
+								Name: "fake-name-4",
+							},
+						},
+					},
+					shouldFail: true,
+				},
+				{
+					//exactly the same with last one, besides it will success
+					name:      "fake-name-4",
+					namespace: "fake-namespace-4",
+					tr: authenticationv1.TokenRequest{
+						Spec: authenticationv1.TokenRequestSpec{
+							BoundObjectRef: &authenticationv1.BoundObjectReference{
+								UID:  "fake-uid-4",
+								Name: "fake-name-4",
+							},
+						},
+					},
+					shouldFail: false,
+				},
+			}
+			testMgr := NewManager(nil)
+			testMgr.clock = testingclock.NewFakeClock(time.Time{}.Add(30 * 24 * time.Hour))
+
+			successGetToken := func(_, _ string, tr *authenticationv1.TokenRequest) (*authenticationv1.TokenRequest, error) {
+				tr.Status = authenticationv1.TokenRequestStatus{
+					ExpirationTimestamp: metav1.Time{Time: testMgr.clock.Now().Add(10 * time.Hour)},
+				}
+				return tr, nil
+			}
+			failGetToken := func(_, _ string, tr *authenticationv1.TokenRequest) (*authenticationv1.TokenRequest, error) {
+				return nil, fmt.Errorf("fail tr")
+			}
+
+			for _, index := range c.requestIndex {
+				req := requests[index]
+				if req.shouldFail {
+					testMgr.getToken = failGetToken
+				} else {
+					testMgr.getToken = successGetToken
+				}
+				testMgr.GetServiceAccountToken(req.namespace, req.name, &req.tr)
+			}
+
+			for _, uid := range c.deletePodUID {
+				testMgr.DeleteServiceAccountToken(uid)
+			}
+			if len(c.expLeftIndex) != len(testMgr.cache) {
+				t.Errorf("%s got unexpected result: expected left cache size is %d, got %d", c.name, len(c.expLeftIndex), len(testMgr.cache))
+			}
+			for _, leftIndex := range c.expLeftIndex {
+				r := requests[leftIndex]
+				_, ok := testMgr.get(keyFunc(r.name, r.namespace, &r.tr))
+				if !ok {
+					t.Errorf("%s got unexpected result: expected token request %v exist in cache, but not", c.name, r)
+				}
+			}
+		})
+	}
+}
+
+type fakeTokenGetter struct {
+	count int
+	tr    *authenticationv1.TokenRequest
+	err   error
+}
+
+func (ftg *fakeTokenGetter) getToken(name, namespace string, tr *authenticationv1.TokenRequest) (*authenticationv1.TokenRequest, error) {
+	ftg.count++
+	return ftg.tr, ftg.err
+}
+
+func TestCleanup(t *testing.T) {
+	cases := []struct {
+		name              string
+		relativeExp       time.Duration
+		expectedCacheSize int
+	}{
+		{
+			name:              "don't cleanup unexpired tokens",
+			relativeExp:       -1 * time.Hour,
+			expectedCacheSize: 0,
+		},
+		{
+			name:              "cleanup expired tokens",
+			relativeExp:       time.Hour,
+			expectedCacheSize: 1,
+		},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			clock := testingclock.NewFakeClock(time.Time{}.Add(24 * time.Hour))
+			mgr := NewManager(nil)
+			mgr.clock = clock
+
+			mgr.set("key", &authenticationv1.TokenRequest{
+				Status: authenticationv1.TokenRequestStatus{
+					ExpirationTimestamp: metav1.Time{Time: mgr.clock.Now().Add(c.relativeExp)},
+				},
+			})
+			mgr.cleanup()
+			if got, want := len(mgr.cache), c.expectedCacheSize; got != want {
+				t.Fatalf("unexpected number of cache entries after cleanup, got: %d, want: %d", got, want)
+			}
+		})
+	}
+}
+
+func TestKeyFunc(t *testing.T) {
+	type tokenRequestUnit struct {
+		name      string
+		namespace string
+		tr        *authenticationv1.TokenRequest
+	}
+	getKeyFunc := func(u tokenRequestUnit) string {
+		return keyFunc(u.name, u.namespace, u.tr)
+	}
+
+	cases := []struct {
+		name   string
+		trus   []tokenRequestUnit
+		target tokenRequestUnit
+
+		shouldHit bool
+	}{
+		{
+			name: "hit",
+			trus: []tokenRequestUnit{
+				{
+					name:      "foo-sa",
+					namespace: "foo-ns",
+					tr: &authenticationv1.TokenRequest{
+						Spec: authenticationv1.TokenRequestSpec{
+							Audiences:         []string{"foo1", "foo2"},
+							ExpirationSeconds: getInt64Point(2000),
+							BoundObjectRef: &authenticationv1.BoundObjectReference{
+								Kind: "pod",
+								Name: "foo-pod",
+								UID:  "foo-uid",
+							},
+						},
+					},
+				},
+				{
+					name:      "ame-sa",
+					namespace: "ame-ns",
+					tr: &authenticationv1.TokenRequest{
+						Spec: authenticationv1.TokenRequestSpec{
+							Audiences:         []string{"ame1", "ame2"},
+							ExpirationSeconds: getInt64Point(2000),
+							BoundObjectRef: &authenticationv1.BoundObjectReference{
+								Kind: "pod",
+								Name: "ame-pod",
+								UID:  "ame-uid",
+							},
+						},
+					},
+				},
+			},
+			target: tokenRequestUnit{
+				name:      "foo-sa",
+				namespace: "foo-ns",
+				tr: &authenticationv1.TokenRequest{
+					Spec: authenticationv1.TokenRequestSpec{
+						Audiences:         []string{"foo1", "foo2"},
+						ExpirationSeconds: getInt64Point(2000),
+						BoundObjectRef: &authenticationv1.BoundObjectReference{
+							Kind: "pod",
+							Name: "foo-pod",
+							UID:  "foo-uid",
+						},
+					},
+				},
+			},
+			shouldHit: true,
+		},
+		{
+			name: "not hit due to different ExpirationSeconds",
+			trus: []tokenRequestUnit{
+				{
+					name:      "foo-sa",
+					namespace: "foo-ns",
+					tr: &authenticationv1.TokenRequest{
+						Spec: authenticationv1.TokenRequestSpec{
+							Audiences:         []string{"foo1", "foo2"},
+							ExpirationSeconds: getInt64Point(2000),
+							BoundObjectRef: &authenticationv1.BoundObjectReference{
+								Kind: "pod",
+								Name: "foo-pod",
+								UID:  "foo-uid",
+							},
+						},
+					},
+				},
+			},
+			target: tokenRequestUnit{
+				name:      "foo-sa",
+				namespace: "foo-ns",
+				tr: &authenticationv1.TokenRequest{
+					Spec: authenticationv1.TokenRequestSpec{
+						Audiences: []string{"foo1", "foo2"},
+						//everthing is same besides ExpirationSeconds
+						ExpirationSeconds: getInt64Point(2001),
+						BoundObjectRef: &authenticationv1.BoundObjectReference{
+							Kind: "pod",
+							Name: "foo-pod",
+							UID:  "foo-uid",
+						},
+					},
+				},
+			},
+			shouldHit: false,
+		},
+		{
+			name: "not hit due to different BoundObjectRef",
+			trus: []tokenRequestUnit{
+				{
+					name:      "foo-sa",
+					namespace: "foo-ns",
+					tr: &authenticationv1.TokenRequest{
+						Spec: authenticationv1.TokenRequestSpec{
+							Audiences:         []string{"foo1", "foo2"},
+							ExpirationSeconds: getInt64Point(2000),
+							BoundObjectRef: &authenticationv1.BoundObjectReference{
+								Kind: "pod",
+								Name: "foo-pod",
+								UID:  "foo-uid",
+							},
+						},
+					},
+				},
+			},
+			target: tokenRequestUnit{
+				name:      "foo-sa",
+				namespace: "foo-ns",
+				tr: &authenticationv1.TokenRequest{
+					Spec: authenticationv1.TokenRequestSpec{
+						Audiences:         []string{"foo1", "foo2"},
+						ExpirationSeconds: getInt64Point(2000),
+						BoundObjectRef: &authenticationv1.BoundObjectReference{
+							Kind: "pod",
+							//everthing is same besides BoundObjectRef.Name
+							Name: "diff-pod",
+							UID:  "foo-uid",
+						},
+					},
+				},
+			},
+			shouldHit: false,
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			mgr := NewManager(nil)
+			mgr.clock = testingclock.NewFakeClock(time.Time{}.Add(30 * 24 * time.Hour))
+			for _, tru := range c.trus {
+				mgr.set(getKeyFunc(tru), &authenticationv1.TokenRequest{
+					Status: authenticationv1.TokenRequestStatus{
+						//make sure the token cache would not be cleaned by token manager clenaup func
+						ExpirationTimestamp: metav1.Time{Time: mgr.clock.Now().Add(50 * time.Minute)},
+					},
+				})
+			}
+			_, hit := mgr.get(getKeyFunc(c.target))
+
+			if hit != c.shouldHit {
+				t.Errorf("%s got unexpected hit result: expected to be %t, got %t", c.name, c.shouldHit, hit)
+			}
+		})
+	}
+
+}
+
+func getTokenRequest() *authenticationv1.TokenRequest {
+	return &authenticationv1.TokenRequest{
+		Spec: authenticationv1.TokenRequestSpec{
+			Audiences:         []string{"foo1", "foo2"},
+			ExpirationSeconds: getInt64Point(2000),
+			BoundObjectRef: &authenticationv1.BoundObjectReference{
+				Kind: "pod",
+				Name: "foo-pod",
+				UID:  "foo-uid",
+			},
+		},
+	}
+}
+
+func getInt64Point(v int64) *int64 {
+	return &v
+}

log/klogv2/klogv2_test.go

@@ -4,7 +4,7 @@
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
-//     http://www.apache.org/licenses/LICENSE-2.0
+//	http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,

node/api/exec.go

@@ -27,7 +27,6 @@ import (
 	"github.com/virtual-kubelet/virtual-kubelet/internal/kubernetes/remotecommand"
 	"k8s.io/apimachinery/pkg/types"
 	remoteutils "k8s.io/client-go/tools/remotecommand"
-	api "k8s.io/kubernetes/pkg/apis/core"
 )
 
 // ContainerExecHandlerFunc defines the handler function used for "execing" into a
@@ -136,11 +135,18 @@ func HandleContainerExec(h ContainerExecHandlerFunc, opts ...ContainerExecHandle
 	})
 }
 
+const (
+	execTTYParam    = "tty"
+	execStdinParam  = "input"
+	execStdoutParam = "output"
+	execStderrParam = "error"
+)
+
 func getExecOptions(req *http.Request) (*remotecommand.Options, error) {
-	tty := req.FormValue(api.ExecTTYParam) == "1"
-	stdin := req.FormValue(api.ExecStdinParam) == "1"
-	stdout := req.FormValue(api.ExecStdoutParam) == "1"
-	stderr := req.FormValue(api.ExecStderrParam) == "1"
+	tty := req.FormValue(execTTYParam) == "1"
+	stdin := req.FormValue(execStdinParam) == "1"
+	stdout := req.FormValue(execStdoutParam) == "1"
+	stderr := req.FormValue(execStderrParam) == "1"
 	if tty && stderr {
 		return nil, errors.New("cannot exec with tty and stderr")
 	}

node/api/helpers.go

@@ -33,7 +33,9 @@ func handleError(f handlerFunc) http.HandlerFunc {
 
 		code := httpStatusCode(err)
 		w.WriteHeader(code)
-		io.WriteString(w, err.Error()) //nolint:errcheck
+		if _, err := io.WriteString(w, err.Error()); err != nil {
+			log.G(req.Context()).WithError(err).Error("error writing error response")
+		}
 		logger := log.G(req.Context()).WithError(err).WithField("httpStatusCode", code)
 
 		if code >= 500 {

node/api/pods.go

@@ -24,14 +24,15 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/serializer"
 )
 
-type PodListerFunc func(context.Context) ([]*v1.Pod, error) // nolint:golint
+type PodListerFunc func(context.Context) ([]*v1.Pod, error) //nolint:golint
 
-func HandleRunningPods(getPods PodListerFunc) http.HandlerFunc { // nolint:golint
+func HandleRunningPods(getPods PodListerFunc) http.HandlerFunc { //nolint:golint
 	if getPods == nil {
 		return NotImplemented
 	}
 
 	scheme := runtime.NewScheme()
+	/* #nosec */
 	v1.SchemeBuilder.AddToScheme(scheme) //nolint:errcheck
 	codecs := serializer.NewCodecFactory(scheme)
 

node/api/server.go

@@ -33,7 +33,7 @@ type ServeMux interface {
 	Handle(path string, h http.Handler)
 }
 
-type PodHandlerConfig struct { // nolint:golint
+type PodHandlerConfig struct { //nolint:golint
 	RunInContainer   ContainerExecHandlerFunc
 	GetContainerLogs ContainerLogsHandlerFunc
 	// GetPods is meant to enumerate the pods that the provider knows about
@@ -79,7 +79,7 @@ func PodHandler(p PodHandlerConfig, debug bool) http.Handler {
 // PodStatsSummaryHandler creates an http handler for serving pod metrics.
 //
 // If the passed in handler func is nil this will create handlers which only
-//  serves http.StatusNotImplemented
+// serves http.StatusNotImplemented
 func PodStatsSummaryHandler(f PodStatsSummaryHandlerFunc) http.Handler {
 	if f == nil {
 		return http.HandlerFunc(NotImplemented)

node/api/stats.go

@@ -20,7 +20,7 @@ import (
 	"net/http"
 
 	"github.com/pkg/errors"
-	stats "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
+	stats "k8s.io/kubelet/pkg/apis/stats/v1alpha1"
 )
 
 // PodStatsSummaryHandlerFunc defines the handler for getting pod stats summaries

node/env_test.go

@@ -7,7 +7,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/bombsimon/logrusr"
+	"github.com/bombsimon/logrusr/v3"
 	"github.com/sirupsen/logrus"
 	"github.com/virtual-kubelet/virtual-kubelet/log"
 	logruslogger "github.com/virtual-kubelet/virtual-kubelet/log/logrus"
@@ -16,8 +16,9 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/client-go/kubernetes"
-	klogv2 "k8s.io/klog/v2"
+	"k8s.io/klog/v2"
 	"sigs.k8s.io/controller-runtime/pkg/envtest"
 )
 
@@ -25,6 +26,9 @@ func TestEnvtest(t *testing.T) {
 	if !*enableEnvTest || os.Getenv("VK_ENVTEST") != "" {
 		t.Skip("test only runs when -envtest is passed or if VK_ENVTEST is set to a non-empty value")
 	}
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
 	env := &envtest.Environment{}
 	_, err := env.Start()
 	assert.NilError(t, err)
@@ -33,15 +37,17 @@ func TestEnvtest(t *testing.T) {
 	}()
 
 	t.Log("Env test environment ready")
-	t.Run("E2ERunWithoutLeases", func(t *testing.T) {
+	t.Run("E2ERunWithoutLeases", wrapE2ETest(ctx, env, func(ctx context.Context, t *testing.T, environment *envtest.Environment) {
 		testNodeE2ERun(t, env, false)
-	})
-	t.Run("E2ERunWithLeases", func(t *testing.T) {
+	}))
+	t.Run("E2ERunWithLeases", wrapE2ETest(ctx, env, func(ctx context.Context, t *testing.T, environment *envtest.Environment) {
 		testNodeE2ERun(t, env, true)
-	})
+	}))
+
+	t.Run("E2EPodStatusUpdate", wrapE2ETest(ctx, env, testPodStatusUpdate))
 }
 
-func nodeNameForTest(t *testing.T) string {
+func kubernetesNameForTest(t *testing.T) string {
 	name := t.Name()
 	name = strings.ToLower(name)
 	name = strings.ReplaceAll(name, "/", "-")
@@ -49,17 +55,91 @@ func nodeNameForTest(t *testing.T) string {
 	return name
 }
 
+func wrapE2ETest(ctx context.Context, env *envtest.Environment, f func(context.Context, *testing.T, *envtest.Environment)) func(*testing.T) {
+	return func(t *testing.T) {
+		log.G(ctx)
+		sl := logrus.StandardLogger()
+		sl.SetLevel(logrus.DebugLevel)
+		logger := logruslogger.FromLogrus(sl.WithField("test", t.Name()))
+		ctx = log.WithLogger(ctx, logger)
+
+		// The following requires that E2E tests are performed *sequentially*
+		log.L = logger
+		klog.SetLogger(logrusr.New(sl))
+		f(ctx, t, env)
+	}
+}
+
+func testPodStatusUpdate(ctx context.Context, t *testing.T, env *envtest.Environment) {
+	provider := newMockProvider()
+
+	clientset, err := kubernetes.NewForConfig(env.Config)
+	assert.NilError(t, err)
+	pods := clientset.CoreV1().Pods(testNamespace)
+
+	assert.NilError(t, wireUpSystemWithClient(ctx, provider, clientset, func(ctx context.Context, s *system) {
+		p := newPod(forRealAPIServer, nameBasedOnTest(t))
+		// In real API server, we don't set the resource version
+		p.ResourceVersion = ""
+		newPod, err := pods.Create(ctx, p, metav1.CreateOptions{})
+		assert.NilError(t, err)
+
+		key, err := buildKey(newPod)
+		assert.NilError(t, err)
+
+		listOptions := metav1.ListOptions{
+			FieldSelector: fields.OneTermEqualSelector("metadata.name", p.ObjectMeta.Name).String(),
+		}
+
+		// Setup a watch to check if the pod is in running
+		watcher, err := s.client.CoreV1().Pods(testNamespace).Watch(ctx, listOptions)
+		assert.NilError(t, err)
+		defer watcher.Stop()
+		// Start the pod controller
+		assert.NilError(t, s.start(ctx))
+		var serverPod *corev1.Pod
+		for {
+			select {
+			case <-ctx.Done():
+				t.Fatalf("Context ended early: %s", ctx.Err().Error())
+			case ev := <-watcher.ResultChan():
+				serverPod = ev.Object.(*corev1.Pod)
+				if serverPod.Status.Phase == corev1.PodRunning {
+					goto running
+				}
+			}
+		}
+	running:
+		t.Log("Observed pod in running state")
+
+		providerPod, ok := provider.pods.Load(key)
+		assert.Assert(t, ok)
+		providerPodCopy := providerPod.(*corev1.Pod).DeepCopy()
+		providerPodCopy.Status = serverPod.Status
+		if providerPodCopy.Annotations == nil {
+			providerPodCopy.Annotations = make(map[string]string, 1)
+		}
+		providerPodCopy.Annotations["testannotation"] = "testvalue"
+		provider.notifier(providerPodCopy)
+
+		for {
+			select {
+			case <-ctx.Done():
+				t.Fatalf("Context ended early: %s", ctx.Err().Error())
+			case ev := <-watcher.ResultChan():
+				annotations := ev.Object.(*corev1.Pod).Annotations
+				if annotations != nil && annotations["testannotation"] == "testvalue" {
+					return
+				}
+			}
+		}
+	}))
+}
+
 func testNodeE2ERun(t *testing.T, env *envtest.Environment, withLeases bool) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	sl := logrus.StandardLogger()
-	sl.SetLevel(logrus.DebugLevel)
-	logger := logruslogger.FromLogrus(sl.WithField("test", t.Name()))
-	ctx = log.WithLogger(ctx, logger)
-	log.L = logger
-	klogv2.SetLogger(logrusr.NewLogger(sl))
-
 	clientset, err := kubernetes.NewForConfig(env.Config)
 	assert.NilError(t, err)
 	nodes := clientset.CoreV1().Nodes()
@@ -70,7 +150,7 @@ func testNodeE2ERun(t *testing.T, env *envtest.Environment, withLeases bool) {
 
 	testNode := &corev1.Node{
 		ObjectMeta: metav1.ObjectMeta{
-			Name: nodeNameForTest(t),
+			Name: kubernetesNameForTest(t),
 		},
 	}
 

node/lease_controller_v1.go

@@ -114,6 +114,7 @@ func (c *leaseController) sync(ctx context.Context) {
 	pingResult, err := c.nodeController.nodePingController.getResult(ctx)
 	if err != nil {
 		log.G(ctx).WithError(err).Error("Could not get ping status")
+		return
 	}
 	if pingResult.error != nil {
 		log.G(ctx).WithError(pingResult.error).Error("Ping result is not clean, not updating lease")
@@ -332,7 +333,7 @@ func (e *nodeNotReadyError) Is(target error) bool {
 	return ok
 }
 
-func (e *nodeNotReadyError) As(target error) bool {
+func (e *nodeNotReadyError) As(target interface{}) bool {
 	val, ok := target.(*nodeNotReadyError)
 	if ok {
 		*val = *e

node/lifecycle_test.go

@@ -21,13 +21,14 @@ import (
 	"k8s.io/apimachinery/pkg/util/uuid"
 	"k8s.io/apimachinery/pkg/watch"
 	kubeinformers "k8s.io/client-go/informers"
+	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/kubernetes/fake"
 	corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
 	ktesting "k8s.io/client-go/testing"
 	"k8s.io/client-go/tools/cache"
 	"k8s.io/client-go/tools/record"
 	watchutils "k8s.io/client-go/tools/watch"
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 )
 
 var (
@@ -37,7 +38,7 @@ var (
 const (
 	// There might be a constant we can already leverage here
 	testNamespace        = "default"
-	informerResyncPeriod = time.Duration(1 * time.Second)
+	informerResyncPeriod = 1 * time.Second
 	testNodeName         = "testnode"
 	podSyncWorkers       = 3
 )
@@ -226,12 +227,12 @@ func TestPodLifecycle(t *testing.T) {
 type testFunction func(ctx context.Context, s *system)
 type system struct {
 	pc                  *PodController
-	client              *fake.Clientset
+	client              kubernetes.Interface
 	podControllerConfig PodControllerConfig
 }
 
 func (s *system) start(ctx context.Context) error {
-	go s.pc.Run(ctx, podSyncWorkers) // nolint:errcheck
+	go s.pc.Run(ctx, podSyncWorkers) //nolint:errcheck
 	select {
 	case <-s.pc.Ready():
 	case <-s.pc.Done():
@@ -262,6 +263,13 @@ func wireUpSystem(ctx context.Context, provider PodLifecycleHandler, f testFunct
 		return false, nil, nil
 	})
 
+	return wireUpSystemWithClient(ctx, provider, client, f)
+}
+
+func wireUpSystemWithClient(ctx context.Context, provider PodLifecycleHandler, client kubernetes.Interface, f testFunction) error {
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
 	// This is largely copy and pasted code from the root command
 	sharedInformerFactory := kubeinformers.NewSharedInformerFactoryWithOptions(
 		client,
@@ -359,6 +367,14 @@ func testDanglingPodScenario(ctx context.Context, t *testing.T, s *system, m tes
 
 }
 
+func sendErr(ctx context.Context, ch chan error, err error) {
+	select {
+	case <-ctx.Done():
+		log.G(ctx).WithError(err).Warn("timeout waiting to send test error")
+	case ch <- err:
+	}
+}
+
 func testDanglingPodScenarioWithDeletionTimestamp(ctx context.Context, t *testing.T, s *system, m testingProvider) {
 	t.Parallel()
 
@@ -382,18 +398,18 @@ func testDanglingPodScenarioWithDeletionTimestamp(ctx context.Context, t *testin
 	_, e := s.client.CoreV1().Pods(testNamespace).Create(ctx, podCopyWithDeletionTimestamp, metav1.CreateOptions{})
 	assert.NilError(t, e)
 
-	// Start the pod controller
-	assert.NilError(t, s.start(ctx))
 	watchErrCh := make(chan error)
-
 	go func() {
 		_, watchErr := watchutils.UntilWithoutRetry(ctx, watcher,
 			func(ev watch.Event) (bool, error) {
 				return ev.Type == watch.Deleted, nil
 			})
-		watchErrCh <- watchErr
+		sendErr(ctx, watchErrCh, watchErr)
 	}()
 
+	// Start the pod controller
+	assert.NilError(t, s.start(ctx))
+
 	select {
 	case <-ctx.Done():
 		t.Fatalf("Context ended early: %s", ctx.Err().Error())
@@ -428,7 +444,7 @@ func testCreateStartDeleteScenario(ctx context.Context, t *testing.T, s *system,
 				return pod.Name == p.ObjectMeta.Name, nil
 			})
 
-		watchErrCh <- watchErr
+		sendErr(ctx, watchErrCh, watchErr)
 	}()
 
 	// Create the Pod
@@ -457,7 +473,7 @@ func testCreateStartDeleteScenario(ctx context.Context, t *testing.T, s *system,
 				return pod.Status.Phase == corev1.PodRunning, nil
 			})
 
-		watchErrCh <- watchErr
+		sendErr(ctx, watchErrCh, watchErr)
 	}()
 
 	assert.NilError(t, s.start(ctx))
@@ -479,7 +495,7 @@ func testCreateStartDeleteScenario(ctx context.Context, t *testing.T, s *system,
 		_, watchDeleteErr := watchutils.UntilWithoutRetry(ctx, watcher2, func(ev watch.Event) (bool, error) {
 			return ev.Type == watch.Deleted, nil
 		})
-		waitDeleteCh <- watchDeleteErr
+		sendErr(ctx, waitDeleteCh, watchDeleteErr)
 	}()
 
 	// Setup a watch prior to pod deletion
@@ -487,7 +503,7 @@ func testCreateStartDeleteScenario(ctx context.Context, t *testing.T, s *system,
 	assert.NilError(t, err)
 	defer watcher.Stop()
 	go func() {
-		watchErrCh <- waitFunction(ctx, watcher)
+		sendErr(ctx, watchErrCh, waitFunction(ctx, watcher))
 	}()
 
 	// Delete the pod via deletiontimestamp
@@ -551,7 +567,7 @@ func testUpdatePodWhileRunningScenario(ctx context.Context, t *testing.T, s *sys
 			})
 		// This deepcopy is required to please the race detector
 		p = newPod.Object.(*corev1.Pod).DeepCopy()
-		watchErrCh <- watchErr
+		sendErr(ctx, watchErrCh, watchErr)
 	}()
 
 	// Start the pod controller
@@ -620,6 +636,17 @@ func randomizeName(pod *corev1.Pod) {
 	pod.Name = name
 }
 
+func forRealAPIServer(pod *corev1.Pod) {
+	pod.ResourceVersion = ""
+	pod.ObjectMeta.UID = ""
+}
+
+func nameBasedOnTest(t *testing.T) podModifier {
+	return func(pod *corev1.Pod) {
+		pod.Name = kubernetesNameForTest(t)
+	}
+}
+
 func newPod(podmodifiers ...podModifier) *corev1.Pod {
 	var terminationGracePeriodSeconds int64 = 30
 	pod := &corev1.Pod{

node/main_test.go

@@ -6,14 +6,14 @@ import (
 	"os"
 	"testing"
 
-	klogv1 "k8s.io/klog"
+	"k8s.io/klog/v2"
 )
 
 var enableEnvTest = flag.Bool("envtest", false, "Enable envtest based tests")
 
 func TestMain(m *testing.M) {
 	flagset := flag.NewFlagSet("klog", flag.PanicOnError)
-	klogv1.InitFlags(flagset)
+	klog.InitFlags(flagset)
 	flagset.VisitAll(func(f *flag.Flag) {
 		flag.Var(f.Value, "klog."+f.Name, f.Usage)
 	})

node/node.go

@@ -54,7 +54,7 @@ var (
 //
 // Note: Implementers can choose to manage a node themselves, in which case
 // it is not needed to provide an implementation for this interface.
-type NodeProvider interface { // nolint:golint
+type NodeProvider interface { //nolint:revive
 	// Ping checks if the node is still active.
 	// This is intended to be lightweight as it will be called periodically as a
 	// heartbeat to keep the node marked as ready in Kubernetes.
@@ -84,6 +84,7 @@ func NewNodeController(p NodeProvider, node *corev1.Node, nodes v1.NodeInterface
 		serverNode: node,
 		nodes:      nodes,
 		chReady:    make(chan struct{}),
+		chDone:     make(chan struct{}),
 	}
 	for _, o := range opts {
 		if err := o(n); err != nil {
@@ -104,7 +105,7 @@ func NewNodeController(p NodeProvider, node *corev1.Node, nodes v1.NodeInterface
 }
 
 // NodeControllerOpt are the functional options used for configuring a node
-type NodeControllerOpt func(*NodeController) error // nolint:golint
+type NodeControllerOpt func(*NodeController) error //nolint:revive
 
 // WithNodeEnableLeaseV1 enables support for v1 leases.
 // V1 Leases share all the same properties as v1beta1 leases, except they do not fallback like
@@ -207,7 +208,7 @@ type ErrorHandler func(context.Context, error) error
 // NodeController deals with creating and managing a node object in Kubernetes.
 // It can register a node with Kubernetes and periodically update its status.
 // NodeController manages a single node entity.
-type NodeController struct { // nolint:golint
+type NodeController struct { //nolint:revive
 	p NodeProvider
 
 	// serverNode must be updated each time it is updated in API Server
@@ -223,7 +224,12 @@ type NodeController struct { // nolint:golint
 
 	nodeStatusUpdateErrorHandler ErrorHandler
 
+	// chReady is closed once the controller is ready to start the control loop
 	chReady chan struct{}
+	// chDone is closed once the control loop has exited
+	chDone chan struct{}
+	errMu  sync.Mutex
+	err    error
 
 	nodePingController *nodePingController
 	pingTimeout        *time.Duration
@@ -249,7 +255,14 @@ const (
 // node status update (because some things still expect the node to be updated
 // periodically), otherwise it will only use node status update with the configured
 // ping interval.
-func (n *NodeController) Run(ctx context.Context) error {
+func (n *NodeController) Run(ctx context.Context) (retErr error) {
+	defer func() {
+		n.errMu.Lock()
+		n.err = retErr
+		n.errMu.Unlock()
+		close(n.chDone)
+	}()
+
 	n.chStatusUpdate = make(chan *corev1.Node, 1)
 	n.p.NotifyNodeStatus(ctx, func(node *corev1.Node) {
 		n.chStatusUpdate <- node
@@ -273,6 +286,22 @@ func (n *NodeController) Run(ctx context.Context) error {
 	return n.controlLoop(ctx, providerNode)
 }
 
+// Done signals to the caller when the controller is done and the control loop is exited.
+//
+// Call n.Err() to find out if there was an error.
+func (n *NodeController) Done() <-chan struct{} {
+	return n.chDone
+}
+
+// Err returns any errors that have occurred that trigger the control loop to exit.
+//
+// Err only returns a non-nil error after `<-n.Done()` returns.
+func (n *NodeController) Err() error {
+	n.errMu.Lock()
+	defer n.errMu.Unlock()
+	return n.err
+}
+
 func (n *NodeController) ensureNode(ctx context.Context, providerNode *corev1.Node) (err error) {
 	ctx, span := trace.StartSpan(ctx, "node.ensureNode")
 	defer span.End()
@@ -307,14 +336,12 @@ func (n *NodeController) ensureNode(ctx context.Context, providerNode *corev1.No
 
 // Ready returns a channel that gets closed when the node is fully up and
 // running. Note that if there is an error on startup this channel will never
-// be started.
+// be closed.
 func (n *NodeController) Ready() <-chan struct{} {
 	return n.chReady
 }
 
 func (n *NodeController) controlLoop(ctx context.Context, providerNode *corev1.Node) error {
-	close(n.chReady)
-
 	defer n.group.Wait()
 
 	var sleepInterval time.Duration
@@ -355,6 +382,7 @@ func (n *NodeController) controlLoop(ctx context.Context, providerNode *corev1.N
 		return false
 	}
 
+	close(n.chReady)
 	for {
 		shouldTerminate := loop()
 		if shouldTerminate {
@@ -657,10 +685,9 @@ func (t taintsStringer) String() string {
 
 func addNodeAttributes(ctx context.Context, span trace.Span, n *corev1.Node) context.Context {
 	return span.WithFields(ctx, log.Fields{
-		"node.UID":     string(n.UID),
-		"node.name":    n.Name,
-		"node.cluster": n.ClusterName,
-		"node.taints":  taintsStringer(n.Spec.Taints),
+		"node.UID":    string(n.UID),
+		"node.name":   n.Name,
+		"node.taints": taintsStringer(n.Spec.Taints),
 	})
 }
 

node/node_test.go

@@ -65,16 +65,13 @@ func testNodeRun(t *testing.T, enableLease bool) {
 	node, err := NewNodeController(testP, testNode, nodes, opts...)
 	assert.NilError(t, err)
 
-	chErr := make(chan error)
 	defer func() {
 		cancel()
-		assert.NilError(t, <-chErr)
+		<-node.Done()
+		assert.NilError(t, node.Err())
 	}()
 
-	go func() {
-		chErr <- node.Run(ctx)
-		close(chErr)
-	}()
+	go node.Run(ctx) //nolint:errcheck
 
 	nw := makeWatch(ctx, t, nodes, testNodeCopy.Name)
 	defer nw.Stop()
@@ -103,8 +100,8 @@ func testNodeRun(t *testing.T, enableLease bool) {
 		case <-time.After(time.Second):
 			t.Errorf("timeout waiting for event")
 			continue
-		case err := <-chErr:
-			t.Fatal(err) // if this returns at all it is an error regardless if err is nil
+		case <-node.Done():
+			t.Fatal(node.Err()) // if this returns at all it is an error regardless if err is nil
 		case <-nr:
 			nodeUpdates++
 			continue
@@ -152,8 +149,8 @@ func testNodeRun(t *testing.T, enableLease bool) {
 	defer eCancel()
 
 	select {
-	case err := <-chErr:
-		t.Fatal(err) // if this returns at all it is an error regardless if err is nil
+	case <-node.Done():
+		t.Fatal(node.Err()) // if this returns at all it is an error regardless if err is nil
 	case err := <-waitForEvent(eCtx, nr, func(e watch.Event) bool {
 		node := e.Object.(*corev1.Node)
 		if len(node.Status.Conditions) == 0 {
@@ -192,10 +189,7 @@ func TestNodeCustomUpdateStatusErrorHandler(t *testing.T) {
 	)
 	assert.NilError(t, err)
 
-	chErr := make(chan error, 1)
-	go func() {
-		chErr <- node.Run(ctx)
-	}()
+	go node.Run(ctx) //nolint:errcheck
 
 	timer := time.NewTimer(10 * time.Second)
 	defer timer.Stop()
@@ -204,8 +198,8 @@ func TestNodeCustomUpdateStatusErrorHandler(t *testing.T) {
 	select {
 	case <-timer.C:
 		t.Fatal("timeout waiting for node to be ready")
-	case <-chErr:
-		t.Fatalf("node.Run returned earlier than expected: %v", err)
+	case <-node.Done():
+		t.Fatalf("node.Run returned earlier than expected: %v", node.Err())
 	case <-node.Ready():
 	}
 
@@ -218,8 +212,8 @@ func TestNodeCustomUpdateStatusErrorHandler(t *testing.T) {
 	defer timer.Stop()
 
 	select {
-	case err := <-chErr:
-		assert.Equal(t, err, nil)
+	case <-node.Done():
+		assert.NilError(t, node.Err())
 	case <-timer.C:
 		t.Fatal("timeout waiting for node shutdown")
 	}
@@ -301,9 +295,11 @@ func TestPingAfterStatusUpdate(t *testing.T) {
 	node, err := NewNodeController(testP, testNode, nodes, opts...)
 	assert.NilError(t, err)
 
-	chErr := make(chan error, 1)
-	go func() {
-		chErr <- node.Run(ctx)
+	go node.Run(ctx) //nolint:errcheck
+	defer func() {
+		cancel()
+		<-node.Done()
+		assert.NilError(t, node.Err())
 	}()
 
 	timer := time.NewTimer(10 * time.Second)
@@ -313,10 +309,11 @@ func TestPingAfterStatusUpdate(t *testing.T) {
 	select {
 	case <-timer.C:
 		t.Fatal("timeout waiting for node to be ready")
-	case <-chErr:
-		t.Fatalf("node.Run returned earlier than expected: %v", err)
+	case <-node.Done():
+		t.Fatalf("node.Run returned earlier than expected: %v", node.Err())
 	case <-node.Ready():
 	}
+	timer.Stop()
 
 	notifyTimer := time.After(interval * time.Duration(10))
 	<-notifyTimer
@@ -360,16 +357,13 @@ func TestBeforeAnnotationsPreserved(t *testing.T) {
 	node, err := NewNodeController(testP, testNode, nodes, opts...)
 	assert.NilError(t, err)
 
-	chErr := make(chan error)
 	defer func() {
 		cancel()
-		assert.NilError(t, <-chErr)
+		<-node.Done()
+		assert.NilError(t, node.Err())
 	}()
 
-	go func() {
-		chErr <- node.Run(ctx)
-		close(chErr)
-	}()
+	go node.Run(ctx) //nolint:errcheck
 
 	nw := makeWatch(ctx, t, nodes, testNodeCopy.Name)
 	defer nw.Stop()
@@ -427,16 +421,13 @@ func TestManualConditionsPreserved(t *testing.T) {
 	node, err := NewNodeController(testP, testNode, nodes, opts...)
 	assert.NilError(t, err)
 
-	chErr := make(chan error)
 	defer func() {
 		cancel()
-		assert.NilError(t, <-chErr)
+		<-node.Done()
+		assert.NilError(t, node.Err())
 	}()
 
-	go func() {
-		chErr <- node.Run(ctx)
-		close(chErr)
-	}()
+	go node.Run(ctx) //nolint:errcheck
 
 	nw := makeWatch(ctx, t, nodes, testNodeCopy.Name)
 	defer nw.Stop()

node/nodeutil/auth.go

@@ -0,0 +1,220 @@
+package nodeutil
+
+import (
+	"context"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/virtual-kubelet/virtual-kubelet/log"
+	"github.com/virtual-kubelet/virtual-kubelet/trace"
+	"k8s.io/apiserver/pkg/authentication/authenticator"
+	"k8s.io/apiserver/pkg/authentication/authenticatorfactory"
+	"k8s.io/apiserver/pkg/authentication/request/anonymous"
+	"k8s.io/apiserver/pkg/authentication/user"
+	"k8s.io/apiserver/pkg/authorization/authorizer"
+	"k8s.io/apiserver/pkg/authorization/authorizerfactory"
+	"k8s.io/client-go/kubernetes"
+)
+
+// Auth is the interface used to implement authn/authz for http requests
+type Auth interface {
+	authenticator.Request
+	authorizer.RequestAttributesGetter
+	authorizer.Authorizer
+}
+
+type authWrapper struct {
+	authenticator.Request
+	authorizer.RequestAttributesGetter
+	authorizer.Authorizer
+}
+
+// InstrumentAuth wraps the provided Auth in a new instrumented Auth
+//
+// Note: You would only need this if you rolled your own auth.
+// The Auth implementations defined in this package are already instrumented.
+func InstrumentAuth(auth Auth) Auth {
+	if _, ok := auth.(*authWrapper); ok {
+		// This is already instrumented
+		return auth
+	}
+	return &authWrapper{
+		Request:                 auth,
+		RequestAttributesGetter: auth,
+		Authorizer:              auth,
+	}
+}
+
+// NoAuth creates an Auth which allows anonymous access to all resouorces
+func NoAuth() Auth {
+	return &authWrapper{
+		Request:                 anonymous.NewAuthenticator(),
+		RequestAttributesGetter: &NodeRequestAttr{},
+		Authorizer:              authorizerfactory.NewAlwaysAllowAuthorizer(),
+	}
+}
+
+// WithAuth makes a new http handler which wraps the provided handler with authn/authz.
+func WithAuth(auth Auth, h http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		handleAuth(auth, w, r, h)
+	})
+}
+
+func handleAuth(auth Auth, w http.ResponseWriter, r *http.Request, next http.Handler) {
+	ctx := r.Context()
+	ctx, span := trace.StartSpan(ctx, "vk.handleAuth")
+	defer span.End()
+	r = r.WithContext(ctx)
+
+	info, ok, err := auth.AuthenticateRequest(r)
+	if err != nil || !ok {
+		log.G(r.Context()).WithError(err).Error("Authorization error")
+		http.Error(w, "Unauthorized", http.StatusUnauthorized)
+		return
+	}
+
+	logger := log.G(ctx).WithFields(log.Fields{
+		"user-name": info.User.GetName(),
+		"user-id":   info.User.GetUID(),
+	})
+
+	ctx = log.WithLogger(ctx, logger)
+	r = r.WithContext(ctx)
+
+	attrs := auth.GetRequestAttributes(info.User, r)
+
+	decision, _, err := auth.Authorize(ctx, attrs)
+	if err != nil {
+		log.G(r.Context()).WithError(err).Error("Authorization error")
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	if decision != authorizer.DecisionAllow {
+		http.Error(w, "Forbidden", http.StatusForbidden)
+		return
+	}
+
+	next.ServeHTTP(w, r)
+}
+
+// WebhookAuthOption is used as a functional argument to configure webhook auth.
+type WebhookAuthOption func(*WebhookAuthConfig) error
+
+// WebhookAuthConfig stores the configurations for authn/authz and is used by WebhookAuthOption to expose to callers.
+type WebhookAuthConfig struct {
+	AuthnConfig authenticatorfactory.DelegatingAuthenticatorConfig
+	AuthzConfig authorizerfactory.DelegatingAuthorizerConfig
+}
+
+// WebhookAuth creates an Auth suitable to use with kubelet webhook auth.
+// You must provide a CA provider to the authentication config, otherwise mTLS is disabled.
+func WebhookAuth(client kubernetes.Interface, nodeName string, opts ...WebhookAuthOption) (Auth, error) {
+	cfg := WebhookAuthConfig{
+		AuthnConfig: authenticatorfactory.DelegatingAuthenticatorConfig{
+			CacheTTL: 2 * time.Minute, // default taken from k8s.io/kubernetes/pkg/kubelet/apis/config/v1beta1
+			// TODO: After upgrading k8s libs, we need to add the retry backoff option
+		},
+		AuthzConfig: authorizerfactory.DelegatingAuthorizerConfig{
+			AllowCacheTTL: 5 * time.Minute,  // default taken from k8s.io/kubernetes/pkg/kubelet/apis/config/v1beta1
+			DenyCacheTTL:  30 * time.Second, // default taken from k8s.io/kubernetes/pkg/kubelet/apis/config/v1beta1
+			// TODO: After upgrading k8s libs, we need to add the retry backoff option
+		},
+	}
+
+	for _, o := range opts {
+		if err := o(&cfg); err != nil {
+			return nil, err
+		}
+	}
+
+	cfg.AuthnConfig.TokenAccessReviewClient = client.AuthenticationV1()
+	cfg.AuthzConfig.SubjectAccessReviewClient = client.AuthorizationV1()
+
+	authn, _, err := cfg.AuthnConfig.New()
+	if err != nil {
+		return nil, err
+	}
+
+	authz, err := cfg.AuthzConfig.New()
+	if err != nil {
+		return nil, err
+	}
+	return &authWrapper{
+		Request:                 authn,
+		RequestAttributesGetter: NodeRequestAttr{nodeName},
+		Authorizer:              authz,
+	}, nil
+}
+
+func (w *authWrapper) AuthenticateRequest(r *http.Request) (*authenticator.Response, bool, error) {
+	ctx, span := trace.StartSpan(r.Context(), "AuthenticateRequest")
+	defer span.End()
+	return w.Request.AuthenticateRequest(r.WithContext(ctx))
+}
+
+func (w *authWrapper) Authorize(ctx context.Context, a authorizer.Attributes) (authorizer.Decision, string, error) {
+	ctx, span := trace.StartSpan(ctx, "Authorize")
+	defer span.End()
+	return w.Authorizer.Authorize(ctx, a)
+}
+
+// NodeRequestAttr is a authorizor.RequeestAttributesGetter which can be used in the Auth interface.
+type NodeRequestAttr struct {
+	NodeName string
+}
+
+// GetRequestAttributes satisfies the authorizer.RequestAttributesGetter interface for use with an `Auth`.
+func (a NodeRequestAttr) GetRequestAttributes(u user.Info, r *http.Request) authorizer.Attributes {
+	return authorizer.AttributesRecord{
+		User:            u,
+		Verb:            getAPIVerb(r),
+		Namespace:       "",
+		APIGroup:        "",
+		APIVersion:      "v1",
+		Resource:        "nodes",
+		Name:            a.NodeName,
+		ResourceRequest: true,
+		Path:            r.URL.Path,
+		Subresource:     getSubresource(r),
+	}
+}
+
+func getAPIVerb(r *http.Request) string {
+	switch r.Method {
+	case http.MethodPost:
+		return "create"
+	case http.MethodGet:
+		return "get"
+	case http.MethodPut:
+		return "update"
+	case http.MethodPatch:
+		return "patch"
+	case http.MethodDelete:
+		return "delete"
+	}
+	return ""
+}
+
+func isSubpath(subpath, path string) bool {
+	// Taken from k8s.io/kubernetes/pkg/kubelet/server/auth.go
+	return subpath == path || (strings.HasPrefix(subpath, path) && subpath[len(path)] == '/')
+}
+
+func getSubresource(r *http.Request) string {
+	if isSubpath(r.URL.Path, "/stats") {
+		return "stats"
+	}
+	if isSubpath(r.URL.Path, "/metrics") {
+		return "metrics"
+	}
+	if isSubpath(r.URL.Path, "/logs") {
+		// yes, "log", not "logs"
+		// per kubelet code: "log" to match other log subresources (pods/log, etc)
+		return "log"
+	}
+
+	return "proxy"
+}

node/nodeutil/controller.go

@@ -0,0 +1,438 @@
+package nodeutil
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"net"
+	"net/http"
+	"os"
+	"path"
+	"runtime"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/virtual-kubelet/virtual-kubelet/log"
+	"github.com/virtual-kubelet/virtual-kubelet/node"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/informers"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/kubernetes/scheme"
+	corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
+	corev1listers "k8s.io/client-go/listers/core/v1"
+	"k8s.io/client-go/tools/record"
+)
+
+// Node helps manage the startup/shutdown procedure for other controllers.
+// It is intended as a convenience to reduce boiler plate code for starting up controllers.
+//
+// Must be created with constructor `NewNode`.
+type Node struct {
+	nc *node.NodeController
+	pc *node.PodController
+
+	readyCb func(context.Context) error
+
+	ready chan struct{}
+	done  chan struct{}
+	err   error
+
+	podInformerFactory informers.SharedInformerFactory
+	scmInformerFactory informers.SharedInformerFactory
+	client             kubernetes.Interface
+
+	listenAddr string
+	h          http.Handler
+	tlsConfig  *tls.Config
+
+	workers int
+
+	eb record.EventBroadcaster
+}
+
+// NodeController returns the configured node controller.
+func (n *Node) NodeController() *node.NodeController {
+	return n.nc
+}
+
+// PodController returns the configured pod controller.
+func (n *Node) PodController() *node.PodController {
+	return n.pc
+}
+
+func (n *Node) runHTTP(ctx context.Context) (func(), error) {
+	if n.tlsConfig == nil {
+		log.G(ctx).Warn("TLS config not provided, not starting up http service")
+		return func() {}, nil
+	}
+	if n.h == nil {
+		log.G(ctx).Debug("No http handler, not starting up http service")
+		return func() {}, nil
+	}
+
+	l, err := tls.Listen("tcp", n.listenAddr, n.tlsConfig)
+	if err != nil {
+		return nil, errors.Wrap(err, "error starting http listener")
+	}
+
+	log.G(ctx).Debug("Started TLS listener")
+
+	srv := &http.Server{Handler: n.h, TLSConfig: n.tlsConfig, ReadHeaderTimeout: 30 * time.Second}
+	go srv.Serve(l) //nolint:errcheck
+	log.G(ctx).Debug("HTTP server running")
+
+	return func() {
+		/* #nosec */
+		srv.Close()
+		/* #nosec */
+		l.Close()
+	}, nil
+}
+
+// Run starts all the underlying controllers
+func (n *Node) Run(ctx context.Context) (retErr error) {
+	ctx, cancel := context.WithCancel(ctx)
+	defer func() {
+		cancel()
+
+		n.err = retErr
+		close(n.done)
+	}()
+
+	if n.eb != nil {
+		n.eb.StartLogging(log.G(ctx).Infof)
+		n.eb.StartRecordingToSink(&corev1client.EventSinkImpl{Interface: n.client.CoreV1().Events(v1.NamespaceAll)})
+		defer n.eb.Shutdown()
+		log.G(ctx).Debug("Started event broadcaster")
+	}
+
+	cancelHTTP, err := n.runHTTP(ctx)
+	if err != nil {
+		return err
+	}
+	defer cancelHTTP()
+
+	go n.podInformerFactory.Start(ctx.Done())
+	go n.scmInformerFactory.Start(ctx.Done())
+	go n.pc.Run(ctx, n.workers) //nolint:errcheck
+
+	defer func() {
+		cancel()
+		<-n.pc.Done()
+	}()
+
+	select {
+	case <-ctx.Done():
+		return n.err
+	case <-n.pc.Ready():
+	case <-n.pc.Done():
+		return n.pc.Err()
+	}
+
+	log.G(ctx).Debug("pod controller ready")
+
+	go n.nc.Run(ctx) //nolint:errcheck
+
+	defer func() {
+		cancel()
+		<-n.nc.Done()
+	}()
+
+	select {
+	case <-ctx.Done():
+		n.err = ctx.Err()
+		return n.err
+	case <-n.nc.Ready():
+	case <-n.nc.Done():
+		return n.nc.Err()
+	}
+
+	log.G(ctx).Debug("node controller ready")
+
+	if n.readyCb != nil {
+		if err := n.readyCb(ctx); err != nil {
+			return err
+		}
+	}
+	close(n.ready)
+
+	select {
+	case <-n.nc.Done():
+		cancel()
+		return n.nc.Err()
+	case <-n.pc.Done():
+		cancel()
+		return n.pc.Err()
+	}
+}
+
+// WaitReady waits for the specified timeout for the controller to be ready.
+//
+// The timeout is for convenience so the caller doesn't have to juggle an extra context.
+func (n *Node) WaitReady(ctx context.Context, timeout time.Duration) error {
+	if timeout > 0 {
+		var cancel func()
+		ctx, cancel = context.WithTimeout(ctx, timeout)
+		defer cancel()
+	}
+
+	select {
+	case <-n.ready:
+		return nil
+	case <-n.done:
+		return fmt.Errorf("controller exited before ready: %w", n.err)
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+}
+
+// Ready returns a channel that will be closed after the controller is ready.
+func (n *Node) Ready() <-chan struct{} {
+	return n.ready
+}
+
+// Done returns a channel that will be closed when the controller has exited.
+func (n *Node) Done() <-chan struct{} {
+	return n.done
+}
+
+// Err returns any error that occurred with the controller.
+//
+// This always return nil before `<-Done()`.
+func (n *Node) Err() error {
+	select {
+	case <-n.Done():
+		return n.err
+	default:
+		return nil
+	}
+}
+
+// NodeOpt is used as functional options when configuring a new node in NewNodeFromClient
+type NodeOpt func(c *NodeConfig) error
+
+// NodeConfig is used to hold configuration items for a Node.
+// It gets used in conjection with NodeOpt in NewNodeFromClient
+type NodeConfig struct {
+	// Set the client to use, otherwise a client will be created from ClientsetFromEnv
+	Client kubernetes.Interface
+
+	// Set the node spec to register with Kubernetes
+	NodeSpec v1.Node
+	// Set the path to read a kubeconfig from for creating a client.
+	// This is ignored when a client is provided to NewNodeFromClient
+	KubeconfigPath string
+	// Set the period for a full resync for generated client-go informers
+	InformerResyncPeriod time.Duration
+
+	// Set the address to listen on for the http API
+	HTTPListenAddr string
+	// Set a custom API handler to use.
+	// You can use this to setup, for example, authentication middleware.
+	// If one is not provided a default one will be created.
+	//
+	// Note: If you provide your own handler, you'll need to handle all auth, routes, etc.
+	Handler http.Handler
+	// Set the timeout for idle http streams
+	StreamIdleTimeout time.Duration
+	// Set the timeout for creating http streams
+	StreamCreationTimeout time.Duration
+	// Enable http debugging routes
+	DebugHTTP bool
+	// Set the tls config to use for the http server
+	TLSConfig *tls.Config
+
+	// Specify the event recorder to use
+	// If this is not provided, a default one will be used.
+	EventRecorder record.EventRecorder
+
+	// Set the number of workers to reconcile pods
+	// The default value is derived from the number of cores available.
+	NumWorkers int
+
+	routeAttacher func(Provider, NodeConfig, corev1listers.PodLister)
+}
+
+// WithNodeConfig returns a NodeOpt which replaces the NodeConfig with the passed in value.
+func WithNodeConfig(c NodeConfig) NodeOpt {
+	return func(orig *NodeConfig) error {
+		*orig = c
+		return nil
+	}
+}
+
+// WithClient return a NodeOpt that sets the client that will be used to create/manage the node.
+func WithClient(c kubernetes.Interface) NodeOpt {
+	return func(cfg *NodeConfig) error {
+		cfg.Client = c
+		return nil
+	}
+}
+
+// NewNode creates a new node using the provided client and name.
+// This is intended for high-level/low boiler-plate usage.
+// Use the constructors in the `node` package for lower level configuration.
+//
+// Some basic values are set for node status, you'll almost certainly want to modify it.
+//
+// If client is nil, this will construct a client using ClientsetFromEnv
+// It is up to the caller to configure auth on the HTTP handler.
+func NewNode(name string, newProvider NewProviderFunc, opts ...NodeOpt) (*Node, error) {
+	cfg := NodeConfig{
+		NumWorkers:           runtime.NumCPU(),
+		InformerResyncPeriod: time.Minute,
+		KubeconfigPath:       os.Getenv("KUBECONFIG"),
+		HTTPListenAddr:       ":10250",
+		NodeSpec: v1.Node{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: name,
+				Labels: map[string]string{
+					"type":                   "virtual-kubelet",
+					"kubernetes.io/role":     "agent",
+					"kubernetes.io/hostname": name,
+				},
+			},
+			Status: v1.NodeStatus{
+				Phase: v1.NodePending,
+				Conditions: []v1.NodeCondition{
+					{Type: v1.NodeReady},
+					{Type: v1.NodeDiskPressure},
+					{Type: v1.NodeMemoryPressure},
+					{Type: v1.NodePIDPressure},
+					{Type: v1.NodeNetworkUnavailable},
+				},
+			},
+		},
+	}
+
+	cfg.Client = defaultClientFromEnv(cfg.KubeconfigPath)
+
+	for _, o := range opts {
+		if err := o(&cfg); err != nil {
+			return nil, err
+		}
+	}
+
+	if _, _, err := net.SplitHostPort(cfg.HTTPListenAddr); err != nil {
+		return nil, errors.Wrap(err, "error parsing http listen address")
+	}
+
+	if cfg.Client == nil {
+		return nil, errors.New("no client provided")
+	}
+
+	podInformerFactory := informers.NewSharedInformerFactoryWithOptions(
+		cfg.Client,
+		cfg.InformerResyncPeriod,
+		PodInformerFilter(name),
+	)
+
+	scmInformerFactory := informers.NewSharedInformerFactoryWithOptions(
+		cfg.Client,
+		cfg.InformerResyncPeriod,
+	)
+
+	podInformer := podInformerFactory.Core().V1().Pods()
+	secretInformer := scmInformerFactory.Core().V1().Secrets()
+	configMapInformer := scmInformerFactory.Core().V1().ConfigMaps()
+	serviceInformer := scmInformerFactory.Core().V1().Services()
+
+	p, np, err := newProvider(ProviderConfig{
+		Pods:       podInformer.Lister(),
+		ConfigMaps: configMapInformer.Lister(),
+		Secrets:    secretInformer.Lister(),
+		Services:   serviceInformer.Lister(),
+		Node:       &cfg.NodeSpec,
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "error creating provider")
+	}
+
+	if cfg.routeAttacher != nil {
+		cfg.routeAttacher(p, cfg, podInformer.Lister())
+	}
+
+	var readyCb func(context.Context) error
+	if np == nil {
+		nnp := node.NewNaiveNodeProvider()
+		np = nnp
+
+		readyCb = func(ctx context.Context) error {
+			setNodeReady(&cfg.NodeSpec)
+			err := nnp.UpdateStatus(ctx, &cfg.NodeSpec)
+			return errors.Wrap(err, "error marking node as ready")
+		}
+	}
+
+	nc, err := node.NewNodeController(
+		np,
+		&cfg.NodeSpec,
+		cfg.Client.CoreV1().Nodes(),
+		node.WithNodeEnableLeaseV1(NodeLeaseV1Client(cfg.Client), node.DefaultLeaseDuration),
+	)
+	if err != nil {
+		return nil, errors.Wrap(err, "error creating node controller")
+	}
+
+	var eb record.EventBroadcaster
+	if cfg.EventRecorder == nil {
+		eb = record.NewBroadcaster()
+		cfg.EventRecorder = eb.NewRecorder(scheme.Scheme, v1.EventSource{Component: path.Join(name, "pod-controller")})
+	}
+
+	pc, err := node.NewPodController(node.PodControllerConfig{
+		PodClient:         cfg.Client.CoreV1(),
+		EventRecorder:     cfg.EventRecorder,
+		Provider:          p,
+		PodInformer:       podInformer,
+		SecretInformer:    secretInformer,
+		ConfigMapInformer: configMapInformer,
+		ServiceInformer:   serviceInformer,
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "error creating pod controller")
+	}
+
+	return &Node{
+		nc:                 nc,
+		pc:                 pc,
+		readyCb:            readyCb,
+		ready:              make(chan struct{}),
+		done:               make(chan struct{}),
+		eb:                 eb,
+		podInformerFactory: podInformerFactory,
+		scmInformerFactory: scmInformerFactory,
+		client:             cfg.Client,
+		tlsConfig:          cfg.TLSConfig,
+		h:                  cfg.Handler,
+		listenAddr:         cfg.HTTPListenAddr,
+		workers:            cfg.NumWorkers,
+	}, nil
+}
+
+func setNodeReady(n *v1.Node) {
+	n.Status.Phase = v1.NodeRunning
+	for i, c := range n.Status.Conditions {
+		if c.Type != "Ready" {
+			continue
+		}
+
+		c.Message = "Kubelet is ready"
+		c.Reason = "KubeletReady"
+		c.Status = v1.ConditionTrue
+		c.LastHeartbeatTime = metav1.Now()
+		c.LastTransitionTime = metav1.Now()
+		n.Status.Conditions[i] = c
+		return
+	}
+}
+
+func defaultClientFromEnv(kubeconfigPath string) kubernetes.Interface {
+	client, err := ClientsetFromEnv(kubeconfigPath)
+	if err != nil {
+		log.G(context.TODO()).WithError(err).
+			Warn("Failed to create clientset from env. Ignore this error If you use your own client")
+	}
+	return client
+}

node/nodeutil/provider.go

@@ -0,0 +1,70 @@
+package nodeutil
+
+import (
+	"context"
+	"io"
+
+	"github.com/virtual-kubelet/virtual-kubelet/node"
+	"github.com/virtual-kubelet/virtual-kubelet/node/api"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	corev1listers "k8s.io/client-go/listers/core/v1"
+	stats "k8s.io/kubelet/pkg/apis/stats/v1alpha1"
+)
+
+// Provider contains the methods required to implement a virtual-kubelet provider.
+//
+// Errors produced by these methods should implement an interface from
+// github.com/virtual-kubelet/virtual-kubelet/errdefs package in order for the
+// core logic to be able to understand the type of failure
+type Provider interface {
+	node.PodLifecycleHandler
+
+	// GetContainerLogs retrieves the logs of a container by name from the provider.
+	GetContainerLogs(ctx context.Context, namespace, podName, containerName string, opts api.ContainerLogOpts) (io.ReadCloser, error)
+
+	// RunInContainer executes a command in a container in the pod, copying data
+	// between in/out/err and the container's stdin/stdout/stderr.
+	RunInContainer(ctx context.Context, namespace, podName, containerName string, cmd []string, attach api.AttachIO) error
+
+	// GetStatsSummary gets the stats for the node, including running pods
+	GetStatsSummary(context.Context) (*stats.Summary, error)
+}
+
+// ProviderConfig holds objects created by NewNodeFromClient that a provider may need to bootstrap itself.
+type ProviderConfig struct {
+	Pods       corev1listers.PodLister
+	ConfigMaps corev1listers.ConfigMapLister
+	Secrets    corev1listers.SecretLister
+	Services   corev1listers.ServiceLister
+	// Hack to allow the provider to set things on the node
+	// Since the provider is bootstrapped after the node object is configured
+	// Primarily this is due to carry-over from the pre-1.0 interfaces that expect the provider instead of the direct *caller* to configure the node.
+	Node *v1.Node
+}
+
+// NewProviderFunc is used from NewNodeFromClient to bootstrap a provider using the client/listers/etc created there.
+// If a nil node provider is returned a default one will be used.
+type NewProviderFunc func(ProviderConfig) (Provider, node.NodeProvider, error)
+
+// AttachProviderRoutes returns a NodeOpt which uses api.PodHandler to attach the routes to the provider functions.
+//
+// Note this only attaches routes, you'll need to ensure to set the handler in the node config.
+func AttachProviderRoutes(mux api.ServeMux) NodeOpt {
+	return func(cfg *NodeConfig) error {
+		cfg.routeAttacher = func(p Provider, cfg NodeConfig, pods corev1listers.PodLister) {
+			mux.Handle("/", api.PodHandler(api.PodHandlerConfig{
+				RunInContainer:   p.RunInContainer,
+				GetContainerLogs: p.GetContainerLogs,
+				GetPods:          p.GetPods,
+				GetPodsFromKubernetes: func(context.Context) ([]*v1.Pod, error) {
+					return pods.List(labels.Everything())
+				},
+				GetStatsSummary:       p.GetStatsSummary,
+				StreamIdleTimeout:     cfg.StreamIdleTimeout,
+				StreamCreationTimeout: cfg.StreamCreationTimeout,
+			}, true))
+		}
+		return nil
+	}
+}

node/nodeutil/tls.go

@@ -0,0 +1,83 @@
+package nodeutil
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"os"
+)
+
+// WithTLSConfig returns a NodeOpt which creates a base TLSConfig with the default cipher suites and tls min verions.
+// The tls config can be modified through functional options.
+func WithTLSConfig(opts ...func(*tls.Config) error) NodeOpt {
+	return func(cfg *NodeConfig) error {
+		tlsCfg := &tls.Config{
+			MinVersion:               tls.VersionTLS12,
+			PreferServerCipherSuites: true,
+			CipherSuites:             DefaultServerCiphers(),
+			ClientAuth:               tls.RequestClientCert,
+		}
+		for _, o := range opts {
+			if err := o(tlsCfg); err != nil {
+				return err
+			}
+		}
+
+		cfg.TLSConfig = tlsCfg
+		return nil
+	}
+}
+
+// WithCAFromPath makes a TLS config option to set up client auth using the path to a PEM encoded CA cert.
+func WithCAFromPath(p string) func(*tls.Config) error {
+	return func(cfg *tls.Config) error {
+		pem, err := os.ReadFile(p)
+		if err != nil {
+			return fmt.Errorf("error reading ca cert pem: %w", err)
+		}
+		cfg.ClientAuth = tls.RequireAndVerifyClientCert
+		return WithCACert(pem)(cfg)
+	}
+}
+
+// WithKeyPairFromPath make sa TLS config option which loads the key pair paths from disk and appends them to the tls config.
+func WithKeyPairFromPath(cert, key string) func(*tls.Config) error {
+	return func(cfg *tls.Config) error {
+		cert, err := tls.LoadX509KeyPair(cert, key)
+		if err != nil {
+			return err
+		}
+		cfg.Certificates = append(cfg.Certificates, cert)
+		return nil
+	}
+}
+
+// WithCACert makes a TLS config opotion which appends the provided PEM encoded bytes the tls config's cert pool.
+// If a cert pool is not defined on the tls config an empty one will be created.
+func WithCACert(pem []byte) func(*tls.Config) error {
+	return func(cfg *tls.Config) error {
+		if cfg.ClientCAs == nil {
+			cfg.ClientCAs = x509.NewCertPool()
+		}
+		if !cfg.ClientCAs.AppendCertsFromPEM(pem) {
+			return fmt.Errorf("could not parse ca cert pem")
+		}
+		return nil
+	}
+}
+
+// DefaultServerCiphers is the list of accepted TLS ciphers, with known weak ciphers elided
+// Note this list should be a moving target.
+func DefaultServerCiphers() []uint16 {
+	return []uint16{
+		tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
+		tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
+		tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
+		tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
+
+		tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
+		tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
+		tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
+		tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
+	}
+}

node/pod.go

@@ -17,12 +17,14 @@ package node
 import (
 	"context"
 	"fmt"
+	"strings"
 	"time"
 
+	"github.com/virtual-kubelet/virtual-kubelet/internal/queue"
+
 	"github.com/google/go-cmp/cmp"
 	pkgerrors "github.com/pkg/errors"
 	"github.com/virtual-kubelet/virtual-kubelet/internal/podutils"
-	"github.com/virtual-kubelet/virtual-kubelet/internal/queue"
 	"github.com/virtual-kubelet/virtual-kubelet/log"
 	"github.com/virtual-kubelet/virtual-kubelet/trace"
 	corev1 "k8s.io/api/core/v1"
@@ -214,14 +216,7 @@ func (pc *PodController) updatePodStatus(ctx context.Context, podFromKubernetes
 	}
 	kPod := obj.(*knownPod)
 	kPod.Lock()
-
 	podFromProvider := kPod.lastPodStatusReceivedFromProvider.DeepCopy()
-	if cmp.Equal(podFromKubernetes.Status, podFromProvider.Status) && podFromProvider.DeletionTimestamp == nil {
-		kPod.lastPodStatusUpdateSkipped = true
-		kPod.Unlock()
-		return nil
-	}
-	kPod.lastPodStatusUpdateSkipped = false
 	kPod.Unlock()
 	// Pod deleted by provider due some reasons. e.g. a K8s provider, pod created by deployment would be evicted when node is not ready.
 	// If we do not delete pod in K8s, deployment would not create a new one.
@@ -325,12 +320,14 @@ func (pc *PodController) enqueuePodStatusUpdate(ctx context.Context, pod *corev1
 	kpod := obj.(*knownPod)
 	kpod.Lock()
 	if cmp.Equal(kpod.lastPodStatusReceivedFromProvider, pod) {
+		kpod.lastPodStatusUpdateSkipped = true
 		kpod.Unlock()
 		return
 	}
+	kpod.lastPodStatusUpdateSkipped = false
 	kpod.lastPodStatusReceivedFromProvider = pod
 	kpod.Unlock()
-	pc.syncPodStatusFromProvider.Enqueue(key)
+	pc.syncPodStatusFromProvider.Enqueue(ctx, key)
 }
 
 func (pc *PodController) syncPodStatusFromProviderHandler(ctx context.Context, key string) (retErr error) {
@@ -367,7 +364,8 @@ func (pc *PodController) deletePodsFromKubernetesHandler(ctx context.Context, ke
 	ctx, span := trace.StartSpan(ctx, "deletePodsFromKubernetesHandler")
 	defer span.End()
 
-	namespace, name, err := cache.SplitMetaNamespaceKey(key)
+	uid, metaKey := getUIDAndMetaNamespaceKey(key)
+	namespace, name, err := cache.SplitMetaNamespaceKey(metaKey)
 	ctx = span.WithFields(ctx, log.Fields{
 		"namespace": namespace,
 		"name":      name,
@@ -397,15 +395,25 @@ func (pc *PodController) deletePodsFromKubernetesHandler(ctx context.Context, ke
 		span.SetStatus(err)
 		return err
 	}
-
+	if string(k8sPod.UID) != uid {
+		log.G(ctx).WithField("k8sPodUID", k8sPod.UID).WithField("uid", uid).Warn("Not deleting pod because remote pod has different UID")
+		return nil
+	}
 	if running(&k8sPod.Status) {
 		log.G(ctx).Error("Force deleting pod in running state")
 	}
 
 	// We don't check with the provider before doing this delete. At this point, even if an outstanding pod status update
 	// was in progress,
-	err = pc.client.Pods(namespace).Delete(ctx, name, *metav1.NewDeleteOptions(0))
+	deleteOptions := metav1.NewDeleteOptions(0)
+	deleteOptions.Preconditions = metav1.NewUIDPreconditions(uid)
+	err = pc.client.Pods(namespace).Delete(ctx, name, *deleteOptions)
 	if errors.IsNotFound(err) {
+		log.G(ctx).Warnf("Not deleting pod because %v", err)
+		return nil
+	}
+	if errors.IsConflict(err) {
+		log.G(ctx).Warnf("There was a conflict, maybe trying to delete a Pod that has been recreated: %v", err)
 		return nil
 	}
 	if err != nil {
@@ -414,3 +422,10 @@ func (pc *PodController) deletePodsFromKubernetesHandler(ctx context.Context, ke
 	}
 	return nil
 }
+
+func getUIDAndMetaNamespaceKey(key string) (string, string) {
+	idx := strings.LastIndex(key, "/")
+	uid := key[idx+1:]
+	metaKey := key[:idx]
+	return uid, metaKey
+}

node/pod_test.go

@@ -20,16 +20,20 @@ import (
 	"testing"
 	"time"
 
-	testutil "github.com/virtual-kubelet/virtual-kubelet/internal/test/util"
 	"golang.org/x/time/rate"
 	"gotest.tools/assert"
 	is "gotest.tools/assert/cmp"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	kubeinformers "k8s.io/client-go/informers"
 	"k8s.io/client-go/kubernetes/fake"
+	core "k8s.io/client-go/testing"
 	"k8s.io/client-go/util/workqueue"
+
+	testutil "github.com/virtual-kubelet/virtual-kubelet/internal/test/util"
 )
 
 type TestController struct {
@@ -44,11 +48,6 @@ func newTestController() *TestController {
 	rm := testutil.FakeResourceManager()
 	p := newMockProvider()
 	iFactory := kubeinformers.NewSharedInformerFactoryWithOptions(fk8s, 10*time.Minute)
-	rateLimiter := workqueue.NewMaxOfRateLimiter(
-		// The default upper bound is 1000 seconds. Let's not use that.
-		workqueue.NewItemExponentialFailureRateLimiter(5*time.Millisecond, 10*time.Millisecond),
-		&workqueue.BucketRateLimiter{Limiter: rate.NewLimiter(rate.Limit(10), 100)},
-	)
 	podController, err := NewPodController(PodControllerConfig{
 		PodClient:         fk8s.CoreV1(),
 		PodInformer:       iFactory.Core().V1().Pods(),
@@ -57,7 +56,21 @@ func newTestController() *TestController {
 		ConfigMapInformer: iFactory.Core().V1().ConfigMaps(),
 		SecretInformer:    iFactory.Core().V1().Secrets(),
 		ServiceInformer:   iFactory.Core().V1().Services(),
-		RateLimiter:       rateLimiter,
+		SyncPodsFromKubernetesRateLimiter: workqueue.NewMaxOfRateLimiter(
+			// The default upper bound is 1000 seconds. Let's not use that.
+			workqueue.NewItemExponentialFailureRateLimiter(5*time.Millisecond, 10*time.Millisecond),
+			&workqueue.BucketRateLimiter{Limiter: rate.NewLimiter(rate.Limit(10), 100)},
+		),
+		SyncPodStatusFromProviderRateLimiter: workqueue.NewMaxOfRateLimiter(
+			// The default upper bound is 1000 seconds. Let's not use that.
+			workqueue.NewItemExponentialFailureRateLimiter(5*time.Millisecond, 10*time.Millisecond),
+			&workqueue.BucketRateLimiter{Limiter: rate.NewLimiter(rate.Limit(10), 100)},
+		),
+		DeletePodsFromKubernetesRateLimiter: workqueue.NewMaxOfRateLimiter(
+			// The default upper bound is 1000 seconds. Let's not use that.
+			workqueue.NewItemExponentialFailureRateLimiter(5*time.Millisecond, 10*time.Millisecond),
+			&workqueue.BucketRateLimiter{Limiter: rate.NewLimiter(rate.Limit(10), 100)},
+		),
 	})
 
 	if err != nil {
@@ -320,6 +333,123 @@ func TestPodStatusDelete(t *testing.T) {
 	t.Logf("pod updated, container status: %+v, pod delete Time: %v", newPod.Status.ContainerStatuses[0].State.Terminated, newPod.DeletionTimestamp)
 }
 
+func TestReCreatePodRace(t *testing.T) {
+	ctx := context.Background()
+	c := newTestController()
+	pod := &corev1.Pod{}
+	pod.ObjectMeta.Namespace = "default"
+	pod.ObjectMeta.Name = "nginx"
+	pod.Spec = newPodSpec()
+	pod.UID = "aaaaa"
+	podCopy := pod.DeepCopy()
+	podCopy.UID = "bbbbb"
+
+	// test conflict
+	fk8s := &fake.Clientset{}
+	c.client = fk8s
+	c.PodController.client = fk8s.CoreV1()
+	key := fmt.Sprintf("%s/%s/%s", pod.Namespace, pod.Name, pod.UID)
+	c.knownPods.Store(key, &knownPod{lastPodStatusReceivedFromProvider: podCopy})
+	c.deletePodsFromKubernetes.Enqueue(ctx, key)
+	if err := c.podsInformer.Informer().GetStore().Add(pod); err != nil {
+		t.Fatal(err)
+	}
+	c.client.AddReactor("delete", "pods", func(action core.Action) (handled bool, ret runtime.Object, err error) {
+		name := action.(core.DeleteAction).GetName()
+		t.Logf("deleted pod %s", name)
+		return true, nil, errors.NewConflict(schema.GroupResource{Group: "", Resource: "pods"}, "nginx", fmt.Errorf("test conflict"))
+	})
+	c.client.AddReactor("get", "pods", func(action core.Action) (handled bool, ret runtime.Object, err error) {
+		name := action.(core.GetAction).GetName()
+		t.Logf("get pod %s", name)
+		return true, podCopy, nil
+	})
+
+	err := c.deletePodsFromKubernetesHandler(ctx, key)
+	if err != nil {
+		t.Error("Failed")
+	}
+	p, err := c.client.CoreV1().Pods(podCopy.Namespace).Get(ctx, podCopy.Name, v1.GetOptions{})
+	if err != nil {
+		t.Fatalf("Pod not exist, %v", err)
+	}
+	if p.UID != podCopy.UID {
+		t.Errorf("Desired uid: %v, get: %v", podCopy.UID, p.UID)
+	}
+	t.Log("pod conflict test success")
+
+	// test not found
+	c = newTestController()
+	fk8s = &fake.Clientset{}
+	c.client = fk8s
+	c.knownPods.Store(key, &knownPod{lastPodStatusReceivedFromProvider: podCopy})
+	c.deletePodsFromKubernetes.Enqueue(ctx, key)
+	if err = c.podsInformer.Informer().GetStore().Add(pod); err != nil {
+		t.Fatal(err)
+	}
+	c.client.AddReactor("delete", "pods", func(action core.Action) (handled bool, ret runtime.Object, err error) {
+		name := action.(core.DeleteAction).GetName()
+		t.Logf("deleted pod %s", name)
+		return true, nil, errors.NewNotFound(schema.GroupResource{Group: "", Resource: "pods"}, "nginx")
+	})
+
+	c.client.AddReactor("get", "pods", func(action core.Action) (handled bool, ret runtime.Object, err error) {
+		name := action.(core.GetAction).GetName()
+		t.Logf("get pod %s", name)
+		return true, nil, errors.NewNotFound(schema.GroupResource{Group: "", Resource: "pods"}, "nginx")
+	})
+
+	err = c.deletePodsFromKubernetesHandler(ctx, key)
+	if err != nil {
+		t.Error("Failed")
+	}
+	_, err = c.client.CoreV1().Pods(podCopy.Namespace).Get(ctx, podCopy.Name, v1.GetOptions{})
+	if err == nil {
+		t.Log("delete success")
+		return
+	}
+	if !errors.IsNotFound(err) {
+		t.Fatal("Desired pod not exist")
+	}
+	t.Log("pod not found test success")
+
+	// test uid not equal before query
+	c = newTestController()
+	fk8s = &fake.Clientset{}
+	c.client = fk8s
+	c.knownPods.Store(key, &knownPod{lastPodStatusReceivedFromProvider: podCopy})
+	c.deletePodsFromKubernetes.Enqueue(ctx, key)
+	// add new pod
+	if err = c.podsInformer.Informer().GetStore().Add(podCopy); err != nil {
+		t.Fatal(err)
+	}
+	c.client.AddReactor("delete", "pods", func(action core.Action) (handled bool, ret runtime.Object, err error) {
+		name := action.(core.DeleteAction).GetName()
+		t.Logf("deleted pod %s", name)
+		return true, nil, errors.NewNotFound(schema.GroupResource{Group: "", Resource: "pods"}, "nginx")
+	})
+
+	c.client.AddReactor("get", "pods", func(action core.Action) (handled bool, ret runtime.Object, err error) {
+		name := action.(core.GetAction).GetName()
+		t.Logf("get pod %s", name)
+		return true, nil, errors.NewNotFound(schema.GroupResource{Group: "", Resource: "pods"}, "nginx")
+	})
+
+	err = c.deletePodsFromKubernetesHandler(ctx, key)
+	if err != nil {
+		t.Error("Failed")
+	}
+	_, err = c.client.CoreV1().Pods(podCopy.Namespace).Get(ctx, podCopy.Name, v1.GetOptions{})
+	if err == nil {
+		t.Log("delete success")
+		return
+	}
+	if !errors.IsNotFound(err) {
+		t.Fatal("Desired pod not exist")
+	}
+	t.Log("pod uid conflict test success")
+}
+
 func newPodSpec() corev1.PodSpec {
 	return corev1.PodSpec{
 		Containers: []corev1.Container{

node/podcontroller.go

@@ -20,12 +20,11 @@ import (
 	"sync"
 	"time"
 
-	"github.com/virtual-kubelet/virtual-kubelet/internal/queue"
-
 	"github.com/google/go-cmp/cmp"
 	pkgerrors "github.com/pkg/errors"
 	"github.com/virtual-kubelet/virtual-kubelet/errdefs"
 	"github.com/virtual-kubelet/virtual-kubelet/internal/manager"
+	"github.com/virtual-kubelet/virtual-kubelet/internal/queue"
 	"github.com/virtual-kubelet/virtual-kubelet/log"
 	"github.com/virtual-kubelet/virtual-kubelet/trace"
 	corev1 "k8s.io/api/core/v1"
@@ -176,8 +175,20 @@ type PodControllerConfig struct {
 	SecretInformer    corev1informers.SecretInformer
 	ServiceInformer   corev1informers.ServiceInformer
 
-	// RateLimiter defines the rate limit of work queue
-	RateLimiter workqueue.RateLimiter
+	// SyncPodsFromKubernetesRateLimiter defines the rate limit for the SyncPodsFromKubernetes queue
+	SyncPodsFromKubernetesRateLimiter workqueue.RateLimiter
+	// SyncPodsFromKubernetesShouldRetryFunc allows for a custom retry policy for the SyncPodsFromKubernetes queue
+	SyncPodsFromKubernetesShouldRetryFunc ShouldRetryFunc
+
+	// DeletePodsFromKubernetesRateLimiter defines the rate limit for the DeletePodsFromKubernetesRateLimiter queue
+	DeletePodsFromKubernetesRateLimiter workqueue.RateLimiter
+	// DeletePodsFromKubernetesShouldRetryFunc allows for a custom retry policy for the SyncPodsFromKubernetes queue
+	DeletePodsFromKubernetesShouldRetryFunc ShouldRetryFunc
+
+	// SyncPodStatusFromProviderRateLimiter defines the rate limit for the SyncPodStatusFromProviderRateLimiter queue
+	SyncPodStatusFromProviderRateLimiter workqueue.RateLimiter
+	// SyncPodStatusFromProviderShouldRetryFunc allows for a custom retry policy for the SyncPodStatusFromProvider queue
+	SyncPodStatusFromProviderShouldRetryFunc ShouldRetryFunc
 
 	// Add custom filtering for pod informer event handlers
 	// Use this for cases where the pod informer handles more than pods assigned to this node
@@ -210,8 +221,14 @@ func NewPodController(cfg PodControllerConfig) (*PodController, error) {
 	if cfg.Provider == nil {
 		return nil, errdefs.InvalidInput("missing provider")
 	}
-	if cfg.RateLimiter == nil {
-		cfg.RateLimiter = workqueue.DefaultControllerRateLimiter()
+	if cfg.SyncPodsFromKubernetesRateLimiter == nil {
+		cfg.SyncPodsFromKubernetesRateLimiter = workqueue.DefaultControllerRateLimiter()
+	}
+	if cfg.DeletePodsFromKubernetesRateLimiter == nil {
+		cfg.DeletePodsFromKubernetesRateLimiter = workqueue.DefaultControllerRateLimiter()
+	}
+	if cfg.SyncPodStatusFromProviderRateLimiter == nil {
+		cfg.SyncPodStatusFromProviderRateLimiter = workqueue.DefaultControllerRateLimiter()
 	}
 	rm, err := manager.NewResourceManager(cfg.PodInformer.Lister(), cfg.SecretInformer.Lister(), cfg.ConfigMapInformer.Lister(), cfg.ServiceInformer.Lister())
 	if err != nil {
@@ -230,9 +247,9 @@ func NewPodController(cfg PodControllerConfig) (*PodController, error) {
 		podEventFilterFunc: cfg.PodEventFilterFunc,
 	}
 
-	pc.syncPodsFromKubernetes = queue.New(cfg.RateLimiter, "syncPodsFromKubernetes", pc.syncPodFromKubernetesHandler)
-	pc.deletePodsFromKubernetes = queue.New(cfg.RateLimiter, "deletePodsFromKubernetes", pc.deletePodsFromKubernetesHandler)
-	pc.syncPodStatusFromProvider = queue.New(cfg.RateLimiter, "syncPodStatusFromProvider", pc.syncPodStatusFromProviderHandler)
+	pc.syncPodsFromKubernetes = queue.New(cfg.SyncPodsFromKubernetesRateLimiter, "syncPodsFromKubernetes", pc.syncPodFromKubernetesHandler, cfg.SyncPodsFromKubernetesShouldRetryFunc)
+	pc.deletePodsFromKubernetes = queue.New(cfg.DeletePodsFromKubernetesRateLimiter, "deletePodsFromKubernetes", pc.deletePodsFromKubernetesHandler, cfg.DeletePodsFromKubernetesShouldRetryFunc)
+	pc.syncPodStatusFromProvider = queue.New(cfg.SyncPodStatusFromProviderRateLimiter, "syncPodStatusFromProvider", pc.syncPodStatusFromProviderHandler, cfg.SyncPodStatusFromProviderShouldRetryFunc)
 
 	return pc, nil
 }
@@ -292,14 +309,25 @@ func (pc *PodController) Run(ctx context.Context, podSyncWorkers int) (retErr er
 
 	var eventHandler cache.ResourceEventHandler = cache.ResourceEventHandlerFuncs{
 		AddFunc: func(pod interface{}) {
+			ctx, cancel := context.WithCancel(ctx)
+			defer cancel()
+			ctx, span := trace.StartSpan(ctx, "AddFunc")
+			defer span.End()
+
 			if key, err := cache.MetaNamespaceKeyFunc(pod); err != nil {
 				log.G(ctx).Error(err)
 			} else {
+				ctx = span.WithField(ctx, "key", key)
 				pc.knownPods.Store(key, &knownPod{})
-				pc.syncPodsFromKubernetes.Enqueue(key)
+				pc.syncPodsFromKubernetes.Enqueue(ctx, key)
 			}
 		},
 		UpdateFunc: func(oldObj, newObj interface{}) {
+			ctx, cancel := context.WithCancel(ctx)
+			defer cancel()
+			ctx, span := trace.StartSpan(ctx, "UpdateFunc")
+			defer span.End()
+
 			// Create a copy of the old and new pod objects so we don't mutate the cache.
 			oldPod := oldObj.(*corev1.Pod)
 			newPod := newObj.(*corev1.Pod)
@@ -308,6 +336,7 @@ func (pc *PodController) Run(ctx context.Context, podSyncWorkers int) (retErr er
 			if key, err := cache.MetaNamespaceKeyFunc(newPod); err != nil {
 				log.G(ctx).Error(err)
 			} else {
+				ctx = span.WithField(ctx, "key", key)
 				obj, ok := pc.knownPods.Load(key)
 				if !ok {
 					// Pods are only ever *added* to knownPods in the above AddFunc, and removed
@@ -317,30 +346,45 @@ func (pc *PodController) Run(ctx context.Context, podSyncWorkers int) (retErr er
 
 				kPod := obj.(*knownPod)
 				kPod.Lock()
-				if kPod.lastPodStatusUpdateSkipped && !cmp.Equal(newPod.Status, kPod.lastPodStatusReceivedFromProvider.Status) {
+				if kPod.lastPodStatusUpdateSkipped &&
+					(!cmp.Equal(newPod.Status, kPod.lastPodStatusReceivedFromProvider.Status) ||
+						!cmp.Equal(newPod.Annotations, kPod.lastPodStatusReceivedFromProvider.Annotations) ||
+						!cmp.Equal(newPod.Labels, kPod.lastPodStatusReceivedFromProvider.Labels) ||
+						!cmp.Equal(newPod.Finalizers, kPod.lastPodStatusReceivedFromProvider.Finalizers)) {
 					// The last pod from the provider -> kube api server was skipped, but we see they no longer match.
 					// This means that the pod in API server was changed by someone else [this can be okay], but we skipped
 					// a status update on our side because we compared the status received from the provider to the status
 					// received from the k8s api server based on outdated information.
-					pc.syncPodStatusFromProvider.Enqueue(key)
+					pc.syncPodStatusFromProvider.Enqueue(ctx, key)
 					// Reset this to avoid re-adding it continuously
 					kPod.lastPodStatusUpdateSkipped = false
 				}
 				kPod.Unlock()
 
 				if podShouldEnqueue(oldPod, newPod) {
-					pc.syncPodsFromKubernetes.Enqueue(key)
+					pc.syncPodsFromKubernetes.Enqueue(ctx, key)
 				}
 			}
 		},
 		DeleteFunc: func(pod interface{}) {
+			ctx, cancel := context.WithCancel(ctx)
+			defer cancel()
+			ctx, span := trace.StartSpan(ctx, "DeleteFunc")
+			defer span.End()
+
 			if key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(pod); err != nil {
 				log.G(ctx).Error(err)
 			} else {
+				k8sPod, ok := pod.(*corev1.Pod)
+				if !ok {
+					return
+				}
+				ctx = span.WithField(ctx, "key", key)
 				pc.knownPods.Delete(key)
-				pc.syncPodsFromKubernetes.Enqueue(key)
+				pc.syncPodsFromKubernetes.Enqueue(ctx, key)
 				// If this pod was in the deletion queue, forget about it
-				pc.deletePodsFromKubernetes.Forget(key)
+				key = fmt.Sprintf("%v/%v", key, k8sPod.UID)
+				pc.deletePodsFromKubernetes.Forget(ctx, key)
 			}
 		},
 	}
@@ -472,7 +516,8 @@ func (pc *PodController) syncPodInProvider(ctx context.Context, pod *corev1.Pod,
 	// more context is here: https://github.com/virtual-kubelet/virtual-kubelet/pull/760
 	if pod.DeletionTimestamp != nil && !running(&pod.Status) {
 		log.G(ctx).Debug("Force deleting pod from API Server as it is no longer running")
-		pc.deletePodsFromKubernetes.EnqueueWithoutRateLimit(key)
+		key = fmt.Sprintf("%v/%v", key, pod.UID)
+		pc.deletePodsFromKubernetes.EnqueueWithoutRateLimit(ctx, key)
 		return nil
 	}
 	obj, ok := pc.knownPods.Load(key)
@@ -508,7 +553,8 @@ func (pc *PodController) syncPodInProvider(ctx context.Context, pod *corev1.Pod,
 			return err
 		}
 
-		pc.deletePodsFromKubernetes.EnqueueAfter(key, time.Second*time.Duration(*pod.DeletionGracePeriodSeconds))
+		key = fmt.Sprintf("%v/%v", key, pod.UID)
+		pc.deletePodsFromKubernetes.EnqueueWithoutRateLimitWithDelay(ctx, key, time.Second*time.Duration(*pod.DeletionGracePeriodSeconds))
 		return nil
 	}
 

node/queue.go

@@ -0,0 +1,34 @@
+package node
+
+import (
+	"github.com/virtual-kubelet/virtual-kubelet/internal/queue"
+)
+
+// These are exportable definitions of the queue package:
+
+// ShouldRetryFunc is a mechanism to have a custom retry policy
+//
+// it is passed metadata about the work item when the handler returns an error. It returns the following:
+// * The key
+// * The number of attempts that this item has already had (and failed)
+// * The (potentially wrapped) error from the queue handler.
+//
+// The return value is an error, and optionally an amount to delay the work.
+// If an error is returned, the work will be aborted, and the returned error is bubbled up. It can be the error that
+// was passed in or that error can be wrapped.
+//
+// If the work item should be is to be retried, a delay duration may be specified. The delay is used to schedule when
+// the item should begin processing relative to now, it does not necessarily dictate when the item will start work.
+// Items are processed in the order they are scheduled. If the delay is nil, it will fall  back to the default behaviour
+// of the queue, and use the rate limiter that's configured to determine when to start work.
+//
+// If the delay is negative, the item will be scheduled "earlier" than now. This will result in the item being executed
+// earlier than other items in the FIFO work order.
+type ShouldRetryFunc = queue.ShouldRetryFunc
+
+// DefaultRetryFunc is the default function used for retries by the queue subsystem. Its only policy is that it gives up
+// after MaxRetries, and falls back to the rate limiter for all other retries.
+var DefaultRetryFunc = queue.DefaultRetryFunc
+
+// MaxRetries is the number of times we try to process a given key before permanently forgetting it.
+var MaxRetries = queue.MaxRetries

node/sync.go

@@ -134,7 +134,12 @@ func (p *syncProviderWrapper) syncPodStatuses(ctx context.Context) {
 
 	for _, pod := range pods {
 		if shouldSkipPodStatusUpdate(pod) {
-			log.G(ctx).Debug("Skipping pod status update")
+			log.G(ctx).WithFields(log.Fields{
+				"pod":       pod.Name,
+				"namespace": pod.Namespace,
+				"phase":     pod.Status.Phase,
+				"status":    pod.Status.Reason,
+			}).Debug("Skipping pod status update")
 			continue
 		}
 

test/e2e/basic.go

@@ -10,7 +10,7 @@ import (
 	"gotest.tools/assert"
 	v1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
-	"k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
+	stats "k8s.io/kubelet/pkg/apis/stats/v1alpha1"
 )
 
 const (
@@ -401,9 +401,9 @@ func (ts *EndToEndTestSuite) TestCreatePodWithMandatoryInexistentConfigMap(t *te
 
 // findPodInPodStats returns the index of the specified pod in the .pods field of the specified Summary object.
 // It returns an error if the specified pod is not found.
-func findPodInPodStats(summary *v1alpha1.Summary, pod *v1.Pod) (int, error) {
+func findPodInPodStats(summary *stats.Summary, pod *v1.Pod) (int, error) {
 	for i, p := range summary.Pods {
-		if p.PodRef.Namespace == pod.Namespace && p.PodRef.Name == pod.Name && string(p.PodRef.UID) == string(pod.UID) {
+		if p.PodRef.Namespace == pod.Namespace && p.PodRef.Name == pod.Name && p.PodRef.UID == string(pod.UID) {
 			return i, nil
 		}
 	}