Prometheus metrics are encoded as text, not JSON (#1101)

Co-authored-by: Sanchit Mehta <sanchit.mehta602@gmail.com>

.circleci/config.yml

@@ -1,143 +0,0 @@
-version: 2
-jobs:
-  validate:
-    resource_class: xlarge
-    docker:
-      - image: circleci/golang:1.12
-        environment:
-          GO111MODULE: "on"
-          GOPROXY: https://proxy.golang.org
-    working_directory: /go/src/github.com/virtual-kubelet/virtual-kubelet
-    steps:
-      - checkout
-      - restore_cache:
-          keys:
-            - validate-{{ checksum "go.mod" }}-{{ checksum "go.sum" }}
-      - run:
-          name: go vet
-          command: V=1 CI=1 make vet
-      - run:
-          name: Install linters
-          command: curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | bash -s v1.17.1 && mv ./bin/* /go/bin/
-      - run:
-          name: Lint
-          command: golangci-lint run ./...
-      - run:
-          name: Dependencies
-          command: scripts/validate/gomod.sh
-      - save_cache:
-          key: validate-{{ checksum "go.mod" }}-{{ checksum "go.sum" }}
-          paths:
-            - "/go/pkg/mod"
-
-  test:
-    resource_class: xlarge
-    docker:
-      - image: circleci/golang:1.12
-        environment:
-          GO111MODULE: "on"
-    working_directory: /go/src/github.com/virtual-kubelet/virtual-kubelet
-    steps:
-      - checkout
-      - restore_cache:
-          keys:
-            - test-{{ checksum "go.mod" }}-{{ checksum "go.sum" }}
-      - run:
-          name: Build
-          command: V=1 make build
-      - run:
-          name: Tests
-          command: V=1 CI=1 make test
-      - save_cache:
-          key: test-{{ checksum "go.mod" }}-{{ checksum "go.sum" }}
-          paths:
-            - "/go/pkg/mod"
-
-  e2e:
-    machine:
-      image: circleci/classic:201808-01
-    working_directory: /home/circleci/go/src/github.com/virtual-kubelet/virtual-kubelet
-    environment:
-      CHANGE_MINIKUBE_NONE_USER: true
-      GOPATH: /home/circleci/go
-      KUBECONFIG: /home/circleci/.kube/config
-      KUBERNETES_VERSION: v1.15.2
-      MINIKUBE_HOME: /home/circleci
-      MINIKUBE_VERSION: v1.2.0
-      MINIKUBE_WANTUPDATENOTIFICATION: false
-      MINIKUBE_WANTREPORTERRORPROMPT: false
-      SKAFFOLD_VERSION: v0.33.0
-      GO111MODULE: "on"
-    steps:
-      - checkout
-      - run:
-          name: Install kubectl
-          command: |
-            curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/${KUBERNETES_VERSION}/bin/linux/amd64/kubectl
-            chmod +x kubectl
-            sudo mv kubectl /usr/local/bin/
-            mkdir -p ${HOME}/.kube
-            touch ${HOME}/.kube/config
-      - run:
-          name: Install Skaffold
-          command: |
-            curl -Lo skaffold https://storage.googleapis.com/skaffold/releases/${SKAFFOLD_VERSION}/skaffold-linux-amd64
-            chmod +x skaffold
-            sudo mv skaffold /usr/local/bin/
-      - run:
-          name: Install Minikube
-          command: |
-            curl -Lo minikube https://storage.googleapis.com/minikube/releases/${MINIKUBE_VERSION}/minikube-linux-amd64
-            chmod +x minikube
-            sudo mv minikube /usr/local/bin/
-      - run:
-          name: Start Minikube
-          command: |
-            sudo -E minikube start --vm-driver=none --cpus 2 --memory 2048 --kubernetes-version=${KUBERNETES_VERSION}
-      - run:
-          name: Wait for Minikube
-          command: |
-            JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}';
-            until kubectl get nodes -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do
-              sleep 1;
-            done
-      - run:
-          name: Watch pods
-          command: kubectl get pods -o json --watch
-          background: true
-      - run:
-          name: Watch nodes
-          command: kubectl get nodes -o json --watch
-          background: true
-      - restore_cache:
-          keys:
-            - e2e-{{ checksum "go.mod" }}-{{ checksum "go.sum" }}-2
-      - run:
-          name: Run the end-to-end test suite
-          command: |
-            mkdir $HOME/.go
-            export PATH=$HOME/.go/bin:${PATH}
-            curl -fsSL -o "/tmp/go.tar.gz" "https://dl.google.com/go/go1.12.6.linux-amd64.tar.gz"
-            tar -C $HOME/.go --strip-components=1 -xzf "/tmp/go.tar.gz"
-            go version
-            make e2e
-      - save_cache:
-          key: e2e-{{ checksum "go.mod" }}-{{ checksum "go.sum" }}-2
-          paths:
-            - "/home/circleci/go/pkg/mod"
-      - run:
-          name: Collect logs on failure from vkubelet-mock-0
-          command: |
-            kubectl logs vkubelet-mock-0
-          when: on_fail
-
-workflows:
-  version: 2
-  validate_and_test:
-    jobs:
-      - validate
-      - test
-      - e2e:
-          requires:
-            - validate
-            - test

.dockerignore

@@ -2,3 +2,5 @@
 private.env
 *.private.*
 providers/azurebatch/deployment/
+Dockerfile
+.dockerignore

.github/dependabot.yml

@@ -0,0 +1,10 @@
+version: 2
+updates:
+  - package-ecosystem: "gomod"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"

.github/workflows/ci.yml

@@ -0,0 +1,110 @@
+name: CI
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+
+env:
+  GO_VERSION: "1.18"
+
+jobs:
+  lint:
+    name: Lint
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+      - uses: actions/setup-go@v4
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - uses: actions/checkout@v3
+      - uses: golangci/golangci-lint-action@v3
+        with:
+          version: v1.48.0
+          args: --timeout=5m
+
+  unit-tests:
+    name: Unit Tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+      - uses: actions/setup-go@v4
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - uses: actions/checkout@v3
+      - name: Run Tests
+        run: make test
+
+  env-tests:
+    name: Envtest Tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+      - uses: actions/setup-go@v4
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - uses: actions/checkout@v3
+      - name: Run Tests
+        run: make envtest
+
+  e2e:
+    name: E2E
+    runs-on: ubuntu-22.04
+    timeout-minutes: 10
+    env:
+      CHANGE_MINIKUBE_NONE_USER: true
+      KUBERNETES_VERSION: v1.20.1
+      MINIKUBE_HOME: /home/circleci
+      MINIKUBE_VERSION: v1.16.0
+      MINIKUBE_WANTUPDATENOTIFICATION: false
+      MINIKUBE_WANTREPORTERRORPROMPT: false
+      SKAFFOLD_VERSION: v1.17.2
+      GO111MODULE: "on"
+
+    steps:
+      - uses: actions/setup-go@v4
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Checkout repository
+        uses: actions/checkout@v3
+      - name: Install Skaffold
+        run: |
+          curl -Lo skaffold https://storage.googleapis.com/skaffold/releases/${SKAFFOLD_VERSION}/skaffold-linux-amd64
+          chmod +x skaffold
+          sudo mv skaffold /usr/local/bin/
+          echo /usr/local/bin >> $GITHUB_PATH
+      - name: Install Minikube dependencies
+        run: |
+          sudo apt-get update && sudo apt-get install -y apt-transport-https curl
+          curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -
+          cat <<EOF | sudo tee /etc/apt/sources.list.d/kubernetes.list
+          deb https://apt.kubernetes.io/ kubernetes-xenial main
+          EOF
+          sudo apt-get update
+          sudo apt-get install -y kubelet # systemd unit is disabled
+      - name: Install Minikube
+        run: |
+          curl -Lo minikube https://storage.googleapis.com/minikube/releases/${MINIKUBE_VERSION}/minikube-linux-amd64
+          chmod +x minikube
+          sudo mv minikube /usr/local/bin/
+      - name: Start Minikube
+        run: |
+          sudo -E PATH=$PATH minikube start --vm-driver=none --cpus 2 --memory 2048 --kubernetes-version=${KUBERNETES_VERSION}
+      - name: Wait for Minikube
+        run: |
+          JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}';
+          until kubectl get nodes -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do
+            sleep 1;
+          done
+      - name: Run Tests
+        run: make e2e

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,59 @@
+name: "CodeQL"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+on:
+  push:
+    branches: [master]
+  schedule:
+    - cron: "19 18 * * 3"
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: ["go"]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
+        # Learn more:
+        # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      # Initializes the CodeQL tools for scanning.
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v2
+        with:
+          languages: ${{ matrix.language }}
+          # If you wish to specify custom queries, you can do so here or in a config file.
+          # By default, queries listed here will override any specified in a config file.
+          # Prefix the list here with "+" to use these queries and those in the config file.
+          # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+      # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+      # If this step fails, then you should remove it and run the build manually (see below)
+      - name: Autobuild
+        uses: github/codeql-action/autobuild@v2
+
+      # ℹ️ Command-line programs to run using the OS shell.
+      # 📚 https://git.io/JvXDl
+
+      # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+      #    and modify them (or add more) to build your code if your project
+      #    uses a compiled language
+
+      #- run: |
+      #   make bootstrap
+      #   make release
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v2

.gitignore

@@ -41,3 +41,5 @@ loganalytics.json
 **/terraform-provider-kubernetes
 **/*.tfstate*
 debug
+
+vendor/

.golangci.yml

@@ -2,12 +2,38 @@ linter-settings:
   lll:
     line-length: 200
 
+timeout: 10m
+
+run:
+  skip-dirs:
+    # This directory contains copy code from upstream kubernetes/kubernetes, skip it.
+    - internal/kubernetes
+    # This is mostly copied from upstream, rather than fixing that code here just ignore the errors.
+    - internal/podutils
+
 linters:
-  disable-all: true
   enable:
     - errcheck
-    - govet
+    - structcheck
-    - ineffassign
+    - varcheck
-    - golint
+    - staticcheck
-    - goconst
+    - unconvert
+    - gofmt
     - goimports
+    - ineffassign
+    - vet
+    - unused
+    - misspell
+    - gosec
+    - exportloopref # Checks for pointers to enclosing loop variables
+    - tenv # Detects using os.Setenv instead of t.Setenv since Go 1.17
+
+linters-settings:
+  gosec:
+    excludes:
+      - G304
+issues:
+  exclude-use-default: false
+  exclude:
+    # EXC0001 errcheck: Almost all programs ignore errors on these functions and in most cases it's ok
+    - Error return value of .((os\.)?std(out|err)\..*|.*Close|.*Flush|os\.Remove(All)?|.*print(f|ln)?|os\.(Un)?Setenv). (is not checked|Errors unhandled)

ADOPTERS.md

@@ -1,3 +1,15 @@
 ## Virtual Kubelet adopters
 
+* Microsoft Azure
+* AWS
+* Alibaba
+* VMWare
+* Netflix
+* Hashi Corp
+* Admiralty
+* Elotl
+* Tencent Games
+
+Since end-users are specific per provider within VK we have many end-user customers that we don't have permission to list publically. Please contact ribhatia@microsoft.com for more informtation.
+
 Are you currently using Virtual Kubelet in production? Please let us know by adding your company name and a description of your use case to this document!

CONTRIBUTING.md

@@ -77,6 +77,70 @@ git remote add upstream git@github.com:virtual-kubelet/virtual-kubelet.git
 ```
 3. Submit a pull request.
 
+
+## Submission and Review guidelines
+
+We welcome and appreciate everyone to submit and review changes. Here are some guidelines to follow for help ensure
+a successful contribution experience.
+
+Please note these are general guidelines, and while they are a good starting point, they are not specifically rules.
+If you have a question about something, feel free to ask:
+
+- [#virtual-kubelet](https://kubernetes.slack.com/archives/C8YU1QP8W) on Kubernetes Slack
+- [virtualkubelet-dev@lists.cncf.io](mailto:virtualkubelet-dev@lists.cncf.io)
+- GitHub Issues
+
+#### Don't make breaking API changes.
+
+Since Virtual Kubelet has reached 1.0 it is a major goal of the project to keep a stable API.
+Breaking changes must only be considered if a 2.0 release is on the table, which should only come with thoughtful
+consideration of the projects users as well as maintenance burden.
+
+Also note that behavior changes in the runtime can have cascading effects that cause unintended failures. Behavior
+changes should come well documented and with ample consideration for downstream effects. If possible, they should be
+opt-in.
+
+#### Public APIs
+
+Public API's should be extendable and flexible without requiring breaking changes.
+While we can always add a new function (`Foo2()`), a new type, etc, doing so makes it harder for people to update to
+the new behavior.
+
+Build API interfaces that do not need to be changed to adopt new or improved functionality. Opinions on how a particular
+thing should work should be encoded by the user rather than implicit in the runtime. Defaults are fine, but defaults
+should be overridable.
+
+The smaller the surface area of an API, the easier it is to do more interesting things with it.
+
+#### Building blocks
+
+Don't overload functionality. If something is complicated to setup we can provide helpers or wrappers to do that, but
+don't require users to do things a certain way because this tends to diminish the usefulness, especially as it relates
+to runtimes.
+
+We also do not want the maintenance burden of every users individual edge cases.
+
+#### Use context.Context
+
+Probably if it is a public/exported API, it should take a `context.Context`. Even if it doesn't need one today, it may
+need it tomorrow, and then we have a breaking API change.
+
+We use `context.Context` for storing loggers, tracing spans, and cancellation all across the project. Better safe
+than sorry: add a `context.Context`.
+
+#### Errors
+
+Callers can't handle errors if they don't know what the error is, so make sure they can figure that out.
+We use a package `errdefs` to define the types of errors we currently look out for. We do not typically look for
+concrete error types, so check out `errdefs` and see if there is already an error type in there for your needs, or even
+create a new one.
+
+#### Testing
+
+Ideally all behavior would be tested, in practice this is not the case. Unit tests are great, and fast. There is also
+an end-to-end test suite for testing the overall behavior of the system. Please add tests. This is also a great place
+to get started if you are new to the codebase.
+
 ## Code of conduct
 
 Virtual Kubelet follows the [CNCF Code of Conduct](https://github.com/cncf/foundation/blob/master/code-of-conduct.md).

Dockerfile

@@ -1,4 +1,6 @@
-FROM golang:1.12 as builder
+ARG GOLANG_CI_LINT_VERSION
+
+FROM golang:1.18 as builder
 ENV PATH /go/bin:/usr/local/go/bin:$PATH
 ENV GOPATH /go
 COPY . /go/src/github.com/virtual-kubelet/virtual-kubelet
@@ -7,6 +9,22 @@ ARG BUILD_TAGS=""
 RUN make VK_BUILD_TAGS="${BUILD_TAGS}" build
 RUN cp bin/virtual-kubelet /usr/bin/virtual-kubelet
 
+FROM golangci/golangci-lint:${GOLANG_CI_LINT_VERSION} as lint
+WORKDIR /app
+COPY go.mod ./
+COPY go.sum ./
+RUN \
+    --mount=type=cache,target=/root/.cache/go-build \
+    --mount=type=cache,target=/go/pkg/mod \
+    go mod download
+COPY . .
+ARG OUT_FORMAT
+RUN \
+    --mount=type=cache,target=/root/.cache/go-build \
+    --mount=type=cache,target=/go/pkg/mod \
+    --mount=type=cache,target=/root/.cache/golangci-lint \
+    golangci-lint run -v --out-format="${OUT_FORMAT:-colored-line-number}"
+
 FROM scratch
 COPY --from=builder /usr/bin/virtual-kubelet /usr/bin/virtual-kubelet
 COPY --from=builder /etc/ssl/certs/ /etc/ssl/certs

Makefile

@@ -5,6 +5,8 @@ exec := $(DOCKER_IMAGE)
 github_repo := virtual-kubelet/virtual-kubelet
 binary := virtual-kubelet
 
+GOTEST ?= go test $(if $V,-v)
+
 export GO111MODULE ?= on
 
 include Makefile.e2e
@@ -13,17 +15,20 @@ include Makefile.e2e
 # Also, we will want to lock our tool versions using go mod:
 # https://github.com/golang/go/wiki/Modules#how-can-i-track-tool-dependencies-for-a-module
 gobin_tool ?= $(shell which gobin || echo $(GOPATH)/bin/gobin)
-goimports := golang.org/x/tools/cmd/goimports@release-branch.go1.12
+goimports := golang.org/x/tools/cmd/goimports@release-branch.go1.15
 gocovmerge := github.com/wadey/gocovmerge@b5bfa59ec0adc420475f97f89b58045c721d761c
 goreleaser := github.com/goreleaser/goreleaser@v0.82.2
 gox := github.com/mitchellh/gox@v1.0.1
 
 # comment this line out for quieter things
-# V := 1 # When V is set, print commands and build progress.
+# V := 1 # When V is set, try to enable extra logging for debugging
 
 # Space separated patterns of packages to skip in list, test, format.
 IGNORED_PACKAGES := /vendor/
 
+TEST_OS := $(shell go env GOOS)
+TEST_ARCH := $(shell go env GOARCH)
+
 .PHONY: all
 all: test build
 
@@ -31,19 +36,19 @@ all: test build
 # safebuild builds inside a docker container with no clingons from your $GOPATH
 safebuild:
 	@echo "Building..."
-	$Q docker build --build-arg BUILD_TAGS="$(VK_BUILD_TAGS)" -t $(DOCKER_IMAGE):$(VERSION) .
+	docker build --build-arg BUILD_TAGS="$(VK_BUILD_TAGS)" -t $(DOCKER_IMAGE):$(VERSION) .
 
 .PHONY: build
 build: build_tags := netgo osusergo
 build: OUTPUT_DIR ?= bin
 build: authors
 	@echo "Building..."
-	$Q CGO_ENABLED=0 go build -ldflags '-extldflags "-static"' -o $(OUTPUT_DIR)/$(binary) $(if $V,-v) $(VERSION_FLAGS) ./cmd/$(binary)
+	CGO_ENABLED=0 go build -ldflags '-extldflags "-static"' -o $(OUTPUT_DIR)/$(binary) $(if $V,-v) $(VERSION_FLAGS) ./cmd/$(binary)
 
 .PHONY: tags
 tags:
 	@echo "Listing tags..."
-	$Q @git tag
+	@git tag
 
 .PHONY: release
 release: build goreleaser
@@ -54,73 +59,63 @@ release: build goreleaser
 .PHONY: clean test list cover format docker
 mod:
 	@echo "Prune Dependencies..."
-	$Q go mod tidy
+	go mod tidy
 
 docker:
 	@echo "Docker Build..."
-	$Q docker build --build-arg BUILD_TAGS="$(VK_BUILD_TAGS)" -t $(DOCKER_IMAGE) .
+	docker build --build-arg BUILD_TAGS="$(VK_BUILD_TAGS)" -t $(DOCKER_IMAGE) .
 
 clean:
 	@echo "Clean..."
-	$Q rm -rf bin
+	rm -rf bin
 
 vet:
 	@echo "go vet'ing..."
 ifndef CI
 	@echo "go vet'ing Outside CI..."
-	$Q go vet $(allpackages)
+	go vet $(TESTDIRS)
 else
 	@echo "go vet'ing in CI..."
-	$Q mkdir -p test
+	mkdir -p test
-	$Q ( go vet $(allpackages); echo $$? ) | \
+	( go vet $(TESTDIRS); echo $$? ) | \
        tee test/vet.txt | sed '$$ d'; exit $$(tail -1 test/vet.txt)
 endif
 
 test:
-ifndef CI
+	$(GOTEST) $(TESTDIRS)
-	@echo "Testing..."
-	$Q go test  $(if $V,-v) $(allpackages)
-else
-	@echo "Testing in CI..."
-	$Q mkdir -p test
-	$Q ( GODEBUG=cgocheck=2 go test -timeout=9m -v $(allpackages); echo $$? ) | \
-       tee test/output.txt | sed '$$ d'; exit $$(tail -1 test/output.txt)
-endif
 
 list:
 	@echo "List..."
-	@echo $(allpackages)
+	@echo $(TESTDIRS)
 
 cover: gocovmerge
 	@echo "Coverage Report..."
 	@echo "NOTE: make cover does not exit 1 on failure, don't use it to check for tests success!"
-	$Q rm -f .GOPATH/cover/*.out cover/all.merged
+	rm -f .GOPATH/cover/*.out cover/all.merged
 	$(if $V,@echo "-- go test -coverpkg=./... -coverprofile=cover/... ./...")
-	@for MOD in $(allpackages); do \
+	@for MOD in $(TESTDIRS); do \
-        go test -coverpkg=`echo $(allpackages)|tr " " ","` \
+        go test -coverpkg=`echo $(TESTDIRS)|tr " " ","` \
             -coverprofile=cover/unit-`echo $$MOD|tr "/" "_"`.out \
             $$MOD 2>&1 | grep -v "no packages being tested depend on"; \
     done
-	$Q $(gobin_tool) -run $(gocovmerge) cover/*.out > cover/all.merged
+	$(gobin_tool) -run $(gocovmerge) cover/*.out > cover/all.merged
 ifndef CI
 	@echo "Coverage Report..."
-	$Q go tool cover -html .GOPATH/cover/all.merged
+	go tool cover -html .GOPATH/cover/all.merged
 else
 	@echo "Coverage Report In CI..."
-	$Q go tool cover -html .GOPATH/cover/all.merged -o .GOPATH/cover/all.html
+	go tool cover -html .GOPATH/cover/all.merged -o .GOPATH/cover/all.html
 endif
 	@echo ""
 	@echo "=====> Total test coverage: <====="
 	@echo ""
-	$Q go tool cover -func .GOPATH/cover/all.merged
+	go tool cover -func .GOPATH/cover/all.merged
 
 format: goimports
 	@echo "Formatting..."
-	$Q find . -iname \*.go | grep -v \
+	find . -iname \*.go | grep -v \
         -e "^$$" $(addprefix -e ,$(IGNORED_PACKAGES)) | xargs $(gobin_tool) -run $(goimports) -w
 
-
-
 ##### =====> Internals <===== #####
 
 .PHONY: setup
@@ -141,11 +136,7 @@ VERSION          := $(shell git describe --tags --always --dirty="-dev")
 DATE             := $(shell date -u '+%Y-%m-%d-%H:%M UTC')
 VERSION_FLAGS    := -ldflags='-X "main.buildVersion=$(VERSION)" -X "main.buildTime=$(DATE)"'
 
-# assuming go 1.9 here!!
+TESTDIRS ?= ./...
-_allpackages = $(shell go list ./...)
-
-# memoize allpackages, so that it's executed only once and only if used
-allpackages = $(if $(__allpackages),,$(eval __allpackages := $$(_allpackages)))$(__allpackages)
 
 .PHONY: goimports
 goimports: $(gobin_tool)
@@ -164,14 +155,38 @@ gox: $(gobin_tool)
 	# We make gox globally available, for people to use by hand
 	$(gobin_tool) $(gox)
 
-Q := $(if $V,,@)
-
 $(gobin_tool):
 	GO111MODULE=off go get -u github.com/myitcv/gobin
 
 authors:
-	$Q git log --all --format='%aN <%cE>' | sort -u  | sed -n '/github/!p' > GITAUTHORS
+	git log --all --format='%aN <%cE>' | sort -u  | sed -n '/github/!p' > GITAUTHORS
-	$Q cat AUTHORS GITAUTHORS  | sort -u > NEWAUTHORS
+	cat AUTHORS GITAUTHORS  | sort -u > NEWAUTHORS
-	$Q mv NEWAUTHORS AUTHORS
+	mv NEWAUTHORS AUTHORS
-	$Q rm -f NEWAUTHORS
+	rm -f NEWAUTHORS
-	$Q rm -f GITAUTHORS
+	rm -f GITAUTHORS
+
+checksums_2.3.1.txt:
+	curl -o checksums_2.3.1.txt -L https://github.com/kubernetes-sigs/kubebuilder/releases/download/v2.3.1/checksums.txt
+kubebuilder_2.3.1_${TEST_OS}_${TEST_ARCH}.tar.gz:
+	curl -C - -O -L https://github.com/kubernetes-sigs/kubebuilder/releases/download/v2.3.1/kubebuilder_2.3.1_${TEST_OS}_${TEST_ARCH}.tar.gz
+kubebuilder_2.3.1_${TEST_OS}_${TEST_ARCH}: kubebuilder_2.3.1_${TEST_OS}_${TEST_ARCH}.tar.gz checksums_2.3.1.txt
+	sha256sum -c --ignore-missing checksums_2.3.1.txt
+	tar -xvf kubebuilder_2.3.1_${TEST_OS}_${TEST_ARCH}.tar.gz
+
+.PHONY: envtest
+envtest: kubebuilder_2.3.1_${TEST_OS}_${TEST_ARCH}
+	# You can add klog flags for debugging, like: -klog.v=10 -klog.logtostderr
+	# klogv2 flags just wraps our existing logrus.
+	KUBEBUILDER_ASSETS=$(PWD)/kubebuilder_2.3.1_${TEST_OS}_${TEST_ARCH}/bin $(GOTEST) -run=TestEnvtest ./node -envtest=true
+
+.PHONY: fmt
+fmt:
+	goimports -w $(shell go list -f '{{.Dir}}' ./...)
+
+
+export GOLANG_CI_LINT_VERSION ?= v1.48.0
+DOCKER_BUILD ?= docker buildx build
+
+.PHONY: lint
+lint:
+	$(DOCKER_BUILD) --target=lint --build-arg GOLANG_CI_LINT_VERSION --build-arg OUT_FORMAT .

Makefile.e2e

@@ -39,7 +39,7 @@ e2e: NODE_NAME := vkubelet-mock-0
 e2e: export VK_BUILD_TAGS += mock_provider
 e2e: e2e.clean bin/e2e/virtual-kubelet skaffold/run
 	@echo Running tests...
-	cd $(PWD)/internal/test/e2e && go test -v -timeout 5m -tags e2e ./... \
+	cd $(PWD)/internal/test/e2e && $(GOTEST) -timeout 5m -tags e2e ./... \
 		-kubeconfig=$(KUBECONFIG) \
 		-namespace=$(NAMESPACE) \
 		-node-name=$(NODE_NAME)

README.md

@@ -1,8 +1,10 @@
 # Virtual Kubelet
 
+[![Go Reference](https://pkg.go.dev/badge/github.com/virtual-kubelet/virtual-kubelet.svg)](https://pkg.go.dev/github.com/virtual-kubelet/virtual-kubelet)
+
 Virtual Kubelet is an open source [Kubernetes kubelet](https://kubernetes.io/docs/reference/generated/kubelet/)
 implementation that masquerades as a kubelet for the purposes of connecting Kubernetes to other APIs.
-This allows the nodes to be backed by other services like ACI, AWS Fargate, [IoT Edge](https://github.com/Azure/iot-edge-virtual-kubelet-provider) etc. The primary scenario for VK is enabling the extension of the Kubernetes API into serverless container platforms like ACI and Fargate, though we are open to others. However, it should be noted that VK is explicitly not intended to be an alternative to Kubernetes federation.
+This allows the nodes to be backed by other services like ACI, AWS Fargate, [IoT Edge](https://github.com/Azure/iot-edge-virtual-kubelet-provider), [Tensile Kube](https://github.com/virtual-kubelet/tensile-kube) etc. The primary scenario for VK is enabling the extension of the Kubernetes API into serverless container platforms like ACI and Fargate, though we are open to others. However, it should be noted that VK is explicitly not intended to be an alternative to Kubernetes federation.
 
 Virtual Kubelet features a pluggable architecture and direct use of Kubernetes primitives, making it much easier to build on.
 
@@ -16,12 +18,16 @@ The best description is "Kubernetes API on top, programmable back."
 * [How It Works](#how-it-works)
 * [Usage](#usage)
 * [Providers](#providers)
+    + [Admiralty Multi-Cluster Scheduler](#admiralty-multi-cluster-scheduler)
     + [Alibaba Cloud ECI Provider](#alibaba-cloud-eci-provider)
     + [Azure Container Instances Provider](#azure-container-instances-provider)
 	+ [Azure Batch GPU Provider](https://github.com/virtual-kubelet/azure-batch/blob/master/README.md)
     + [AWS Fargate Provider](#aws-fargate-provider)
+    + [Elotl Kip](#elotl-kip)
 	+ [HashiCorp Nomad](#hashicorp-nomad-provider)
+    + [Liqo](#liqo-provider)
     + [OpenStack Zun](#openstack-zun-provider)
+    + [Tensile Kube Provider](#tensile-kube-provider)
     + [Adding a New Provider via the Provider Interface](#adding-a-new-provider-via-the-provider-interface)
 * [Testing](#testing)
     + [Unit tests](#unit-tests)
@@ -43,7 +49,7 @@ project to build a custom Kubernetes node agent.
 See godoc for up to date instructions on consuming this project:
 https://godoc.org/github.com/virtual-kubelet/virtual-kubelet
 
-There are implementations available for several provides (listed above), see
+There are implementations available for [several providers](#providers), see
 those repos for details on how to deploy.
 
 ## Current Features
@@ -73,6 +79,9 @@ Providers must provide the following functionality to be considered a supported
 2. Conforms to the current API provided by Virtual Kubelet.
 3. Does not have access to the Kubernetes API Server and has a well-defined callback mechanism for getting data like secrets or configmaps.
 
+### Admiralty Multi-Cluster Scheduler
+
+Admiralty Multi-Cluster Scheduler mutates annotated pods into "proxy pods" scheduled on a virtual-kubelet node and creates corresponding "delegate pods" in remote clusters (actually running the containers). A feedback loop updates the statuses and annotations of the proxy pods to reflect the statuses and annotations of the delegate pods. You can find more details in the [Admiralty Multi-Cluster Scheduler documentation](https://github.com/admiraltyio/multicluster-scheduler).
 
 ### Alibaba Cloud ECI Provider
 
@@ -99,10 +108,6 @@ You can find detailed instructions on how to set it up and how to test it in the
 The Azure connector can use a configuration file specified by the `--provider-config` flag.
 The config file is in TOML format, and an example lives in `providers/azure/example.toml`.
 
-#### More Details
-
-See the [ACI Readme](https://github.com/virtual-kubelet/alibabacloud-eci/blob/master/eci.toml)
-
 ### AWS Fargate Provider
 
 [AWS Fargate](https://aws.amazon.com/fargate/) is a technology that allows you to run containers
@@ -114,7 +119,13 @@ IP addresses to connect to the internet, private IP addresses to connect to your
 security groups, IAM roles, CloudWatch Logs and many other AWS services. Pods on Fargate can
 co-exist with pods on regular worker nodes in the same Kubernetes cluster.
 
-Easy instructions and a sample configuration file is available in the [AWS Fargate provider documentation](https://github.com/virtual-kubelet/aws-fargate/blob/master/README.md).
+Easy instructions and a sample configuration file is available in the [AWS Fargate provider documentation](https://github.com/virtual-kubelet/aws-fargate). Please note that this provider is not currently supported. 
+
+### Elotl Kip
+
+[Kip](https://github.com/elotl/kip) is a provider that runs pods in cloud instances, allowing a Kubernetes cluster to transparently scale workloads into a cloud. When a pod is scheduled onto the virtual node, Kip starts a right-sized cloud instance for the pod's workload and dispatches the pod onto the instance. When the pod is finished running, the cloud instance is terminated.
+
+When workloads run on Kip, your cluster size naturally scales with the cluster workload, pods are strongly isolated from each other and the user is freed from managing worker nodes and strategically packing pods onto nodes.
 
 ### HashiCorp Nomad Provider
 
@@ -126,6 +137,12 @@ would on a Kubernetes node.
 
 For detailed instructions, follow the guide [here](https://github.com/virtual-kubelet/nomad/blob/master/README.md).
 
+### Liqo Provider
+
+[Liqo](https://liqo.io) implements a provider for Virtual Kubelet designed to transparently offload pods and services to "peered" Kubernetes remote cluster. Liqo is capable of discovering neighbor clusters (using DNS, mDNS) and "peer" with them, or in other words, establish a relationship to share part of the cluster resources. When a cluster has established a peering, a new instance of the Liqo Virtual Kubelet is spawned to seamlessly extend the capacity of the cluster, by providing an abstraction of the resources of the remote cluster. The provider combined with the Liqo network fabric extends the cluster networking by enabling Pod-to-Pod traffic and multi-cluster east-west services, supporting endpoints on both clusters.
+
+For detailed instruction, follow the guide [here](https://github.com/liqotech/liqo/blob/master/README.md)
+
 ### OpenStack Zun Provider
 
 OpenStack [Zun](https://docs.openstack.org/zun/latest/) provider for Virtual Kubelet connects
@@ -141,6 +158,11 @@ and bind-mount Cinder volumes into a path inside a pod's container.
 
 For detailed instructions, follow the guide [here](https://github.com/virtual-kubelet/openstack-zun/blob/master/README.md).
 
+### Tensile Kube Provider
+
+[Tensile kube](https://github.com/virtual-kubelet/tensile-kube/blob/master/README.md) is contributed by [tencent
+ games](https://game.qq.com), which is provider for Virtual Kubelet connects your Kubernetes cluster with other Kubernetes clusters. This provider enables us extending Kubernetes to an unlimited one. By using the provider, pods that are scheduled on the virtual node registered on Kubernetes will run as jobs on other Kubernetes clusters' nodes.
+
 ### Adding a New Provider via the Provider Interface
 
 Providers consume this project as a library which implements the core logic of
@@ -251,6 +273,11 @@ One of the roles of a Kubelet is to accept requests from the API server for
 things like `kubectl logs` and  `kubectl exec`. Helpers for setting this up are
 provided [here](https://godoc.org/github.com/virtual-kubelet/virtual-kubelet/node/api)
 
+#### Scrape Pod metrics
+
+If you want to use HPA(Horizontal Pod Autoscaler) in your cluster, the provider should implement the `GetStatsSummary` function. Then metrics-server will be able to get the metrics of the pods on virtual-kubelet. Otherwise, you may see `No metrics for pod ` on metrics-server, which means the metrics of the pods on virtual-kubelet are not collected.
+
+
 ## Testing
 
 ### Unit tests
@@ -280,8 +307,8 @@ Enable the ServiceNodeExclusion flag, by modifying the Controller Manager manife
 Virtual Kubelet follows the [CNCF Code of Conduct](https://github.com/cncf/foundation/blob/master/code-of-conduct.md).
 Sign the [CNCF CLA](https://github.com/kubernetes/community/blob/master/CLA.md) to be able to make Pull Requests to this repo.
 
-Bi-weekly Virtual Kubelet Architecture meetings are held at 11am PST in this [zoom meeting room](https://zoom.us/j/245165908).  Check out the calendar [here](https://calendar.google.com/calendar?cid=bjRtbGMxYWNtNXR0NXQ1a2hqZmRkNTRncGNAZ3JvdXAuY2FsZW5kYXIuZ29vZ2xlLmNvbQ).
+Monthly Virtual Kubelet Office Hours are held at 10am PST on the second Thursday of every month in this [zoom meeting room](https://zoom.us/j/94701509915). Check out the calendar [here](https://calendar.google.com/calendar/embed?src=b119ced62134053de07d6c261b50d21ebde0da54f4163f5771b60ecf906e8b90%40group.calendar.google.com&ctz=America%2FLos_Angeles).
 
 Our google drive with design specifications and meeting notes are [here](https://drive.google.com/drive/folders/19Ndu11WBCCBDowo9CrrGUHoIfd2L8Ueg?usp=sharing).
 
-
+We also have a community slack channel named virtual-kubelet in the Kubernetes slack. You can also connect with the Virtual Kubelet community via the [mailing list](https://lists.cncf.io/g/virtualkubelet-dev).

charts/README.md

@@ -1,14 +0,0 @@
-# The Virtual Kubelet Helm chart
-
-Each version of Virtual Kubelet has a dedicated [Helm](https://helm.sh) chart. Those charts are served as static assets directly from GitHub.
-
-## The `index.yaml` file
-
-This subdirectory has an `index.yaml` file, which is necessary for it to act as a Helm chart repository. To re-generate the `index.yaml` file (assuming that you have Helm installed):
-
-```shell
-cd /path/to/virtual-kubelet
-helm repo index charts
-```
-
-The `index.yaml` then needs to be committed to Git and merged to `master`.

charts/index.yaml

@@ -1,231 +0,0 @@
-apiVersion: v1
-entries:
-  virtual-kubelet:
-  - appVersion: "0.5"
-    created: 2019-01-28T11:18:23.622097-08:00
-    description: A Helm chart to install virtual kubelet inside a Kubernetes cluster.
-    digest: b4af3cc07e1a914b862ebcd05facbf155b16e098a138fcd7f5dc8ac715aabfa2
-    icon: https://avatars2.githubusercontent.com/u/34250142
-    maintainers:
-    - email: junjiez@microsoft.com
-      name: Robbie Zhang
-    name: virtual-kubelet
-    sources:
-    - https://github.com/virtual-kubelet/virtual-kubelet
-    urls:
-    - virtual-kubelet-0.5.0.tgz
-    version: 0.5.0
-  - appVersion: "0.5"
-    created: 2019-01-28T11:18:23.62762-08:00
-    description: A Helm chart to install virtual kubelet inside a Kubernetes cluster.
-    digest: b4af3cc07e1a914b862ebcd05facbf155b16e098a138fcd7f5dc8ac715aabfa2
-    icon: https://avatars2.githubusercontent.com/u/34250142
-    maintainers:
-    - email: junjiez@microsoft.com
-      name: Robbie Zhang
-    name: virtual-kubelet
-    sources:
-    - https://github.com/virtual-kubelet/virtual-kubelet
-    urls:
-    - virtual-kubelet-latest.tgz
-    version: 0.5.0
-  - appVersion: "0.4"
-    created: 2019-01-28T11:18:23.62156-08:00
-    description: A Helm chart to install virtual kubelet inside a Kubernetes cluster.
-    digest: ae4b8be9d69129f1002ea2228848ec790ea1280d9ff0f7dd99d0d6e3e13922f2
-    icon: https://avatars2.githubusercontent.com/u/34250142
-    maintainers:
-    - email: junjiez@microsoft.com
-      name: Robbie Zhang
-    name: virtual-kubelet
-    sources:
-    - https://github.com/virtual-kubelet/virtual-kubelet
-    urls:
-    - virtual-kubelet-0.4.0.tgz
-    version: 0.4.0
-  - appVersion: "0.3"
-    created: 2019-01-28T11:18:23.620905-08:00
-    description: A Helm chart to install virtual kubelet inside a Kubernetes cluster.
-    digest: f472f181c0724420d4f12218f8fc7f65d09fa02a8292f418d1537bf71231d643
-    icon: https://avatars2.githubusercontent.com/u/34250142
-    maintainers:
-    - email: junjiez@microsoft.com
-      name: Robbie Zhang
-    name: virtual-kubelet
-    sources:
-    - https://github.com/virtual-kubelet/virtual-kubelet
-    urls:
-    - virtual-kubelet-0.3.0.tgz
-    version: 0.3.0
-  - appVersion: "0.3"
-    created: 2019-01-28T11:18:23.6202-08:00
-    description: A Helm chart to install virtual kubelet inside a Kubernetes cluster.
-    digest: 5a2d0a269620ffe49498686161e853f49761ca4f905577fe1b8e552ab85fcaca
-    icon: https://avatars2.githubusercontent.com/u/34250142
-    maintainers:
-    - email: junjiez@microsoft.com
-      name: Robbie Zhang
-    name: virtual-kubelet
-    sources:
-    - https://github.com/virtual-kubelet/virtual-kubelet
-    urls:
-    - virtual-kubelet-0.2.0.tgz
-    version: 0.2.0
-  - created: 2019-01-28T11:18:23.619614-08:00
-    description: a Helm chart to install virtual kubelet inside a Kubernetes cluster.
-    digest: be2778949548cfb0cebe638cbe044d89045eb34ffc8d62180b86ff2f0f30b323
-    maintainers:
-    - email: junjiez@microsoft.com
-      name: Robbie Zhang
-    name: virtual-kubelet
-    sources:
-    - https://github.com/virtual-kubelet/virtual-kubelet
-    urls:
-    - virtual-kubelet-0.1.3.tgz
-    version: 0.1.3
-  - created: 2019-01-28T11:18:23.618974-08:00
-    description: a Helm chart to install virtual kubelet inside a Kubernetes cluster.
-    digest: b8519c66766d06a68671a2b8940cf44cccdf9321f144dead543f62d16325331e
-    maintainers:
-    - email: junjiez@microsoft.com
-      name: Robbie Zhang
-    name: virtual-kubelet
-    sources:
-    - https://github.com/virtual-kubelet/virtual-kubelet
-    urls:
-    - virtual-kubelet-0.1.2.tgz
-    version: 0.1.2
-  - created: 2019-01-28T11:18:23.618406-08:00
-    description: a Helm chart to install virtual kubelet inside a Kubernetes cluster.
-    digest: b15b3d5acde2cc264b5e8624c3bafcc249440c662ae353ef7012493eb0b6abf6
-    maintainers:
-    - email: junjiez@microsoft.com
-      name: Robbie Zhang
-    name: virtual-kubelet
-    sources:
-    - https://github.com/virtual-kubelet/virtual-kubelet
-    urls:
-    - virtual-kubelet-0.1.1.tgz
-    version: 0.1.1
-  - created: 2019-01-28T11:18:23.617865-08:00
-    description: a Helm chart to install virtual kubelet inside a Kubernetes cluster.
-    digest: 22c60ad2f7ea71abf58d65e52b91c2b9feca1ad6a15091321f7f12e5c43ecf81
-    maintainers:
-    - email: junjiez@microsoft.com
-      name: Robbie Zhang
-    name: virtual-kubelet
-    sources:
-    - https://github.com/virtual-kubelet/virtual-kubelet
-    urls:
-    - virtual-kubelet-0.1.0.tgz
-    version: 0.1.0
-  virtual-kubelet-for-aks:
-  - created: 2019-01-28T11:18:23.622632-08:00
-    description: a Helm chart to install virtual kubelet in an AKS or ACS cluster.
-    digest: 345cda5aeb537129e1ecc3d23c0cf47651454f672694a4402a7bbb5d3f3a91ba
-    maintainers:
-    - email: junjiez@microsoft.com
-      name: Robbie Zhang
-    name: virtual-kubelet-for-aks
-    sources:
-    - https://github.com/virtual-kubelet/virtual-kubelet
-    urls:
-    - virtual-kubelet-for-aks-0.1.10.tgz
-    version: 0.1.10
-  - created: 2019-01-28T11:18:23.627017-08:00
-    description: a Helm chart to install virtual kubelet in an AKS or ACS cluster.
-    digest: 345cda5aeb537129e1ecc3d23c0cf47651454f672694a4402a7bbb5d3f3a91ba
-    maintainers:
-    - email: junjiez@microsoft.com
-      name: Robbie Zhang
-    name: virtual-kubelet-for-aks
-    sources:
-    - https://github.com/virtual-kubelet/virtual-kubelet
-    urls:
-    - virtual-kubelet-for-aks-latest.tgz
-    version: 0.1.10
-  - created: 2019-01-28T11:18:23.626524-08:00
-    description: a Helm chart to install virtual kubelet in an AKS or ACS cluster.
-    digest: 4a30821657ff4ef4522060c9905e5659656c035126a7a9e650dd9bb54621b9eb
-    maintainers:
-    - email: junjiez@microsoft.com
-      name: Robbie Zhang
-    name: virtual-kubelet-for-aks
-    sources:
-    - https://github.com/virtual-kubelet/virtual-kubelet
-    urls:
-    - virtual-kubelet-for-aks-0.1.9.tgz
-    version: 0.1.9
-  - created: 2019-01-28T11:18:23.626081-08:00
-    description: a Helm chart to install virtual kubelet in an AKS or ACS cluster.
-    digest: 18d988bfb4d24b3674c6c690c0445b85cf3969471cfce74f7c49e87483cb497d
-    maintainers:
-    - email: junjiez@microsoft.com
-      name: Robbie Zhang
-    name: virtual-kubelet-for-aks
-    sources:
-    - https://github.com/virtual-kubelet/virtual-kubelet
-    urls:
-    - virtual-kubelet-for-aks-0.1.8.tgz
-    version: 0.1.8
-  - created: 2019-01-28T11:18:23.625442-08:00
-    description: a Helm chart to install virtual kubelet in an AKS or ACS cluster.
-    digest: 21c0719fe330981a170810ee4c3e84375106c176de08894827c8208a66f52112
-    maintainers:
-    - email: junjiez@microsoft.com
-      name: Robbie Zhang
-    name: virtual-kubelet-for-aks
-    sources:
-    - https://github.com/virtual-kubelet/virtual-kubelet
-    urls:
-    - virtual-kubelet-for-aks-0.1.7.tgz
-    version: 0.1.7
-  - created: 2019-01-28T11:18:23.624524-08:00
-    description: a Helm chart to install virtual kubelet in an AKS or ACS cluster.
-    digest: b5e72d03f04113a46350fa800135a67f5af9b4ffe3d6650bdeebd271b885e5aa
-    maintainers:
-    - email: junjiez@microsoft.com
-      name: Robbie Zhang
-    name: virtual-kubelet-for-aks
-    sources:
-    - https://github.com/virtual-kubelet/virtual-kubelet
-    urls:
-    - virtual-kubelet-for-aks-0.1.6.tgz
-    version: 0.1.6
-  - created: 2019-01-28T11:18:23.624071-08:00
-    description: a Helm chart to install virtual kubelet in an AKS or ACS cluster.
-    digest: 5b04abcc63dcc71b5ad25c8368ad1b114accf721e9c475c5a7f962a48fb0ed65
-    maintainers:
-    - email: junjiez@microsoft.com
-      name: Robbie Zhang
-    name: virtual-kubelet-for-aks
-    sources:
-    - https://github.com/virtual-kubelet/virtual-kubelet
-    urls:
-    - virtual-kubelet-for-aks-0.1.5.tgz
-    version: 0.1.5
-  - created: 2019-01-28T11:18:23.623547-08:00
-    description: a Helm chart to install virtual kubelet in an AKS or ACS cluster.
-    digest: b639126041dc4f3d0307f6a678d22021ba149f28612eb0bfc3903c75cf1ea414
-    maintainers:
-    - email: junjiez@microsoft.com
-      name: Robbie Zhang
-    name: virtual-kubelet-for-aks
-    sources:
-    - https://github.com/virtual-kubelet/virtual-kubelet
-    urls:
-    - virtual-kubelet-for-aks-0.1.4.tgz
-    version: 0.1.4
-  - created: 2019-01-28T11:18:23.623097-08:00
-    description: a Helm chart to install virtual kubelet in an AKS or ACS cluster.
-    digest: 1b428fd99c667482681c83b61753e7327b85ec2644ef4fa0e9366492a0e73cd7
-    maintainers:
-    - email: junjiez@microsoft.com
-      name: Robbie Zhang
-    name: virtual-kubelet-for-aks
-    sources:
-    - https://github.com/virtual-kubelet/virtual-kubelet
-    urls:
-    - virtual-kubelet-for-aks-0.1.3.tgz
-    version: 0.1.3
-generated: 2019-01-28T11:18:23.615432-08:00

charts/virtual-kubelet-for-aks/Chart.yaml

@@ -1,8 +0,0 @@
-name: virtual-kubelet-for-aks
-version: 0.1.10
-description: a Helm chart to install virtual kubelet in an AKS or ACS cluster.
-sources:
-  - https://github.com/virtual-kubelet/virtual-kubelet
-maintainers:
-  - name: Robbie Zhang
-    email: junjiez@microsoft.com

charts/virtual-kubelet-for-aks/templates/NOTES.txt

@@ -1,12 +0,0 @@
-The virtual kubelet is getting deployed on your cluster.
-
-To verify that virtual kubelet has started, run:
-
-  kubectl --namespace={{ .Release.Namespace }} get pods -l "app={{ template "fullname" . }}"
-
-{{- if (not .Values.env.apiserverCert) and (not .Values.env.apiserverKey) }}
-
-Note: 
-TLS key pair not provided for VK HTTP listener. A key pair was generated for you. This generated key pair is not suitable for production use.
-
-{{- end }}

charts/virtual-kubelet-for-aks/templates/_helpers.tpl

@@ -1,16 +0,0 @@
-{{/* vim: set filetype=mustache: */}}
-{{/*
-Expand the name of the chart.
-*/}}
-{{- define "name" -}}
-{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
-{{- end -}}
-
-{{/*
-Create a default fully qualified app name.
-We truncate at 24 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
-*/}}
-{{- define "fullname" -}}
-{{- $name := default .Chart.Name .Values.nameOverride -}}
-{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
-{{- end -}}

charts/virtual-kubelet-for-aks/templates/clusterrolebinding.yaml

@@ -1,14 +0,0 @@
-{{ if .Values.rbac.install }}
-apiVersion: "rbac.authorization.k8s.io/{{ .Values.rbac.apiVersion }}"
-kind: ClusterRoleBinding
-metadata:
-  name: {{ template "fullname" . }}
-subjects:
-- kind: ServiceAccount
-  name: {{ template "fullname" . }}
-  namespace: {{ .Release.Namespace }}
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: {{ .Values.rbac.roleRef }}
-{{ end }}

charts/virtual-kubelet-for-aks/templates/deployment.yaml

@@ -1,79 +0,0 @@
-apiVersion: extensions/v1beta1
-kind: Deployment
-metadata:
-  name: {{ template "fullname" . }}
-spec:
-  replicas: 1
-  template:
-    metadata:
-      labels:
-        app: {{ template "fullname" . }}
-    spec:
-      containers:
-      - name: {{ template "fullname" . }}
-        image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
-        imagePullPolicy: {{ .Values.image.pullPolicy }}
-        env:
-        - name: KUBELET_PORT
-          value: "10250"
-        - name: ACS_CREDENTIAL_LOCATION
-          value: /etc/acs/azure.json
-        - name: AZURE_TENANT_ID
-          value: {{ .Values.env.azureTenantId }}
-        - name: AZURE_SUBSCRIPTION_ID
-          value: {{ .Values.env.azureSubscriptionId }}
-        - name: AZURE_CLIENT_ID
-          value: {{ .Values.env.azureClientId }}
-        - name: AZURE_CLIENT_SECRET
-          valueFrom:
-            secretKeyRef:
-              name: {{ template "fullname" . }}
-              key: clientSecret
-        - name: ACI_RESOURCE_GROUP
-          value: {{ .Values.env.aciResourceGroup }}
-        - name: ACI_REGION
-          value: {{ default "westus" .Values.env.aciRegion }}
-        - name: APISERVER_CERT_LOCATION
-          value: /etc/virtual-kubelet/cert.pem
-        - name: APISERVER_KEY_LOCATION
-          value: /etc/virtual-kubelet/key.pem
-        - name: VKUBELET_POD_IP
-          valueFrom:
-            fieldRef:
-              fieldPath: status.podIP
-        - name: ACI_EXTRA_USER_AGENT
-          value: {{ printf "helm-chart/aks/%s/%s" .Chart.Name .Chart.Version }}
-        - name: ACI_SUBNET_NAME
-          value: {{ .Values.env.aciVnetSubnetName }}
-        - name: ACI_SUBNET_CIDR
-          value: {{ .Values.env.aciVnetSubnetCIDR }}
-        - name: MASTER_URI
-          value: {{ .Values.env.masterUri }}
-        - name: CLUSTER_CIDR
-          value: {{ .Values.env.clusterCIDR }}
-        - name: KUBE_DNS_IP
-          value: {{ .Values.env.kubeDnsIP }}
-        {{ if .Values.loganalytics.enabled }}
-        - name: LOG_ANALYTICS_AUTH_LOCATION
-          value: /etc/virtual-kubelet/loganalytics.json
-        {{ end }}
-        volumeMounts:
-        - name: credentials
-          mountPath: "/etc/virtual-kubelet"
-        - name: acs-credential
-          mountPath: "/etc/acs/azure.json"
-        command: ["virtual-kubelet"]
-        args: ["--provider", "azure", "--namespace", {{ default "" .Values.env.monitoredNamespace | quote }}, "--nodename", {{ default "virtual-kubelet" .Values.env.nodeName | quote }} , "--os", {{ default "Linux" .Values.env.nodeOsType | quote }} ]
-      volumes:
-      - name: credentials
-        secret:
-          secretName: {{ template "fullname" . }}
-      - name: acs-credential
-        hostPath:
-          path: /etc/kubernetes/azure.json
-          type: File
-      {{ if .Values.rbac.install }}
-      serviceAccountName: {{ template "fullname" . }}
-      {{ end }}
-      nodeSelector:
-        beta.kubernetes.io/os: linux

charts/virtual-kubelet-for-aks/templates/secrets.yaml

@@ -1,22 +0,0 @@
-apiVersion: v1
-kind: Secret
-metadata:
-  name: {{ template "fullname" . }}
-type: Opaque
-data:
-  {{- if (not .Values.env.apiserverCert) and (not .Values.env.apiserverKey) }}
-  {{- $ca := genCA "virtual-kubelet-ca" 3650 }}
-  {{- $cn := printf "%s-virtual-kubelet-apiserver" .Release.Name }}
-  {{- $altName1 := printf "%s-virtual-kubelet-apiserver.%s" .Release.Name .Release.Namespace }}
-  {{- $altName2 := printf "%s-virtual-kubelet-apiserver.%s.svc" .Release.Name .Release.Namespace }}
-  {{- $cert := genSignedCert $cn nil (list $altName1 $altName2) 3650 $ca }}
-  cert.pem: {{ b64enc $cert.Cert }}
-  key.pem: {{ b64enc $cert.Key }}
-  {{ else }}
-  cert.pem: {{ quote .Values.env.apiserverCert }}
-  key.pem: {{ quote .Values.env.apiserverKey }}
-  {{ end}}
-  clientSecret: {{ default "" .Values.env.azureClientKey | b64enc | quote }}
-  {{ if .Values.loganalytics.enabled }}
-  loganalytics.json: {{ printf "{\"workspaceID\": \"%s\",\"workspaceKey\": \"%s\"}" (required "workspaceID is required for loganalytics" .Values.loganalytics.workspaceID ) (required "workspaceKey is required for loganalytics" .Values.loganalytics.workspaceKey ) | b64enc | quote }}
-  {{ end }}

charts/virtual-kubelet-for-aks/templates/serviceaccount.yaml

@@ -1,6 +0,0 @@
-{{ if .Values.rbac.install }}
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: {{ template "fullname" . }}
-{{ end }}

charts/virtual-kubelet-for-aks/values.yaml

@@ -1,34 +0,0 @@
-image:
-  repository: microsoft/virtual-kubelet
-  tag: latest
-  pullPolicy: Always
-env:
-  azureClientId: 
-  azureClientKey: 
-  azureTenantId: 
-  azureSubscriptionId: 
-  aciResourceGroup: 
-  aciRegion: 
-  nodeName: 
-  nodeTaint: 
-  nodeOsType: 
-  apiserverCert: 
-  apiserverKey: 
-  monitoredNamespace:
-  aciVnetSubnetName:
-  aciVnetSubnetCidr:
-  masterUri:
-  clusterCidr:
-  kubeDnsIp:
-loganalytics:
-  enabled: false
-  workspaceID:
-  workspaceKey:
-
-# Install Default RBAC roles and bindings
-rbac:
-  install: true
-  ## RBAC api version
-  apiVersion: v1beta1
-  # Cluster role reference
-  roleRef: cluster-admin

charts/virtual-kubelet/Chart.yaml

@@ -1,10 +0,0 @@
-name: virtual-kubelet
-version: 0.5.0
-appVersion: 0.5
-description: A Helm chart to install virtual kubelet inside a Kubernetes cluster.
-icon: https://avatars2.githubusercontent.com/u/34250142
-sources:
-  - https://github.com/virtual-kubelet/virtual-kubelet
-maintainers:
-  - name: Robbie Zhang
-    email: junjiez@microsoft.com

charts/virtual-kubelet/templates/NOTES.txt

@@ -1,12 +0,0 @@
-The virtual kubelet is getting deployed on your cluster.
-
-To verify that virtual kubelet has started, run:
-
-  kubectl --namespace={{ .Release.Namespace }} get pods -l "app={{ template "vk.name" . }}"
-
-{{- if (not .Values.apiserverCert) and (not .Values.apiserverKey) }}
-
-Note: 
-TLS key pair not provided for VK HTTP listener. A key pair was generated for you. This generated key pair is not suitable for production use.
-
-{{- end }}

charts/virtual-kubelet/templates/_helpers.tpl

@@ -1,29 +0,0 @@
-{{/* vim: set filetype=mustache: */}}
-{{/*
-Expand the name of the chart.
-*/}}
-{{- define "vk.name" -}}
-{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
-{{- end -}}
-
-{{/*
-Create a default fully qualified app name.
-We truncate at 24 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
-*/}}
-{{- define "vk.fullname" -}}
-{{- $name := default .Chart.Name .Values.nameOverride -}}
-{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
-{{- end -}}
-
-{{/*
-Standard labels for helm resources
-*/}}
-{{- define "vk.labels" -}}
-labels:
-  heritage: "{{ .Release.Service }}"
-  release: "{{ .Release.Name }}"
-  revision: "{{ .Release.Revision }}"
-  chart: "{{ .Chart.Name }}"
-  chartVersion: "{{ .Chart.Version }}"
-  app: {{ template "vk.name" . }}
-{{- end -}}

charts/virtual-kubelet/templates/clusterrolebinding.yaml

@@ -1,15 +0,0 @@
-{{ if .Values.rbac.install }}
-apiVersion: "rbac.authorization.k8s.io/{{ .Values.rbac.apiVersion }}"
-kind: ClusterRoleBinding
-metadata:
-  name: {{ template "vk.fullname" . }}
-{{ include "vk.labels" . | indent 2 }}
-subjects:
-- kind: ServiceAccount
-  name: {{ template "vk.fullname" . }}
-  namespace: {{ .Release.Namespace }}
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: {{ .Values.rbac.roleRef }}
-{{ end }}

charts/virtual-kubelet/templates/deployment.yaml

@@ -1,159 +0,0 @@
-apiVersion: extensions/v1beta1
-kind: Deployment
-metadata:
-  name: {{ template "vk.fullname" . }}
-{{ include "vk.labels" . | indent 2 }}
-    component: kubelet
-spec:
-  replicas: 1
-  template:
-    metadata:
-{{ include "vk.labels" . | indent 6 }}
-        component: kubelet
-      annotations:
-        checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }}
-    spec:
-      containers:
-{{- if eq .Values.trace.exporter "jaeger" }}
-{{- with .Values.traceExporters.jaeger }}
-{{- if eq .endpoint "" }}
-      - name: {{ tpl .name $ }}
-        image: {{ .image.repository}}:{{.image.tag}}
-        imagePullPolicy: {{ .image.pullPolicy }}
-        ports:
-          - containerPort: 16686
-{{- end }}
-{{- end }}
-{{- end }}
-      - name: {{ template "vk.fullname" . }}
-        image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
-        imagePullPolicy: {{ .Values.image.pullPolicy }}
-        env:
-        - name: KUBELET_PORT
-          value: "10250"
-        - name: APISERVER_CERT_LOCATION
-          value: /etc/virtual-kubelet/cert.pem
-        - name: APISERVER_KEY_LOCATION
-          value: /etc/virtual-kubelet/key.pem
-        - name: VKUBELET_POD_IP
-          valueFrom:
-            fieldRef:
-              fieldPath: status.podIP
-        - name: VKUBELET_TAINT_KEY
-          value: {{ .Values.taint.key }}
-        - name: VKUBELET_TAINT_VALUE
-          value: {{ tpl .Values.taint.value $ }}
-        - name: VKUBELET_TAINT_EFFECT
-          value: {{ .Values.taint.effect }}
-{{- if eq (required "You must specify a Virtual Kubelet provider" .Values.provider) "azure" }}
-{{- with .Values.providers.azure }}
-{{- if .loganalytics.enabled }}
-        - name: LOG_ANALYTICS_AUTH_LOCATION
-          value: /etc/virtual-kubelet/loganalytics.json
-        - name: CLUSTER_RESOURCE_ID
-          value:  {{ .loganalytics.clusterResourceId }}
-{{- end }}
-{{- if .targetAKS }}
-        - name: ACS_CREDENTIAL_LOCATION
-          value: /etc/acs/azure.json
-        - name: AZURE_TENANT_ID
-          value: {{ .tenantId }}
-        - name: AZURE_SUBSCRIPTION_ID
-          value: {{ .subscriptionId }}
-        - name: AZURE_CLIENT_ID
-          value: {{ .clientId }}
-        - name: AZURE_CLIENT_SECRET
-          valueFrom:
-            secretKeyRef:
-              name: {{ template "vk.fullname" $ }}
-              key: clientSecret
-        - name: ACI_RESOURCE_GROUP
-          value: {{ .aciResourceGroup }}
-        - name: ACI_REGION
-          value: {{ .aciRegion }}
-        - name: ACI_EXTRA_USER_AGENT
-          value: {{ printf "helm-chart/aks/%s/%s" $.Chart.Name $.Chart.Version }}
-{{- else }}
-        - name: AZURE_AUTH_LOCATION
-          value: /etc/virtual-kubelet/credentials.json
-        - name: ACI_RESOURCE_GROUP
-          value: {{ required "aciResourceGroup is required" .aciResourceGroup }}
-        - name: ACI_REGION
-          value: {{ required "aciRegion is required" .aciRegion }}
-        - name: ACI_EXTRA_USER_AGENT
-          value: {{ printf "helm-chart/other/%s/%s" $.Chart.Name $.Chart.Version }}
-{{- end }}
-{{- if .vnet.enabled }}
-        - name: ACI_SUBNET_NAME
-          value: {{ required "subnetName is required" .vnet.subnetName }}
-        - name: ACI_SUBNET_CIDR
-          value: {{ .vnet.subnetCidr }}
-        - name: MASTER_URI
-          value: {{ required "masterUri is required" .masterUri }}
-        - name: CLUSTER_CIDR
-          value: {{ required "clusterCidr is required" .vnet.clusterCidr }}
-        - name: KUBE_DNS_IP
-          value: {{ required "kubeDnsIp is required" .vnet.kubeDnsIp }}
-{{- else }}
-        - name: MASTER_URI
-          value: {{ .masterUri }}
-{{- end }}          
-{{- end }}
-{{- end }}
-{{- if eq .Values.trace.exporter "jaeger" }}
-        - name: JAEGER_ENDPOINT
-{{- with .Values.traceExporters.jaeger }}
-{{- if eq .endpoint "" }}
-          value: "http://127.0.0.1:14268"
-{{- else }}
-          value: {{.endpoint}}
-{{- end }}
-{{- end }}
-{{- end }}
-        volumeMounts:
-        - name: credentials
-          mountPath: "/etc/virtual-kubelet"
-{{- if eq (required "You must specify a Virtual Kubelet provider" .Values.provider) "azure" }}
-{{- if .Values.providers.azure.targetAKS }}
-        - name: acs-credential
-          mountPath: "/etc/acs/azure.json"
-{{- end }}
-{{- end }}
-        command: ["virtual-kubelet"]
-        args: [
-{{- if not .Values.taint.enabled }}
-          "--disable-taint", "true",
-{{- end }}
-          "--provider", "{{ required "You must specify a Virtual Kubelet provider" .Values.provider }}",
-          "--namespace", "{{ .Values.monitoredNamespace }}",
-          "--nodename", "{{ required "nodeName is required" .Values.nodeName }}",
-          {{- if .Values.logLevel }}
-          "--log-level", "{{.Values.logLevel}}",
-          {{- end }}
-{{- if ne .Values.trace.exporter "" }}
-          "--trace-exporter", "{{ .Values.trace.exporter }}",
-{{- if gt .Values.trace.sampleRate 0.0 }}
-          "--trace-sample-rate", "{{ .Values.trace.sampleRate }}",
-{{- end }}
-{{- $serviceName := tpl .Values.trace.serviceName $ }}
-{{- if ne $serviceName "" }}
-          "--trace-service-name", "{{ $serviceName }}",
-{{- end}}
-{{- end}}
-          "--os", "{{ .Values.nodeOsType }}"
-        ]
-      volumes:
-      - name: credentials
-        secret:
-          secretName: {{ template "vk.fullname" . }}
-{{- if eq (required "You must specify a Virtual Kubelet provider" .Values.provider) "azure" }}
-{{- if .Values.providers.azure.targetAKS }}
-      - name: acs-credential
-        hostPath:
-          path: /etc/kubernetes/azure.json
-          type: File
-{{- end }}
-{{- end }}
-      serviceAccountName: {{ if .Values.rbac.install }} "{{ template "vk.fullname" . }}" {{ end }}
-      nodeSelector:
-        beta.kubernetes.io/os: linux

charts/virtual-kubelet/templates/secret.yaml

@@ -1,31 +0,0 @@
-apiVersion: v1
-kind: Secret
-metadata:
-  name: {{ template "vk.fullname" . }}
-{{ include "vk.labels" . | indent 2 }}
-type: Opaque
-data:
-{{- if (not .Values.apiserverCert) and (not .Values.apiserverKey) }}
-{{- $ca := genCA "virtual-kubelet-ca" 3650 }}
-{{- $cn := printf "%s-virtual-kubelet-apiserver" .Release.Name }}
-{{- $altName1 := printf "%s-virtual-kubelet-apiserver.%s" .Release.Name .Release.Namespace }}
-{{- $altName2 := printf "%s-virtual-kubelet-apiserver.%s.svc" .Release.Name .Release.Namespace }}
-{{- $cert := genSignedCert $cn nil (list $altName1 $altName2) 3650 $ca }}
-  cert.pem: {{ b64enc $cert.Cert }}
-  key.pem: {{ b64enc $cert.Key }}
-{{- else }}
-  cert.pem: {{ quote .Values.apiserverCert }}
-  key.pem: {{ quote .Values.apiserverKey }}
-{{- end }}
-{{- if eq (required "You must specify a Virtual Kubelet provider" .Values.provider) "azure" }}
-{{- with .Values.providers.azure }}
-{{- if .loganalytics.enabled }}
-  loganalytics.json: {{ printf "{\"workspaceID\": \"%s\",\"workspaceKey\": \"%s\"}" (required "workspaceId is required for loganalytics" .loganalytics.workspaceId ) (required "workspaceKey is required for loganalytics" .loganalytics.workspaceKey ) | b64enc | quote }}
-{{- end }}
-{{- if .targetAKS }}
-  clientSecret: {{ default "" .clientKey | b64enc | quote }}
-{{- else }}
-  credentials.json: {{ printf "{ \"clientId\": \"%s\", \"clientSecret\": \"%s\", \"subscriptionId\": \"%s\", \"tenantId\": \"%s\", \"activeDirectoryEndpointUrl\": \"https://login.microsoftonline.com/\", \"resourceManagerEndpointUrl\": \"https://management.azure.com/\", \"activeDirectoryGraphResourceId\": \"https://graph.windows.net/\", \"sqlManagementEndpointUrl\": \"database.windows.net\", \"galleryEndpointUrl\": \"https://gallery.azure.com/\", \"managementEndpointUrl\": \"https://management.core.windows.net/\" }" (default "MISSING" .clientId) (default "MISSING" .clientKey) (default "MISSING" .subscriptionId) (default "MISSING" .tenantId) | b64enc | quote }}
-{{- end }}
-{{- end }}
-{{- end }}

charts/virtual-kubelet/templates/serviceaccount.yaml

@@ -1,7 +0,0 @@
-{{ if .Values.rbac.install }}
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: {{ template "vk.fullname" . }}
-{{ include "vk.labels" . | indent 2 }}
-{{ end }}

charts/virtual-kubelet/templates/tests/helloworld.yaml

@@ -1,30 +0,0 @@
-apiVersion: v1
-kind: Pod
-metadata:
-  name: "{{ .Release.Name }}-{{ .Release.Revision }}-test"
-{{ include "vk.labels" . | indent 2 }}
-    component: test
-  annotations:
-    "helm.sh/hook": test-success
-spec:
-  containers:
-  - image: hello-world:linux
-    imagePullPolicy: Always
-    name: helloworld
-    resources:
-      requests:
-        memory: "0.1G"
-        cpu: 10m
-      limits:
-        memory: "0.1G"
-        cpu: 10m
-  dnsPolicy: ClusterFirst
-  nodeSelector:
-    kubernetes.io/hostname: "{{ .Values.nodeName }}"
-  restartPolicy: Never
-  tolerations:
-{{- if .Values.taint.enabled }}
-  - key: "{{ .Values.taint.key }}"
-    value: "{{ tpl .Values.taint.value $ }}"
-    effect: "{{ .Values.taint.effect }}"
-{{- end }}

charts/virtual-kubelet/values.yaml

@@ -1,65 +0,0 @@
-image:
-  repository: microsoft/virtual-kubelet
-  tag: latest
-  pullPolicy: Always
-
-nodeName: "virtual-kubelet"
-nodeOsType: "Linux"
-monitoredNamespace: ""
-apiserverCert: 
-apiserverKey: 
-logLevel: 
-
-taint:
-  enabled: true
-  key: virtual-kubelet.io/provider
-  value: "{{ .Values.provider }}"
-  ## `effect` must be `NoSchedule`, `PreferNoSchedule` or `NoExecute`.
-  effect: NoSchedule
-
-trace:
-  exporter: ""
-  serviceName: "{{ .Values.nodeName }}"
-  sampleRate: 0
-
-traceExporters:
-  jaeger:
-    name: "{{ .Values.trace.exporter }}"
-    endpoint: ""
-    image:
-      repository: jaegertracing/all-in-one
-      tag: 1.8
-      pullPolicy: Always
-
-providers:
-  azure:
-    ## Set to true if deploying to Azure Kubernetes Service (AKS), otherwise false
-    targetAKS: true
-    clientId: 
-    clientKey: 
-    tenantId: 
-    subscriptionId: 
-    ## `aciResourceGroup` and `aciRegion` are required only for non-AKS deployments
-    aciResourceGroup: 
-    aciRegion: 
-    masterUri: 
-    loganalytics:
-      enabled: false
-      workspaceId: 
-      workspaceKey: 
-      clusterResourceId: 
-    vnet:
-      enabled: false
-      subnetName: 
-      subnetCidr: 
-      clusterCidr: 
-      kubeDnsIp: 
-
-## Install Default RBAC roles and bindings
-rbac:
-  install: true
-  serviceAccountName: virtual-kubelet
-  ## RBAC api version
-  apiVersion: v1beta1
-  ## Cluster role reference
-  roleRef: cluster-admin

cmd/virtual-kubelet/internal/commands/providers/provider.go

@@ -47,7 +47,6 @@ func NewCommand(s *provider.Store) *cobra.Command {
 				}
 				fmt.Fprintln(cmd.OutOrStdout(), args[0])
 			}
-			return
 		},
 	}
 }

cmd/virtual-kubelet/internal/commands/root/flag.go

@@ -22,7 +22,7 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/spf13/pflag"
-	"k8s.io/klog"
+	klog "k8s.io/klog/v2"
 )
 
 type mapVar map[string]string
@@ -59,7 +59,13 @@ func (mv mapVar) Type() string {
 
 func installFlags(flags *pflag.FlagSet, c *Opts) {
 	flags.StringVar(&c.KubeConfigPath, "kubeconfig", c.KubeConfigPath, "kube config file to use for connecting to the Kubernetes API server")
+
 	flags.StringVar(&c.KubeNamespace, "namespace", c.KubeNamespace, "kubernetes namespace (default is 'all')")
+	/* #nosec */
+	flags.MarkDeprecated("namespace", "Nodes must watch for pods in all namespaces. This option is now ignored.") //nolint:errcheck
+	/* #nosec */
+	flags.MarkHidden("namespace") //nolint:errcheck
+
 	flags.StringVar(&c.KubeClusterDomain, "cluster-domain", c.KubeClusterDomain, "kubernetes cluster-domain (default is 'cluster.local')")
 	flags.StringVar(&c.NodeName, "nodename", c.NodeName, "kubernetes node name")
 	flags.StringVar(&c.OperatingSystem, "os", c.OperatingSystem, "Operating System (Linux/Windows)")
@@ -68,11 +74,18 @@ func installFlags(flags *pflag.FlagSet, c *Opts) {
 	flags.StringVar(&c.MetricsAddr, "metrics-addr", c.MetricsAddr, "address to listen for metrics/stats requests")
 
 	flags.StringVar(&c.TaintKey, "taint", c.TaintKey, "Set node taint key")
+
 	flags.BoolVar(&c.DisableTaint, "disable-taint", c.DisableTaint, "disable the virtual-kubelet node taint")
+	/* #nosec */
 	flags.MarkDeprecated("taint", "Taint key should now be configured using the VK_TAINT_KEY environment variable") //nolint:errcheck
 
 	flags.IntVar(&c.PodSyncWorkers, "pod-sync-workers", c.PodSyncWorkers, `set the number of pod synchronization workers`)
+
 	flags.BoolVar(&c.EnableNodeLease, "enable-node-lease", c.EnableNodeLease, `use node leases (1.13) for node heartbeats`)
+	/* #nosec */
+	flags.MarkDeprecated("enable-node-lease", "leases are always enabled") //nolint:errcheck
+	/* #nosec */
+	flags.MarkHidden("enable-node-lease") //nolint:errcheck
 
 	flags.StringSliceVar(&c.TraceExporters, "trace-exporter", c.TraceExporters, fmt.Sprintf("sets the tracing exporter to use, available exporters: %s", AvailableTraceExporters()))
 	flags.StringVar(&c.TraceConfig.ServiceName, "trace-service-name", c.TraceConfig.ServiceName, "sets the name of the service used to register with the trace exporter")
@@ -81,6 +94,11 @@ func installFlags(flags *pflag.FlagSet, c *Opts) {
 
 	flags.DurationVar(&c.InformerResyncPeriod, "full-resync-period", c.InformerResyncPeriod, "how often to perform a full resync of pods between kubernetes and the provider")
 	flags.DurationVar(&c.StartupTimeout, "startup-timeout", c.StartupTimeout, "How long to wait for the virtual-kubelet to start")
+	flags.DurationVar(&c.StreamIdleTimeout, "stream-idle-timeout", c.StreamIdleTimeout,
+		"stream-idle-timeout is the maximum time a streaming connection can be idle before the connection is"+
+			" automatically closed, default 30s.")
+	flags.DurationVar(&c.StreamCreationTimeout, "stream-creation-timeout", c.StreamCreationTimeout,
+		"stream-creation-timeout is the maximum time for streaming connection, default 30s.")
 
 	flagset := flag.NewFlagSet("klog", flag.PanicOnError)
 	klog.InitFlags(flagset)

cmd/virtual-kubelet/internal/commands/root/http.go

@@ -15,147 +15,32 @@
 package root
 
 import (
-	"context"
-	"crypto/tls"
 	"fmt"
-	"io"
-	"net"
-	"net/http"
 	"os"
-
+	"time"
-	"github.com/pkg/errors"
-	"github.com/virtual-kubelet/virtual-kubelet/cmd/virtual-kubelet/internal/provider"
-	"github.com/virtual-kubelet/virtual-kubelet/log"
-	"github.com/virtual-kubelet/virtual-kubelet/node/api"
 )
 
-// AcceptedCiphers is the list of accepted TLS ciphers, with known weak ciphers elided
-// Note this list should be a moving target.
-var AcceptedCiphers = []uint16{
-	tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
-	tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
-	tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
-	tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
-
-	tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
-	tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
-	tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
-	tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
-}
-
-func loadTLSConfig(certPath, keyPath string) (*tls.Config, error) {
-	cert, err := tls.LoadX509KeyPair(certPath, keyPath)
-	if err != nil {
-		return nil, errors.Wrap(err, "error loading tls certs")
-	}
-
-	return &tls.Config{
-		Certificates:             []tls.Certificate{cert},
-		MinVersion:               tls.VersionTLS12,
-		PreferServerCipherSuites: true,
-		CipherSuites:             AcceptedCiphers,
-	}, nil
-}
-
-func setupHTTPServer(ctx context.Context, p provider.Provider, cfg *apiServerConfig) (_ func(), retErr error) {
-	var closers []io.Closer
-	cancel := func() {
-		for _, c := range closers {
-			c.Close()
-		}
-	}
-	defer func() {
-		if retErr != nil {
-			cancel()
-		}
-	}()
-
-	if cfg.CertPath == "" || cfg.KeyPath == "" {
-		log.G(ctx).
-			WithField("certPath", cfg.CertPath).
-			WithField("keyPath", cfg.KeyPath).
-			Error("TLS certificates not provided, not setting up pod http server")
-	} else {
-		tlsCfg, err := loadTLSConfig(cfg.CertPath, cfg.KeyPath)
-		if err != nil {
-			return nil, err
-		}
-		l, err := tls.Listen("tcp", cfg.Addr, tlsCfg)
-		if err != nil {
-			return nil, errors.Wrap(err, "error setting up listener for pod http server")
-		}
-
-		mux := http.NewServeMux()
-
-		podRoutes := api.PodHandlerConfig{
-			RunInContainer:   p.RunInContainer,
-			GetContainerLogs: p.GetContainerLogs,
-			GetPods:          p.GetPods,
-		}
-		api.AttachPodRoutes(podRoutes, mux, true)
-
-		s := &http.Server{
-			Handler:   mux,
-			TLSConfig: tlsCfg,
-		}
-		go serveHTTP(ctx, s, l, "pods")
-		closers = append(closers, s)
-	}
-
-	if cfg.MetricsAddr == "" {
-		log.G(ctx).Info("Pod metrics server not setup due to empty metrics address")
-	} else {
-		l, err := net.Listen("tcp", cfg.MetricsAddr)
-		if err != nil {
-			return nil, errors.Wrap(err, "could not setup listener for pod metrics http server")
-		}
-
-		mux := http.NewServeMux()
-
-		var summaryHandlerFunc api.PodStatsSummaryHandlerFunc
-		if mp, ok := p.(provider.PodMetricsProvider); ok {
-			summaryHandlerFunc = mp.GetStatsSummary
-		}
-		podMetricsRoutes := api.PodMetricsConfig{
-			GetStatsSummary: summaryHandlerFunc,
-		}
-		api.AttachPodMetricsRoutes(podMetricsRoutes, mux)
-		s := &http.Server{
-			Handler: mux,
-		}
-		go serveHTTP(ctx, s, l, "pod metrics")
-		closers = append(closers, s)
-	}
-
-	return cancel, nil
-}
-
-func serveHTTP(ctx context.Context, s *http.Server, l net.Listener, name string) {
-	if err := s.Serve(l); err != nil {
-		select {
-		case <-ctx.Done():
-		default:
-			log.G(ctx).WithError(err).Errorf("Error setting up %s http server", name)
-		}
-	}
-	l.Close()
-}
-
 type apiServerConfig struct {
-	CertPath    string
+	CertPath              string
-	KeyPath     string
+	KeyPath               string
-	Addr        string
+	CACertPath            string
-	MetricsAddr string
+	Addr                  string
+	MetricsAddr           string
+	StreamIdleTimeout     time.Duration
+	StreamCreationTimeout time.Duration
 }
 
 func getAPIConfig(c Opts) (*apiServerConfig, error) {
 	config := apiServerConfig{
-		CertPath: os.Getenv("APISERVER_CERT_LOCATION"),
+		CertPath:   os.Getenv("APISERVER_CERT_LOCATION"),
-		KeyPath:  os.Getenv("APISERVER_KEY_LOCATION"),
+		KeyPath:    os.Getenv("APISERVER_KEY_LOCATION"),
+		CACertPath: os.Getenv("APISERVER_CA_CERT_LOCATION"),
 	}
 
 	config.Addr = fmt.Sprintf(":%d", c.ListenPort)
 	config.MetricsAddr = c.MetricsAddr
+	config.StreamIdleTimeout = c.StreamIdleTimeout
+	config.StreamCreationTimeout = c.StreamCreationTimeout
 
 	return &config, nil
 }

cmd/virtual-kubelet/internal/commands/root/node.go

@@ -15,54 +15,10 @@
 package root
 
 import (
-	"context"
-	"strings"
-
-	"github.com/virtual-kubelet/virtual-kubelet/cmd/virtual-kubelet/internal/provider"
 	"github.com/virtual-kubelet/virtual-kubelet/errdefs"
 	corev1 "k8s.io/api/core/v1"
-	v1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-const osLabel = "beta.kubernetes.io/os"
-
-// NodeFromProvider builds a kubernetes node object from a provider
-// This is a temporary solution until node stuff actually split off from the provider interface itself.
-func NodeFromProvider(ctx context.Context, name string, taint *v1.Taint, p provider.Provider, version string) *v1.Node {
-	taints := make([]v1.Taint, 0)
-
-	if taint != nil {
-		taints = append(taints, *taint)
-	}
-
-	node := &v1.Node{
-		ObjectMeta: metav1.ObjectMeta{
-			Name: name,
-			Labels: map[string]string{
-				"type":                   "virtual-kubelet",
-				"kubernetes.io/role":     "agent",
-				"kubernetes.io/hostname": name,
-			},
-		},
-		Spec: v1.NodeSpec{
-			Taints: taints,
-		},
-		Status: v1.NodeStatus{
-			NodeInfo: v1.NodeSystemInfo{
-				Architecture:   "amd64",
-				KubeletVersion: version,
-			},
-		},
-	}
-
-	p.ConfigureNode(ctx, node)
-	if _, ok := node.ObjectMeta.Labels[osLabel]; !ok {
-		node.ObjectMeta.Labels[osLabel] = strings.ToLower(node.Status.NodeInfo.OperatingSystem)
-	}
-	return node
-}
-
 // getTaint creates a taint using the provided key/value.
 // Taint effect is read from the environment
 // The taint key/value may be overwritten by the environment.
@@ -80,7 +36,7 @@ func getTaint(c Opts) (*corev1.Taint, error) {
 
 	key = getEnv("VKUBELET_TAINT_KEY", key)
 	value = getEnv("VKUBELET_TAINT_VALUE", value)
-	effectEnv := getEnv("VKUBELET_TAINT_EFFECT", string(c.TaintEffect))
+	effectEnv := getEnv("VKUBELET_TAINT_EFFECT", c.TaintEffect)
 
 	var effect corev1.TaintEffect
 	switch effectEnv {

cmd/virtual-kubelet/internal/commands/root/opts.go

@@ -15,6 +15,7 @@
 package root
 
 import (
+	"fmt"
 	"os"
 	"path/filepath"
 	"strconv"
@@ -28,7 +29,7 @@ import (
 // Defaults for root command options
 const (
 	DefaultNodeName             = "virtual-kubelet"
-	DefaultOperatingSystem      = "Linux"
+	DefaultOperatingSystem      = "linux"
 	DefaultInformerResyncPeriod = 1 * time.Minute
 	DefaultMetricsAddr          = ":10255"
 	DefaultListenPort           = 10250 // TODO(cpuguy83)(VK1.0): Change this to an addr instead of just a port.. we should not be listening on all interfaces.
@@ -36,8 +37,10 @@ const (
 	DefaultKubeNamespace        = corev1.NamespaceAll
 	DefaultKubeClusterDomain    = "cluster.local"
 
-	DefaultTaintEffect = string(corev1.TaintEffectNoSchedule)
+	DefaultTaintEffect           = string(corev1.TaintEffectNoSchedule)
-	DefaultTaintKey    = "virtual-kubelet.io/provider"
+	DefaultTaintKey              = "virtual-kubelet.io/provider"
+	DefaultStreamIdleTimeout     = 30 * time.Second
+	DefaultStreamCreationTimeout = 30 * time.Second
 )
 
 // Opts stores all the options for configuring the root virtual-kubelet command.
@@ -84,10 +87,17 @@ type Opts struct {
 
 	// Startup Timeout is how long to wait for the kubelet to start
 	StartupTimeout time.Duration
+	// StreamIdleTimeout is the maximum time a streaming connection
+	// can be idle before the connection is automatically closed.
+	StreamIdleTimeout time.Duration
+	// StreamCreationTimeout is the maximum time for streaming connection
+	StreamCreationTimeout time.Duration
 
 	Version string
 }
 
+const maxInt32 = 1<<31 - 1
+
 // SetDefaultOpts sets default options for unset values on the passed in option struct.
 // Fields tht are already set will not be modified.
 func SetDefaultOpts(c *Opts) error {
@@ -121,6 +131,10 @@ func SetDefaultOpts(c *Opts) error {
 			if err != nil {
 				return errors.Wrap(err, "error parsing KUBELET_PORT environment variable")
 			}
+			if p > maxInt32 {
+				return fmt.Errorf("KUBELET_PORT environment variable is too large")
+			}
+			/* #nosec */
 			c.ListenPort = int32(p)
 		} else {
 			c.ListenPort = DefaultListenPort
@@ -152,5 +166,13 @@ func SetDefaultOpts(c *Opts) error {
 		}
 	}
 
+	if c.StreamIdleTimeout == 0 {
+		c.StreamIdleTimeout = DefaultStreamIdleTimeout
+	}
+
+	if c.StreamCreationTimeout == 0 {
+		c.StreamCreationTimeout = DefaultStreamCreationTimeout
+	}
+
 	return nil
 }

cmd/virtual-kubelet/internal/commands/root/root.go

@@ -16,8 +16,10 @@ package root
 
 import (
 	"context"
+	"crypto/tls"
+	"net/http"
 	"os"
-	"path"
+	"runtime"
 
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
@@ -26,18 +28,10 @@ import (
 	"github.com/virtual-kubelet/virtual-kubelet/internal/manager"
 	"github.com/virtual-kubelet/virtual-kubelet/log"
 	"github.com/virtual-kubelet/virtual-kubelet/node"
+	"github.com/virtual-kubelet/virtual-kubelet/node/api"
+	"github.com/virtual-kubelet/virtual-kubelet/node/nodeutil"
 	corev1 "k8s.io/api/core/v1"
-	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apiserver/pkg/server/dynamiccertificates"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/fields"
-	kubeinformers "k8s.io/client-go/informers"
-	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/kubernetes/scheme"
-	"k8s.io/client-go/kubernetes/typed/coordination/v1beta1"
-	corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
-	"k8s.io/client-go/rest"
-	"k8s.io/client-go/tools/clientcmd"
-	"k8s.io/client-go/tools/record"
 )
 
 // NewCommand creates a new top-level command.
@@ -79,31 +73,40 @@ func runRootCommand(ctx context.Context, s *provider.Store, c Opts) error {
 		}
 	}
 
-	client, err := newClient(c.KubeConfigPath)
+	// Ensure API client.
+	clientSet, err := nodeutil.ClientsetFromEnv(c.KubeConfigPath)
 	if err != nil {
 		return err
 	}
 
-	// Create a shared informer factory for Kubernetes pods in the current namespace (if specified) and scheduled to the current node.
+	// Set-up the node provider.
-	podInformerFactory := kubeinformers.NewSharedInformerFactoryWithOptions(
+	mux := http.NewServeMux()
-		client,
+	newProvider := func(cfg nodeutil.ProviderConfig) (nodeutil.Provider, node.NodeProvider, error) {
-		c.InformerResyncPeriod,
+		rm, err := manager.NewResourceManager(cfg.Pods, cfg.Secrets, cfg.ConfigMaps, cfg.Services)
-		kubeinformers.WithNamespace(c.KubeNamespace),
+		if err != nil {
-		kubeinformers.WithTweakListOptions(func(options *metav1.ListOptions) {
+			return nil, nil, errors.Wrap(err, "could not create resource manager")
-			options.FieldSelector = fields.OneTermEqualSelector("spec.nodeName", c.NodeName).String()
+		}
-		}))
+		initConfig := provider.InitConfig{
-	podInformer := podInformerFactory.Core().V1().Pods()
+			ConfigPath:        c.ProviderConfigPath,
+			NodeName:          c.NodeName,
+			OperatingSystem:   c.OperatingSystem,
+			ResourceManager:   rm,
+			DaemonPort:        c.ListenPort,
+			InternalIP:        os.Getenv("VKUBELET_POD_IP"),
+			KubeClusterDomain: c.KubeClusterDomain,
+		}
+		pInit := s.Get(c.Provider)
+		if pInit == nil {
+			return nil, nil, errors.Errorf("provider %q not found", c.Provider)
+		}
 
-	// Create another shared informer factory for Kubernetes secrets and configmaps (not subject to any selectors).
+		p, err := pInit(initConfig)
-	scmInformerFactory := kubeinformers.NewSharedInformerFactoryWithOptions(client, c.InformerResyncPeriod)
+		if err != nil {
-	// Create a secret informer and a config map informer so we can pass their listers to the resource manager.
+			return nil, nil, errors.Wrapf(err, "error initializing provider %s", c.Provider)
-	secretInformer := scmInformerFactory.Core().V1().Secrets()
+		}
-	configMapInformer := scmInformerFactory.Core().V1().ConfigMaps()
+		p.ConfigureNode(ctx, cfg.Node)
-	serviceInformer := scmInformerFactory.Core().V1().Services()
+		cfg.Node.Status.NodeInfo.KubeletVersion = c.Version
-
+		return p, nil, nil
-	rm, err := manager.NewResourceManager(podInformer.Lister(), secretInformer.Lister(), configMapInformer.Lister(), serviceInformer.Lister())
-	if err != nil {
-		return errors.Wrap(err, "could not create resource manager")
 	}
 
 	apiConfig, err := getAPIConfig(c)
@@ -111,28 +114,40 @@ func runRootCommand(ctx context.Context, s *provider.Store, c Opts) error {
 		return err
 	}
 
-	if err := setupTracing(ctx, c); err != nil {
+	cm, err := nodeutil.NewNode(c.NodeName, newProvider, func(cfg *nodeutil.NodeConfig) error {
+		cfg.KubeconfigPath = c.KubeConfigPath
+		cfg.Handler = mux
+		cfg.InformerResyncPeriod = c.InformerResyncPeriod
+
+		if taint != nil {
+			cfg.NodeSpec.Spec.Taints = append(cfg.NodeSpec.Spec.Taints, *taint)
+		}
+		cfg.NodeSpec.Status.NodeInfo.Architecture = runtime.GOARCH
+		cfg.NodeSpec.Status.NodeInfo.OperatingSystem = c.OperatingSystem
+
+		cfg.HTTPListenAddr = apiConfig.Addr
+		cfg.StreamCreationTimeout = apiConfig.StreamCreationTimeout
+		cfg.StreamIdleTimeout = apiConfig.StreamIdleTimeout
+		cfg.DebugHTTP = true
+
+		cfg.NumWorkers = c.PodSyncWorkers
+
+		return nil
+	},
+		nodeutil.WithClient(clientSet),
+		setAuth(c.NodeName, apiConfig),
+		nodeutil.WithTLSConfig(
+			nodeutil.WithKeyPairFromPath(apiConfig.CertPath, apiConfig.KeyPath),
+			maybeCA(apiConfig.CACertPath),
+		),
+		nodeutil.AttachProviderRoutes(mux),
+	)
+	if err != nil {
 		return err
 	}
 
-	initConfig := provider.InitConfig{
+	if err := setupTracing(ctx, c); err != nil {
-		ConfigPath:        c.ProviderConfigPath,
+		return err
-		NodeName:          c.NodeName,
-		OperatingSystem:   c.OperatingSystem,
-		ResourceManager:   rm,
-		DaemonPort:        int32(c.ListenPort),
-		InternalIP:        os.Getenv("VKUBELET_POD_IP"),
-		KubeClusterDomain: c.KubeClusterDomain,
-	}
-
-	pInit := s.Get(c.Provider)
-	if pInit == nil {
-		return errors.Errorf("provider %q not found", c.Provider)
-	}
-
-	p, err := pInit(initConfig)
-	if err != nil {
-		return errors.Wrapf(err, "error initializing provider %s", c.Provider)
 	}
 
 	ctx = log.WithLogger(ctx, log.G(ctx).WithFields(log.Fields{
@@ -142,117 +157,54 @@ func runRootCommand(ctx context.Context, s *provider.Store, c Opts) error {
 		"watchedNamespace": c.KubeNamespace,
 	}))
 
-	var leaseClient v1beta1.LeaseInterface
+	go cm.Run(ctx) //nolint:errcheck
-	if c.EnableNodeLease {
-		leaseClient = client.CoordinationV1beta1().Leases(corev1.NamespaceNodeLease)
-	}
 
-	pNode := NodeFromProvider(ctx, c.NodeName, taint, p, c.Version)
+	defer func() {
-	nodeRunner, err := node.NewNodeController(
+		log.G(ctx).Debug("Waiting for controllers to be done")
-		node.NaiveNodeProvider{},
+		cancel()
-		pNode,
+		<-cm.Done()
-		client.CoreV1().Nodes(),
+	}()
-		node.WithNodeEnableLeaseV1Beta1(leaseClient, nil),
-		node.WithNodeStatusUpdateErrorHandler(func(ctx context.Context, err error) error {
-			if !k8serrors.IsNotFound(err) {
-				return err
-			}
 
-			log.G(ctx).Debug("node not found")
+	log.G(ctx).Info("Waiting for controller to be ready")
-			newNode := pNode.DeepCopy()
+	if err := cm.WaitReady(ctx, c.StartupTimeout); err != nil {
-			newNode.ResourceVersion = ""
-			_, err = client.CoreV1().Nodes().Create(newNode)
-			if err != nil {
-				return err
-			}
-			log.G(ctx).Debug("created new node")
-			return nil
-		}),
-	)
-	if err != nil {
-		log.G(ctx).Fatal(err)
-	}
-
-	eb := record.NewBroadcaster()
-	eb.StartLogging(log.G(ctx).Infof)
-	eb.StartRecordingToSink(&corev1client.EventSinkImpl{Interface: client.CoreV1().Events(c.KubeNamespace)})
-
-	pc, err := node.NewPodController(node.PodControllerConfig{
-		PodClient:         client.CoreV1(),
-		PodInformer:       podInformer,
-		EventRecorder:     eb.NewRecorder(scheme.Scheme, corev1.EventSource{Component: path.Join(pNode.Name, "pod-controller")}),
-		Provider:          p,
-		SecretInformer:    secretInformer,
-		ConfigMapInformer: configMapInformer,
-		ServiceInformer:   serviceInformer,
-	})
-	if err != nil {
-		return errors.Wrap(err, "error setting up pod controller")
-	}
-
-	go podInformerFactory.Start(ctx.Done())
-	go scmInformerFactory.Start(ctx.Done())
-
-	cancelHTTP, err := setupHTTPServer(ctx, p, apiConfig)
-	if err != nil {
 		return err
 	}
-	defer cancelHTTP()
 
-	go func() {
+	log.G(ctx).Info("Ready")
-		if err := pc.Run(ctx, c.PodSyncWorkers); err != nil && errors.Cause(err) != context.Canceled {
-			log.G(ctx).Fatal(err)
-		}
-	}()
 
-	if c.StartupTimeout > 0 {
+	select {
-		ctx, cancel := context.WithTimeout(ctx, c.StartupTimeout)
+	case <-ctx.Done():
-		log.G(ctx).Info("Waiting for pod controller / VK to be ready")
+	case <-cm.Done():
-		select {
+		return cm.Err()
-		case <-ctx.Done():
-			cancel()
-			return ctx.Err()
-		case <-pc.Ready():
-		}
-		cancel()
-		if err := pc.Err(); err != nil {
-			return err
-		}
 	}
-
-	go func() {
-		if err := nodeRunner.Run(ctx); err != nil {
-			log.G(ctx).Fatal(err)
-		}
-	}()
-
-	log.G(ctx).Info("Initialized")
-
-	<-ctx.Done()
 	return nil
 }
 
-func newClient(configPath string) (*kubernetes.Clientset, error) {
+func setAuth(node string, apiCfg *apiServerConfig) nodeutil.NodeOpt {
-	var config *rest.Config
+	if apiCfg.CACertPath == "" {
-
+		return func(cfg *nodeutil.NodeConfig) error {
-	// Check if the kubeConfig file exists.
+			cfg.Handler = api.InstrumentHandler(nodeutil.WithAuth(nodeutil.NoAuth(), cfg.Handler))
-	if _, err := os.Stat(configPath); !os.IsNotExist(err) {
+			return nil
-		// Get the kubeconfig from the filepath.
-		config, err = clientcmd.BuildConfigFromFlags("", configPath)
-		if err != nil {
-			return nil, errors.Wrap(err, "error building client config")
-		}
-	} else {
-		// Set to in-cluster config.
-		config, err = rest.InClusterConfig()
-		if err != nil {
-			return nil, errors.Wrap(err, "error building in cluster config")
 		}
 	}
 
-	if masterURI := os.Getenv("MASTER_URI"); masterURI != "" {
+	return func(cfg *nodeutil.NodeConfig) error {
-		config.Host = masterURI
+		auth, err := nodeutil.WebhookAuth(cfg.Client, node, func(cfg *nodeutil.WebhookAuthConfig) error {
+			var err error
+			cfg.AuthnConfig.ClientCertificateCAContentProvider, err = dynamiccertificates.NewDynamicCAContentFromFile("ca-cert-bundle", apiCfg.CACertPath)
+			return err
+		})
+		if err != nil {
+			return err
+		}
+		cfg.Handler = api.InstrumentHandler(nodeutil.WithAuth(auth, cfg.Handler))
+		return nil
 	}
-
+}
-	return kubernetes.NewForConfig(config)
+
+func maybeCA(p string) func(*tls.Config) error {
+	if p == "" {
+		return func(*tls.Config) error { return nil }
+	}
+	return nodeutil.WithCAFromPath(p)
 }

cmd/virtual-kubelet/internal/commands/root/tracing.go

@@ -21,6 +21,7 @@ import (
 	"os"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/pkg/errors"
 	"github.com/virtual-kubelet/virtual-kubelet/errdefs"
@@ -105,7 +106,8 @@ func setupZpages(ctx context.Context) {
 	zpages.Handle(mux, "/debug")
 	go func() {
 		// This should never terminate, if it does, it will always terminate with an error
-		e := http.Serve(listener, mux)
+		srv := &http.Server{Handler: mux, ReadHeaderTimeout: 30 * time.Second}
+		e := srv.Serve(listener)
 		if e == http.ErrServerClosed {
 			return
 		}

cmd/virtual-kubelet/internal/commands/root/tracing_register.go

@@ -19,7 +19,8 @@ import (
 	"go.opencensus.io/trace"
 )
 
-type TracingExporterOptions struct {
+// TracingExporterOptions is the options passed to the tracing exporter init function.
+type TracingExporterOptions struct { //nolint: golint
 	Tags        map[string]string
 	ServiceName string
 }
@@ -29,7 +30,7 @@ var (
 )
 
 // TracingExporterInitFunc is the function that is called to initialize an exporter.
-// This is used when registering an exporter and called when a user specifed they want to use the exporter.
+// This is used when registering an exporter and called when a user specified they want to use the exporter.
 type TracingExporterInitFunc func(TracingExporterOptions) (trace.Exporter, error)
 
 // RegisterTracingExporter registers a tracing exporter.

cmd/virtual-kubelet/internal/commands/root/tracing_register_jaeger.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build !no_jaeger_exporter
 // +build !no_jaeger_exporter
 
 package root
@@ -31,17 +32,17 @@ func init() {
 // NewJaegerExporter creates a new opencensus tracing exporter.
 func NewJaegerExporter(opts TracingExporterOptions) (trace.Exporter, error) {
 	jOpts := jaeger.Options{
-		Endpoint:      os.Getenv("JAEGER_ENDPOINT"),
+		CollectorEndpoint: os.Getenv("JAEGER_COLLECTOR_ENDPOINT"),
-		AgentEndpoint: os.Getenv("JAEGER_AGENT_ENDPOINT"),
+		AgentEndpoint:     os.Getenv("JAEGER_AGENT_ENDPOINT"),
-		Username:      os.Getenv("JAEGER_USER"),
+		Username:          os.Getenv("JAEGER_USER"),
-		Password:      os.Getenv("JAEGER_PASSWORD"),
+		Password:          os.Getenv("JAEGER_PASSWORD"),
 		Process: jaeger.Process{
 			ServiceName: opts.ServiceName,
 		},
 	}
 
-	if jOpts.Endpoint == "" && jOpts.AgentEndpoint == "" {
+	if jOpts.CollectorEndpoint == "" && jOpts.AgentEndpoint == "" { // nolintlint:staticcheck
-		return nil, errors.New("Must specify either JAEGER_ENDPOINT or JAEGER_AGENT_ENDPOINT")
+		return nil, errors.New("must specify either JAEGER_COLLECTOR_ENDPOINT or JAEGER_AGENT_ENDPOINT")
 	}
 
 	for k, v := range opts.Tags {

cmd/virtual-kubelet/internal/commands/root/tracing_register_ocagent.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build !no_ocagent_exporter
 // +build !no_ocagent_exporter
 
 package root

cmd/virtual-kubelet/internal/provider/mock/mock.go

@@ -5,19 +5,20 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"math/rand"
+	"os"
 	"strings"
 	"time"
 
+	dto "github.com/prometheus/client_model/go"
 	"github.com/virtual-kubelet/virtual-kubelet/errdefs"
 	"github.com/virtual-kubelet/virtual-kubelet/log"
 	"github.com/virtual-kubelet/virtual-kubelet/node/api"
+	stats "github.com/virtual-kubelet/virtual-kubelet/node/api/statsv1alpha1"
 	"github.com/virtual-kubelet/virtual-kubelet/trace"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	stats "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
 )
 
 const (
@@ -41,8 +42,8 @@ var (
 )
 */
 
-// MockV0Provider implements the virtual-kubelet provider interface and stores pods in memory.
+// MockProvider implements the virtual-kubelet provider interface and stores pods in memory.
-type MockV0Provider struct { //nolint:golint
+type MockProvider struct { //nolint:golint
 	nodeName           string
 	operatingSystem    string
 	internalIP         string
@@ -53,21 +54,18 @@ type MockV0Provider struct { //nolint:golint
 	notifier           func(*v1.Pod)
 }
 
-// MockProvider is like MockV0Provider, but implements the PodNotifier interface
-type MockProvider struct { //nolint:golint
-	*MockV0Provider
-}
-
 // MockConfig contains a mock virtual-kubelet's configurable parameters.
 type MockConfig struct { //nolint:golint
-	CPU    string `json:"cpu,omitempty"`
+	CPU        string            `json:"cpu,omitempty"`
-	Memory string `json:"memory,omitempty"`
+	Memory     string            `json:"memory,omitempty"`
-	Pods   string `json:"pods,omitempty"`
+	Pods       string            `json:"pods,omitempty"`
+	Others     map[string]string `json:"others,omitempty"`
+	ProviderID string            `json:"providerID,omitempty"`
 }
 
 // NewMockProviderMockConfig creates a new MockV0Provider. Mock legacy provider does not implement the new asynchronous podnotifier interface
-func NewMockV0ProviderMockConfig(config MockConfig, nodeName, operatingSystem string, internalIP string, daemonEndpointPort int32) (*MockV0Provider, error) {
+func NewMockProviderMockConfig(config MockConfig, nodeName, operatingSystem string, internalIP string, daemonEndpointPort int32) (*MockProvider, error) {
-	//set defaults
+	// set defaults
 	if config.CPU == "" {
 		config.CPU = defaultCPUCapacity
 	}
@@ -77,7 +75,7 @@ func NewMockV0ProviderMockConfig(config MockConfig, nodeName, operatingSystem st
 	if config.Pods == "" {
 		config.Pods = defaultPodCapacity
 	}
-	provider := MockV0Provider{
+	provider := MockProvider{
 		nodeName:           nodeName,
 		operatingSystem:    operatingSystem,
 		internalIP:         internalIP,
@@ -85,32 +83,11 @@ func NewMockV0ProviderMockConfig(config MockConfig, nodeName, operatingSystem st
 		pods:               make(map[string]*v1.Pod),
 		config:             config,
 		startTime:          time.Now(),
-		// By default notifier is set to a function which is a no-op. In the event we've implemented the PodNotifier interface,
-		// it will be set, and then we'll call a real underlying implementation.
-		// This makes it easier in the sense we don't need to wrap each method.
-		notifier: func(*v1.Pod) {},
 	}
 
 	return &provider, nil
 }
 
-// NewMockV0Provider creates a new MockV0Provider
-func NewMockV0Provider(providerConfig, nodeName, operatingSystem string, internalIP string, daemonEndpointPort int32) (*MockV0Provider, error) {
-	config, err := loadConfig(providerConfig, nodeName)
-	if err != nil {
-		return nil, err
-	}
-
-	return NewMockV0ProviderMockConfig(config, nodeName, operatingSystem, internalIP, daemonEndpointPort)
-}
-
-// NewMockProviderMockConfig creates a new MockProvider with the given config
-func NewMockProviderMockConfig(config MockConfig, nodeName, operatingSystem string, internalIP string, daemonEndpointPort int32) (*MockProvider, error) {
-	p, err := NewMockV0ProviderMockConfig(config, nodeName, operatingSystem, internalIP, daemonEndpointPort)
-
-	return &MockProvider{MockV0Provider: p}, err
-}
-
 // NewMockProvider creates a new MockProvider, which implements the PodNotifier interface
 func NewMockProvider(providerConfig, nodeName, operatingSystem string, internalIP string, daemonEndpointPort int32) (*MockProvider, error) {
 	config, err := loadConfig(providerConfig, nodeName)
@@ -123,7 +100,7 @@ func NewMockProvider(providerConfig, nodeName, operatingSystem string, internalI
 
 // loadConfig loads the given json configuration files.
 func loadConfig(providerConfig, nodeName string) (config MockConfig, err error) {
-	data, err := ioutil.ReadFile(providerConfig)
+	data, err := os.ReadFile(providerConfig)
 	if err != nil {
 		return config, err
 	}
@@ -154,11 +131,16 @@ func loadConfig(providerConfig, nodeName string) (config MockConfig, err error)
 	if _, err = resource.ParseQuantity(config.Pods); err != nil {
 		return config, fmt.Errorf("Invalid pods value %v", config.Pods)
 	}
+	for _, v := range config.Others {
+		if _, err = resource.ParseQuantity(v); err != nil {
+			return config, fmt.Errorf("Invalid other value %v", v)
+		}
+	}
 	return config, nil
 }
 
 // CreatePod accepts a Pod definition and stores it in memory.
-func (p *MockV0Provider) CreatePod(ctx context.Context, pod *v1.Pod) error {
+func (p *MockProvider) CreatePod(ctx context.Context, pod *v1.Pod) error {
 	ctx, span := trace.StartSpan(ctx, "CreatePod")
 	defer span.End()
 
@@ -215,7 +197,7 @@ func (p *MockV0Provider) CreatePod(ctx context.Context, pod *v1.Pod) error {
 }
 
 // UpdatePod accepts a Pod definition and updates its reference.
-func (p *MockV0Provider) UpdatePod(ctx context.Context, pod *v1.Pod) error {
+func (p *MockProvider) UpdatePod(ctx context.Context, pod *v1.Pod) error {
 	ctx, span := trace.StartSpan(ctx, "UpdatePod")
 	defer span.End()
 
@@ -236,7 +218,7 @@ func (p *MockV0Provider) UpdatePod(ctx context.Context, pod *v1.Pod) error {
 }
 
 // DeletePod deletes the specified pod out of memory.
-func (p *MockV0Provider) DeletePod(ctx context.Context, pod *v1.Pod) (err error) {
+func (p *MockProvider) DeletePod(ctx context.Context, pod *v1.Pod) (err error) {
 	ctx, span := trace.StartSpan(ctx, "DeletePod")
 	defer span.End()
 
@@ -277,7 +259,7 @@ func (p *MockV0Provider) DeletePod(ctx context.Context, pod *v1.Pod) (err error)
 }
 
 // GetPod returns a pod by name that is stored in memory.
-func (p *MockV0Provider) GetPod(ctx context.Context, namespace, name string) (pod *v1.Pod, err error) {
+func (p *MockProvider) GetPod(ctx context.Context, namespace, name string) (pod *v1.Pod, err error) {
 	ctx, span := trace.StartSpan(ctx, "GetPod")
 	defer func() {
 		span.SetStatus(err)
@@ -301,7 +283,7 @@ func (p *MockV0Provider) GetPod(ctx context.Context, namespace, name string) (po
 }
 
 // GetContainerLogs retrieves the logs of a container by name from the provider.
-func (p *MockV0Provider) GetContainerLogs(ctx context.Context, namespace, podName, containerName string, opts api.ContainerLogOpts) (io.ReadCloser, error) {
+func (p *MockProvider) GetContainerLogs(ctx context.Context, namespace, podName, containerName string, opts api.ContainerLogOpts) (io.ReadCloser, error) {
 	ctx, span := trace.StartSpan(ctx, "GetContainerLogs")
 	defer span.End()
 
@@ -309,19 +291,26 @@ func (p *MockV0Provider) GetContainerLogs(ctx context.Context, namespace, podNam
 	ctx = addAttributes(ctx, span, namespaceKey, namespace, nameKey, podName, containerNameKey, containerName)
 
 	log.G(ctx).Infof("receive GetContainerLogs %q", podName)
-	return ioutil.NopCloser(strings.NewReader("")), nil
+	return io.NopCloser(strings.NewReader("")), nil
 }
 
 // RunInContainer executes a command in a container in the pod, copying data
 // between in/out/err and the container's stdin/stdout/stderr.
-func (p *MockV0Provider) RunInContainer(ctx context.Context, namespace, name, container string, cmd []string, attach api.AttachIO) error {
+func (p *MockProvider) RunInContainer(ctx context.Context, namespace, name, container string, cmd []string, attach api.AttachIO) error {
 	log.G(context.TODO()).Infof("receive ExecInContainer %q", container)
 	return nil
 }
 
+// AttachToContainer attaches to the executing process of a container in the pod, copying data
+// between in/out/err and the container's stdin/stdout/stderr.
+func (p *MockProvider) AttachToContainer(ctx context.Context, namespace, name, container string, attach api.AttachIO) error {
+	log.G(ctx).Infof("receive AttachToContainer %q", container)
+	return nil
+}
+
 // GetPodStatus returns the status of a pod by name that is "running".
 // returns nil if a pod by that name is not found.
-func (p *MockV0Provider) GetPodStatus(ctx context.Context, namespace, name string) (*v1.PodStatus, error) {
+func (p *MockProvider) GetPodStatus(ctx context.Context, namespace, name string) (*v1.PodStatus, error) {
 	ctx, span := trace.StartSpan(ctx, "GetPodStatus")
 	defer span.End()
 
@@ -339,7 +328,7 @@ func (p *MockV0Provider) GetPodStatus(ctx context.Context, namespace, name strin
 }
 
 // GetPods returns a list of all pods known to be "running".
-func (p *MockV0Provider) GetPods(ctx context.Context) ([]*v1.Pod, error) {
+func (p *MockProvider) GetPods(ctx context.Context) ([]*v1.Pod, error) {
 	ctx, span := trace.StartSpan(ctx, "GetPods")
 	defer span.End()
 
@@ -354,10 +343,13 @@ func (p *MockV0Provider) GetPods(ctx context.Context) ([]*v1.Pod, error) {
 	return pods, nil
 }
 
-func (p *MockV0Provider) ConfigureNode(ctx context.Context, n *v1.Node) {
+func (p *MockProvider) ConfigureNode(ctx context.Context, n *v1.Node) { //nolint:golint
-	ctx, span := trace.StartSpan(ctx, "mock.ConfigureNode") //nolint:ineffassign
+	ctx, span := trace.StartSpan(ctx, "mock.ConfigureNode") //nolint:staticcheck,ineffassign
 	defer span.End()
 
+	if p.config.ProviderID != "" {
+		n.Spec.ProviderID = p.config.ProviderID
+	}
 	n.Status.Capacity = p.capacity()
 	n.Status.Allocatable = p.capacity()
 	n.Status.Conditions = p.nodeConditions()
@@ -365,34 +357,39 @@ func (p *MockV0Provider) ConfigureNode(ctx context.Context, n *v1.Node) {
 	n.Status.DaemonEndpoints = p.nodeDaemonEndpoints()
 	os := p.operatingSystem
 	if os == "" {
-		os = "Linux"
+		os = "linux"
 	}
 	n.Status.NodeInfo.OperatingSystem = os
 	n.Status.NodeInfo.Architecture = "amd64"
 	n.ObjectMeta.Labels["alpha.service-controller.kubernetes.io/exclude-balancer"] = "true"
+	n.ObjectMeta.Labels["node.kubernetes.io/exclude-from-external-load-balancers"] = "true"
 }
 
 // Capacity returns a resource list containing the capacity limits.
-func (p *MockV0Provider) capacity() v1.ResourceList {
+func (p *MockProvider) capacity() v1.ResourceList {
-	return v1.ResourceList{
+	rl := v1.ResourceList{
 		"cpu":    resource.MustParse(p.config.CPU),
 		"memory": resource.MustParse(p.config.Memory),
 		"pods":   resource.MustParse(p.config.Pods),
 	}
+	for k, v := range p.config.Others {
+		rl[v1.ResourceName(k)] = resource.MustParse(v)
+	}
+	return rl
 }
 
 // NodeConditions returns a list of conditions (Ready, OutOfDisk, etc), for updates to the node status
 // within Kubernetes.
-func (p *MockV0Provider) nodeConditions() []v1.NodeCondition {
+func (p *MockProvider) nodeConditions() []v1.NodeCondition {
 	// TODO: Make this configurable
 	return []v1.NodeCondition{
 		{
 			Type:               "Ready",
-			Status:             v1.ConditionTrue,
+			Status:             v1.ConditionFalse,
 			LastHeartbeatTime:  metav1.Now(),
 			LastTransitionTime: metav1.Now(),
-			Reason:             "KubeletReady",
+			Reason:             "KubeletPending",
-			Message:            "kubelet is ready.",
+			Message:            "kubelet is pending.",
 		},
 		{
 			Type:               "OutOfDisk",
@@ -432,7 +429,7 @@ func (p *MockV0Provider) nodeConditions() []v1.NodeCondition {
 
 // NodeAddresses returns a list of addresses for the node status
 // within Kubernetes.
-func (p *MockV0Provider) nodeAddresses() []v1.NodeAddress {
+func (p *MockProvider) nodeAddresses() []v1.NodeAddress {
 	return []v1.NodeAddress{
 		{
 			Type:    "InternalIP",
@@ -443,7 +440,7 @@ func (p *MockV0Provider) nodeAddresses() []v1.NodeAddress {
 
 // NodeDaemonEndpoints returns NodeDaemonEndpoints for the node status
 // within Kubernetes.
-func (p *MockV0Provider) nodeDaemonEndpoints() v1.NodeDaemonEndpoints {
+func (p *MockProvider) nodeDaemonEndpoints() v1.NodeDaemonEndpoints {
 	return v1.NodeDaemonEndpoints{
 		KubeletEndpoint: v1.DaemonEndpoint{
 			Port: p.daemonEndpointPort,
@@ -452,9 +449,9 @@ func (p *MockV0Provider) nodeDaemonEndpoints() v1.NodeDaemonEndpoints {
 }
 
 // GetStatsSummary returns dummy stats for all pods known by this provider.
-func (p *MockV0Provider) GetStatsSummary(ctx context.Context) (*stats.Summary, error) {
+func (p *MockProvider) GetStatsSummary(ctx context.Context) (*stats.Summary, error) {
 	var span trace.Span
-	ctx, span = trace.StartSpan(ctx, "GetStatsSummary") //nolint: ineffassign
+	ctx, span = trace.StartSpan(ctx, "GetStatsSummary") //nolint: ineffassign,staticcheck
 	defer span.End()
 
 	// Grab the current timestamp so we can report it as the time the stats were generated.
@@ -492,10 +489,14 @@ func (p *MockV0Provider) GetStatsSummary(ctx context.Context) (*stats.Summary, e
 		for _, container := range pod.Spec.Containers {
 			// Grab a dummy value to be used as the total CPU usage.
 			// The value should fit a uint32 in order to avoid overflows later on when computing pod stats.
+
+			/* #nosec */
 			dummyUsageNanoCores := uint64(rand.Uint32())
 			totalUsageNanoCores += dummyUsageNanoCores
 			// Create a dummy value to be used as the total RAM usage.
 			// The value should fit a uint32 in order to avoid overflows later on when computing pod stats.
+
+			/* #nosec */
 			dummyUsageBytes := uint64(rand.Uint32())
 			totalUsageBytes += dummyUsageBytes
 			// Append a ContainerStats object containing the dummy stats to the PodStats object.
@@ -529,6 +530,129 @@ func (p *MockV0Provider) GetStatsSummary(ctx context.Context) (*stats.Summary, e
 	return res, nil
 }
 
+func (p *MockProvider) generateMockMetrics(metricsMap map[string][]*dto.Metric, resourceType string, label []*dto.LabelPair) map[string][]*dto.Metric {
+	var (
+		cpuMetricSuffix    = "_cpu_usage_seconds_total"
+		memoryMetricSuffix = "_memory_working_set_bytes"
+		dummyValue         = float64(100)
+	)
+
+	if metricsMap == nil {
+		metricsMap = map[string][]*dto.Metric{}
+	}
+
+	finalCpuMetricName := resourceType + cpuMetricSuffix
+	finalMemoryMetricName := resourceType + memoryMetricSuffix
+
+	newCPUMetric := dto.Metric{
+		Label: label,
+		Counter: &dto.Counter{
+			Value: &dummyValue,
+		},
+	}
+	newMemoryMetric := dto.Metric{
+		Label: label,
+		Gauge: &dto.Gauge{
+			Value: &dummyValue,
+		},
+	}
+	// if metric family exists add to metric array
+	if cpuMetrics, ok := metricsMap[finalCpuMetricName]; ok {
+		metricsMap[finalCpuMetricName] = append(cpuMetrics, &newCPUMetric)
+	} else {
+		metricsMap[finalCpuMetricName] = []*dto.Metric{&newCPUMetric}
+	}
+	if memoryMetrics, ok := metricsMap[finalMemoryMetricName]; ok {
+		metricsMap[finalMemoryMetricName] = append(memoryMetrics, &newMemoryMetric)
+	} else {
+		metricsMap[finalMemoryMetricName] = []*dto.Metric{&newMemoryMetric}
+	}
+
+	return metricsMap
+}
+
+func (p *MockProvider) getMetricType(metricName string) *dto.MetricType {
+	var (
+		dtoCounterMetricType = dto.MetricType_COUNTER
+		dtoGaugeMetricType   = dto.MetricType_GAUGE
+		cpuMetricSuffix      = "_cpu_usage_seconds_total"
+		memoryMetricSuffix   = "_memory_working_set_bytes"
+	)
+	if strings.HasSuffix(metricName, cpuMetricSuffix) {
+		return &dtoCounterMetricType
+	}
+	if strings.HasSuffix(metricName, memoryMetricSuffix) {
+		return &dtoGaugeMetricType
+	}
+
+	return nil
+}
+
+func (p *MockProvider) GetMetricsResource(ctx context.Context) ([]*dto.MetricFamily, error) {
+	var span trace.Span
+	ctx, span = trace.StartSpan(ctx, "GetMetricsResource") //nolint: ineffassign,staticcheck
+	defer span.End()
+
+	var (
+		nodeNameStr      = "NodeName"
+		podNameStr       = "PodName"
+		containerNameStr = "containerName"
+	)
+	nodeLabels := []*dto.LabelPair{
+		{
+			Name:  &nodeNameStr,
+			Value: &p.nodeName,
+		},
+	}
+
+	metricsMap := p.generateMockMetrics(nil, "node", nodeLabels)
+	for _, pod := range p.pods {
+		podLabels := []*dto.LabelPair{
+			{
+				Name:  &nodeNameStr,
+				Value: &p.nodeName,
+			},
+			{
+				Name:  &podNameStr,
+				Value: &pod.Name,
+			},
+		}
+		metricsMap = p.generateMockMetrics(metricsMap, "pod", podLabels)
+		for _, container := range pod.Spec.Containers {
+			containerLabels := []*dto.LabelPair{
+				{
+					Name:  &nodeNameStr,
+					Value: &p.nodeName,
+				},
+				{
+					Name:  &podNameStr,
+					Value: &pod.Name,
+				},
+				{
+					Name:  &containerNameStr,
+					Value: &container.Name,
+				},
+			}
+			metricsMap = p.generateMockMetrics(metricsMap, "container", containerLabels)
+		}
+	}
+
+	res := []*dto.MetricFamily{}
+	for metricName := range metricsMap {
+		tempName := metricName
+		tempMetrics := metricsMap[tempName]
+
+		metricFamily := dto.MetricFamily{
+			Name:   &tempName,
+			Type:   p.getMetricType(tempName),
+			Metric: tempMetrics,
+		}
+		res = append(res, &metricFamily)
+	}
+
+	return res, nil
+}
+
 // NotifyPods is called to set a pod notifier callback function. This should be called before any operations are done
 // within the provider.
 func (p *MockProvider) NotifyPods(ctx context.Context, notifier func(*v1.Pod)) {

cmd/virtual-kubelet/internal/provider/provider.go

@@ -2,35 +2,15 @@ package provider
 
 import (
 	"context"
-	"io"
 
-	"github.com/virtual-kubelet/virtual-kubelet/node"
+	"github.com/virtual-kubelet/virtual-kubelet/node/nodeutil"
-	"github.com/virtual-kubelet/virtual-kubelet/node/api"
 	v1 "k8s.io/api/core/v1"
-	stats "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
 )
 
-// Provider contains the methods required to implement a virtual-kubelet provider.
+// Provider wraps the core provider type with an extra function needed to bootstrap the node
-//
-// Errors produced by these methods should implement an interface from
-// github.com/virtual-kubelet/virtual-kubelet/errdefs package in order for the
-// core logic to be able to understand the type of failure.
 type Provider interface {
-	node.PodLifecycleHandler
+	nodeutil.Provider
-
-	// GetContainerLogs retrieves the logs of a container by name from the provider.
-	GetContainerLogs(ctx context.Context, namespace, podName, containerName string, opts api.ContainerLogOpts) (io.ReadCloser, error)
-
-	// RunInContainer executes a command in a container in the pod, copying data
-	// between in/out/err and the container's stdin/stdout/stderr.
-	RunInContainer(ctx context.Context, namespace, podName, containerName string, cmd []string, attach api.AttachIO) error
-
 	// ConfigureNode enables a provider to configure the node object that
 	// will be used for Kubernetes.
 	ConfigureNode(context.Context, *v1.Node)
 }
-
-// PodMetricsProvider is an optional interface that providers can implement to expose pod stats
-type PodMetricsProvider interface {
-	GetStatsSummary(context.Context) (*stats.Summary, error)
-}

cmd/virtual-kubelet/internal/provider/store.go

@@ -13,7 +13,7 @@ type Store struct {
 	ls map[string]InitFunc
 }
 
-func NewStore() *Store {
+func NewStore() *Store { //nolint:golint
 	return &Store{
 		ls: make(map[string]InitFunc),
 	}
@@ -71,4 +71,4 @@ type InitConfig struct {
 	ResourceManager   *manager.ResourceManager
 }
 
-type InitFunc func(InitConfig) (Provider, error)
+type InitFunc func(InitConfig) (Provider, error) //nolint:golint

cmd/virtual-kubelet/internal/provider/types.go

@@ -2,12 +2,12 @@ package provider
 
 const (
 	// OperatingSystemLinux is the configuration value for defining Linux.
-	OperatingSystemLinux = "Linux"
+	OperatingSystemLinux = "linux"
 	// OperatingSystemWindows is the configuration value for defining Windows.
-	OperatingSystemWindows = "Windows"
+	OperatingSystemWindows = "windows"
 )
 
-type OperatingSystems map[string]bool
+type OperatingSystems map[string]bool //nolint:golint
 
 var (
 	// ValidOperatingSystems defines the group of operating systems
@@ -18,7 +18,7 @@ var (
 	}
 )
 
-func (o OperatingSystems) Names() []string {
+func (o OperatingSystems) Names() []string { //nolint:golint
 	keys := make([]string, 0, len(o))
 	for k := range o {
 		keys = append(keys, k)

cmd/virtual-kubelet/register.go

@@ -6,6 +6,7 @@ import (
 )
 
 func registerMock(s *provider.Store) {
+	/* #nosec */
 	s.Register("mock", func(cfg provider.InitConfig) (provider.Provider, error) { //nolint:errcheck
 		return mock.NewMockProvider(
 			cfg.ConfigPath,
@@ -15,14 +16,4 @@ func registerMock(s *provider.Store) {
 			cfg.DaemonPort,
 		)
 	})
-
-	s.Register("mockV0", func(cfg provider.InitConfig) (provider.Provider, error) { //nolint:errcheck
-		return mock.NewMockProvider(
-			cfg.ConfigPath,
-			cfg.NodeName,
-			cfg.OperatingSystem,
-			cfg.InternalIP,
-			cfg.DaemonPort,
-		)
-	})
 }

docs/proposals/MetricsUpdateProposal.md

@@ -0,0 +1,296 @@
+#  Virtual Kubelet Metrics Update
+
+<!-- toc -->
+- [Summary](#summary)
+- [Motivation](#motivation)
+  - [Goals](#goals)
+  - [Non-Goals](#non-goals)
+- [Proposal](#proposal)
+- [Design Details](#design-details)
+  - [API](#api)
+  - [Data](#data)
+  - [Changes to the Provider](#changes-to-the-provider)
+  - [Test Plan](#test-plan)
+<!-- /toc -->
+
+## Summary
+
+Add the new /metrics/resource endpoint in the virtual-kubelet to support the metrics server update for new Kubernetes versions `>=1.24`
+
+
+## Motivation
+
+The Kubernetes metrics server now tries to get metrics from the kubelet using the new metrics endpoint [/metrics/resource](https://github.com/kubernetes-sigs/metrics-server/commit/a2d732e5cdbfd93a6ebce221e8df0e8b463eecc6#diff-6e5b914d1403a14af1cc43582a2c9af727113037a3c6a77d8729aaefba084fb5R88),
+while Virtual Kubelet is still exposing the earlier metrics endpoint [/stats/summary](https://github.com/virtual-kubelet/virtual-kubelet/blob/master/node/api/server.go#L90). 
+This causes metrics to break when using virtual kubelet with newer Kubernetes versions (>=1.24). 
+To support the new metrics server, this document proposes adding a new handler to handle the updated metrics endpoint. 
+This will be an additive update, and the old 
+[/stats/summary](https://github.com/virtual-kubelet/virtual-kubelet/blob/master/node/api/server.go#L90) endpoint will still be available to maintain backward compatibility with 
+the older metrics server version.
+
+
+### Goals
+
+- Support metrics for kubernetes version `>=1.24` through adding /metrics/resource endpoint handler.
+
+### Non-Goals
+
+- Ensure pod autoscaling works as expected with the newer kubernetes versions `>=1.24` as expected
+
+## Proposal
+
+Add a new handler for `/metrics/resource` endpoint that calls a new `GetMetricsResource` method in the provider,
+which in-turn returns metrics using the prometheus `model.Samples` data structure as expected by the new metrics server.
+The provider will need to implement the `GetMetricsResource` method in order to add support for the new `/metrics/resource` endpoint with Kubernetes version >=1.24
+
+
+## Design Details
+Currently the virtual kubelet code uses the `PodStatsSummaryHandler` method to set up a http handler for serving pod metrics via the `/stats/summary` endpoint. 
+To support the updated metrics server, we need to add another handler `PodMetricsResourceHandler`  which can serve metrics via the `/metrics/resource` endpoint. 
+The `PodMetricsResourceHandler` calls the new `GetMetricsResource` method of the provider to get the metrics from the specific provider. 
+
+### API
+Add `GetMetricsResource` to `PodHandlerConfig`
+```go
+type PodHandlerConfig struct { //nolint:golint
+	RunInContainer   ContainerExecHandlerFunc
+	GetContainerLogs ContainerLogsHandlerFunc
+	// GetPods is meant to enumerate the pods that the provider knows about
+	GetPods PodListerFunc
+	// GetPodsFromKubernetes is meant to enumerate the pods that the node is meant to be running
+	GetPodsFromKubernetes PodListerFunc
+	GetStatsSummary       PodStatsSummaryHandlerFunc
+	GetMetricsResource    PodMetricsResourceHandlerFunc
+	StreamIdleTimeout     time.Duration
+	StreamCreationTimeout time.Duration
+}
+```
+Add endpoint to `PodHandler` method 
+```go
+const MetricsResourceRouteSuffix = "/metrics/resource"
+
+func PodHandler(p PodHandlerConfig, debug bool) http.Handler {
+	r := mux.NewRouter()
+
+	// This matches the behaviour in the reference kubelet
+	r.StrictSlash(true)
+	if debug {
+		r.HandleFunc("/runningpods/", HandleRunningPods(p.GetPods)).Methods("GET")
+	}
+
+	r.HandleFunc("/pods", HandleRunningPods(p.GetPodsFromKubernetes)).Methods("GET")
+	r.HandleFunc("/containerLogs/{namespace}/{pod}/{container}", HandleContainerLogs(p.GetContainerLogs)).Methods("GET")
+	r.HandleFunc(
+		"/exec/{namespace}/{pod}/{container}",
+		HandleContainerExec(
+			p.RunInContainer,
+			WithExecStreamCreationTimeout(p.StreamCreationTimeout),
+			WithExecStreamIdleTimeout(p.StreamIdleTimeout),
+		),
+	).Methods("POST", "GET")
+
+	if p.GetStatsSummary != nil {
+		f := HandlePodStatsSummary(p.GetStatsSummary)
+		r.HandleFunc("/stats/summary", f).Methods("GET")
+		r.HandleFunc("/stats/summary/", f).Methods("GET")
+	}
+
+	if p.GetMetricsResource != nil {
+		f := HandlePodMetricsResource(p.GetMetricsResource)
+		r.HandleFunc(MetricsResourceRouteSuffix, f).Methods("GET")
+		r.HandleFunc(MetricsResourceRouteSuffix+"/", f).Methods("GET")
+	}
+	r.NotFoundHandler = http.HandlerFunc(NotFound)
+	return r
+}
+```
+  
+New `PodMetricsResourceHandler` method, that uses the new `PodMetricsResourceHandlerFunc` definition.
+```go
+// PodMetricsResourceHandler creates an http handler for serving pod metrics.
+//
+// If the passed in handler func is nil this will create handlers which only
+// serves http.StatusNotImplemented
+func PodMetricsResourceHandler(f PodMetricsResourceHandlerFunc) http.Handler {
+	if f == nil {
+		return http.HandlerFunc(NotImplemented)
+	}
+
+	r := mux.NewRouter()
+
+	h := HandlePodMetricsResource(f)
+
+	r.Handle(MetricsResourceRouteSuffix, ochttp.WithRouteTag(h, "PodMetricsResourceHandler")).Methods("GET")
+	r.Handle(MetricsResourceRouteSuffix+"/", ochttp.WithRouteTag(h, "PodMetricsResourceHandler")).Methods("GET")
+
+	r.NotFoundHandler = http.HandlerFunc(NotFound)
+	return r
+}
+
+```
+ 
+ 
+`HandlePodMetricsResource` method returns a HandlerFunc which serves the metrics encoded in prometheus' text format encoding as expected by the  metrics-server
+```go
+// HandlePodMetricsResource makes an HTTP handler for implementing the kubelet /metrics/resource endpoint
+func HandlePodMetricsResource(h PodMetricsResourceHandlerFunc) http.HandlerFunc {
+	if h == nil {
+		return NotImplemented
+	}
+	return handleError(func(w http.ResponseWriter, req *http.Request) error {
+		metrics, err := h(req.Context())
+		if err != nil {
+			if isCancelled(err) {
+				return err
+			}
+			return errors.Wrap(err, "error getting status from provider")
+		}
+
+		b, err := json.Marshal(metrics)
+		if err != nil {
+			return errors.Wrap(err, "error marshalling metrics")
+		}
+
+		if _, err := w.Write(b); err != nil {
+			return errors.Wrap(err, "could not write to client")
+		}
+		return nil
+	})
+}
+```
+ 
+The `PodMetricsResourceHandlerFunc` returns the metrics data using Prometheus' `MetricFamily` data structure. More details are provided in the Data subsection
+```go
+// PodMetricsResourceHandlerFunc defines the handler for getting pod metrics
+type PodMetricsResourceHandlerFunc func(context.Context) ([]*dto.MetricFamily, error)
+```
+ 
+### Data
+
+The updated metrics server does not add any new fields to the metrics data but uses the Prometheus textparse series parser to parse and reconstruct the [MetricsBatch](https://github.com/kubernetes-sigs/metrics-server/blob/83b2e01f9825849ae5f562e47aa1a4178b5d06e5/pkg/storage/types.go#L31) data structure.  
+Currently virtual-kubelet is sending data to the server using the [summary](https://github.com/virtual-kubelet/virtual-kubelet/blob/be0a062aec9a5eeea3ad6fbe5aec557a235558f6/node/api/statsv1alpha1/types.go#L24) data structure. The Prometheus text parser expects a series of bytes as in the Prometheus [model.Samples](https://github.com/kubernetes/kubernetes/blob/a93eda9db305611cacd8b6ee930ab3149a08f9b0/vendor/github.com/prometheus/common/model/value.go#L184) data structure, similar to the test [here](https://github.com/prometheus/prometheus/blob/c70d85baed260f6013afd18d6cd0ffcac4339861/model/textparse/promparse_test.go#L31).
+ 
+Examples of how the new metrics are defined may be seen in the Kubernetes e2e test that calls the /metrics/resource endpoint [here](https://github.com/kubernetes/kubernetes/blob/a93eda9db305611cacd8b6ee930ab3149a08f9b0/test/e2e_node/resource_metrics_test.go#L76), and the kubelet metrics defined in the Kubernetes/kubelet code [here](https://github.com/kubernetes/kubernetes/blob/master/pkg/kubelet/metrics/collectors/resource_metrics.go) .
+ 
+```go
+var (
+	nodeCPUUsageDesc = metrics.NewDesc("node_cpu_usage_seconds_total",
+		"Cumulative cpu time consumed by the node in core-seconds",
+		nil,
+		nil,
+		metrics.ALPHA,
+		"")
+
+	nodeMemoryUsageDesc = metrics.NewDesc("node_memory_working_set_bytes",
+		"Current working set of the node in bytes",
+		nil,
+		nil,
+		metrics.ALPHA,
+		"")
+
+	containerCPUUsageDesc = metrics.NewDesc("container_cpu_usage_seconds_total",
+		"Cumulative cpu time consumed by the container in core-seconds",
+		[]string{"container", "pod", "namespace"},
+		nil,
+		metrics.ALPHA,
+		"")
+
+	containerMemoryUsageDesc = metrics.NewDesc("container_memory_working_set_bytes",
+		"Current working set of the container in bytes",
+		[]string{"container", "pod", "namespace"},
+		nil,
+		metrics.ALPHA,
+		"")
+
+	podCPUUsageDesc = metrics.NewDesc("pod_cpu_usage_seconds_total",
+		"Cumulative cpu time consumed by the pod in core-seconds",
+		[]string{"pod", "namespace"},
+		nil,
+		metrics.ALPHA,
+		"")
+
+	podMemoryUsageDesc = metrics.NewDesc("pod_memory_working_set_bytes",
+		"Current working set of the pod in bytes",
+		[]string{"pod", "namespace"},
+		nil,
+		metrics.ALPHA,
+		"")
+
+	resourceScrapeResultDesc = metrics.NewDesc("scrape_error",
+		"1 if there was an error while getting container metrics, 0 otherwise",
+		nil,
+		nil,
+		metrics.ALPHA,
+		"")
+
+	containerStartTimeDesc = metrics.NewDesc("container_start_time_seconds",
+		"Start time of the container since unix epoch in seconds",
+		[]string{"container", "pod", "namespace"},
+		nil,
+		metrics.ALPHA,
+		"")
+)
+```
+ 
+The kubernetes/kubelet code implements Prometheus' [collector](https://github.com/kubernetes/kubernetes/blob/master/pkg/kubelet/metrics/collectors/resource_metrics.go#L88) interface which is used along with the k8s.io/component-base implementation of the [registry](https://github.com/kubernetes/component-base/blob/40d14bdbd62f9e2ea697f97d81d4abc72839901e/metrics/registry.go#L114) interface in order to collect and return the metrics data using the Prometheus' [MetricFamily](https://github.com/prometheus/client_model/blob/master/go/metrics.pb.go#L773) data structure.
+ 
+The Gather method in the registry calls the kubelet collector's Collect method, and returns the data using the MetricFamily data structure. The metrics server expects metrics to be encoded in prometheus'
+text format, and the kubelet uses the http handler from prometheus' promhttp module which returns the metrics data encoded in prometheus' text format encoding.
+```go
+type KubeRegistry interface {
+	// Deprecated
+	RawMustRegister(...prometheus.Collector)
+	// CustomRegister is our internal variant of Prometheus registry.Register
+	CustomRegister(c StableCollector) error
+	// CustomMustRegister is our internal variant of Prometheus registry.MustRegister
+	CustomMustRegister(cs ...StableCollector)
+	// Register conforms to Prometheus registry.Register
+	Register(Registerable) error
+	// MustRegister conforms to Prometheus registry.MustRegister
+	MustRegister(...Registerable)
+	// Unregister conforms to Prometheus registry.Unregister
+	Unregister(collector Collector) bool
+	// Gather conforms to Prometheus gatherer.Gather
+	Gather() ([]*dto.MetricFamily, error)
+	// Reset invokes the Reset() function on all items in the registry
+	// which are added as resettables.
+	Reset()
+}
+```
+
+Prometheus’ MetricsFamily data structure:  
+```go
+type MetricFamily struct {
+	Name                 *string     `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"`
+	Help                 *string     `protobuf:"bytes,2,opt,name=help" json:"help,omitempty"`
+	Type                 *MetricType `protobuf:"varint,3,opt,name=type,enum=io.prometheus.client.MetricType" json:"type,omitempty"`
+	Metric               []*Metric   `protobuf:"bytes,4,rep,name=metric" json:"metric,omitempty"`
+	XXX_NoUnkeyedLiteral struct{}    `json:"-"`
+	XXX_unrecognized     []byte      `json:"-"`
+	XXX_sizecache        int32       `json:"-"`
+}
+```
+
+Therefore the provider's GetMetricsResource method should use the same return type as the Gather method in the registry interface.
+ 
+### Changes to the Provider.
+
+In order to support the new metrics endpoint the Provider must implement the GetMetricsResource method with definition
+
+```golang
+
+import (
+  dto "github.com/prometheus/client_model/go"
+  "context"
+)
+
+func GetMetricsResource(context.Context) ([]*dto.MetricsFamily, error) {
+...
+}
+```
+
+### Test Plan
+
+- Write a provider implementation for GetMetricsResource method in ACI Provider and deploy pods get metrics using kubectl 
+- Run end-to-end tests with the provider implementation
+

errdefs/invalid.go

@@ -5,7 +5,7 @@ import (
 	"fmt"
 )
 
-// InvalidInput is an error interface which denotes whether the opration failed due
+// ErrInvalidInput is an error interface which denotes whether the opration failed due
 // to a the resource not being found.
 type ErrInvalidInput interface {
 	InvalidInput() bool

errdefs/notfound.go

@@ -5,7 +5,7 @@ import (
 	"fmt"
 )
 
-// NotFound is an error interface which denotes whether the opration failed due
+// ErrNotFound is an error interface which denotes whether the opration failed due
 // to a the resource not being found.
 type ErrNotFound interface {
 	NotFound() bool

go.mod

@@ -1,87 +1,113 @@
 module github.com/virtual-kubelet/virtual-kubelet
 
-go 1.12
+go 1.17
 
 require (
-	contrib.go.opencensus.io/exporter/jaeger v0.1.0
+	contrib.go.opencensus.io/exporter/jaeger v0.2.1
-	contrib.go.opencensus.io/exporter/ocagent v0.4.12
+	contrib.go.opencensus.io/exporter/ocagent v0.7.0
-	github.com/davecgh/go-spew v1.1.1
+	github.com/bombsimon/logrusr/v3 v3.0.0
-	github.com/docker/spdystream v0.0.0-20170912183627-bc6354cbbc29 // indirect
+	github.com/google/go-cmp v0.5.9
-	github.com/elazarl/goproxy v0.0.0-20190421051319-9d40249d3c2f // indirect
+	github.com/gorilla/mux v1.8.0
-	github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2 // indirect
-	github.com/evanphx/json-patch v4.1.0+incompatible // indirect
-	github.com/gogo/protobuf v1.2.1 // indirect
-	github.com/golang/groupcache v0.0.0-20181024230925-c65c006176ff // indirect
-	github.com/google/go-cmp v0.3.1
-	github.com/google/gofuzz v1.0.0 // indirect
-	github.com/googleapis/gnostic v0.1.0 // indirect
-	github.com/gorilla/mux v1.7.0
-	github.com/hashicorp/golang-lru v0.5.1 // indirect
-	github.com/json-iterator/go v1.1.6 // indirect
-	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
 	github.com/mitchellh/go-homedir v1.1.0
-	github.com/onsi/ginkgo v1.8.0 // indirect
+	github.com/pkg/errors v0.9.1
-	github.com/onsi/gomega v1.5.0 // indirect
+	github.com/prometheus/client_golang v1.14.0
-	github.com/pkg/errors v0.8.1
+	github.com/prometheus/client_model v0.3.0
-	github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829
+	github.com/prometheus/common v0.42.0
-	github.com/sirupsen/logrus v1.4.1
+	github.com/sirupsen/logrus v1.9.0
-	github.com/spf13/cobra v0.0.2
+	github.com/spf13/cobra v1.5.0
-	github.com/spf13/pflag v1.0.3
+	github.com/spf13/pflag v1.0.5
-	github.com/stretchr/testify v1.3.0 // indirect
+	github.com/stretchr/testify v1.8.0
-	go.opencensus.io v0.21.0
+	go.opencensus.io v0.23.0
-	golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c // indirect
+	go.opentelemetry.io/otel v0.20.0
-	golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 // indirect
+	go.opentelemetry.io/otel/sdk v0.20.0
-	golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e // indirect
+	go.opentelemetry.io/otel/trace v0.20.0
-	golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 // indirect
+	golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4
-	google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107 // indirect
+	golang.org/x/sys v0.5.0
-	google.golang.org/grpc v1.20.0 // indirect
+	golang.org/x/time v0.3.0
-	gopkg.in/inf.v0 v0.9.1 // indirect
+	google.golang.org/protobuf v1.28.1
-	gopkg.in/yaml.v2 v2.2.2 // indirect
 	gotest.tools v2.2.0+incompatible
-	k8s.io/api v0.0.0
+	k8s.io/api v0.26.2
-	k8s.io/apimachinery v0.0.0
+	k8s.io/apimachinery v0.26.2
-	k8s.io/client-go v10.0.0+incompatible
+	k8s.io/apiserver v0.25.0
-	k8s.io/klog v0.3.1
+	k8s.io/client-go v0.25.0
-	k8s.io/kube-openapi v0.0.0-20190510232812-a01b7d5d6c22 // indirect
+	k8s.io/klog/v2 v2.90.1
-	k8s.io/kubernetes v1.15.2
+	k8s.io/utils v0.0.0-20221107191617-1a15be271d1d
+	sigs.k8s.io/controller-runtime v0.13.0
 )
 
-replace k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.0.0-20190805144654-3d5bf3a310c1
+require (
-
+	github.com/NYTimes/gziphandler v1.1.1 // indirect
-replace k8s.io/cloud-provider => k8s.io/cloud-provider v0.0.0-20190805144409-8484242760e7
+	github.com/PuerkitoBio/purell v1.1.1 // indirect
-
+	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
-replace k8s.io/cli-runtime => k8s.io/cli-runtime v0.0.0-20190805143448-a07e59fb081d
+	github.com/beorn7/perks v1.0.1 // indirect
-
+	github.com/blang/semver/v4 v4.0.0 // indirect
-replace k8s.io/apiserver => k8s.io/apiserver v0.0.0-20190805142138-368b2058237c
+	github.com/census-instrumentation/opencensus-proto v0.2.1 // indirect
-
+	github.com/cespare/xxhash/v2 v2.1.2 // indirect
-replace k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.0.0-20190805144531-3985229e1802
+	github.com/coreos/go-semver v0.3.0 // indirect
-
+	github.com/coreos/go-systemd/v22 v22.3.2 // indirect
-replace k8s.io/cri-api => k8s.io/cri-api v0.0.0-20190531030430-6117653b35f1
+	github.com/davecgh/go-spew v1.1.1 // indirect
-
+	github.com/emicklei/go-restful/v3 v3.8.0 // indirect
-replace k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.0.0-20190805142416-fd821fbbb94e
+	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
-
+	github.com/evanphx/json-patch/v5 v5.6.0 // indirect
-replace k8s.io/kubelet => k8s.io/kubelet v0.0.0-20190805143852-517ff267f8d1
+	github.com/felixge/httpsnoop v1.0.1 // indirect
-
+	github.com/fsnotify/fsnotify v1.5.4 // indirect
-replace k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.0.0-20190805144128-269742da31dd
+	github.com/go-logr/logr v1.2.3 // indirect
-
+	github.com/go-openapi/jsonpointer v0.19.5 // indirect
-replace k8s.io/apimachinery => k8s.io/apimachinery v0.0.0-20190612205821-1799e75a0719
+	github.com/go-openapi/jsonreference v0.19.5 // indirect
-
+	github.com/go-openapi/swag v0.19.14 // indirect
-replace k8s.io/api => k8s.io/api v0.0.0-20190805141119-fdd30b57c827
+	github.com/gogo/protobuf v1.3.2 // indirect
-
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
-replace k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.0.0-20190805144246-c01ee70854a1
+	github.com/golang/protobuf v1.5.2 // indirect
-
+	github.com/google/gnostic v0.5.7-v3refs // indirect
-replace k8s.io/kube-proxy => k8s.io/kube-proxy v0.0.0-20190805143734-7f1675b90353
+	github.com/google/gofuzz v1.1.0 // indirect
-
+	github.com/google/uuid v1.1.2 // indirect
-replace k8s.io/component-base => k8s.io/component-base v0.0.0-20190805141645-3a5e5ac800ae
+	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
-
+	github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
-replace k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.0.0-20190805144012-2a1ed1f3d8a4
+	github.com/imdario/mergo v0.3.12 // indirect
-
+	github.com/inconshreveable/mousetrap v1.0.0 // indirect
-replace k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.0.0-20190805143126-cdb999c96590
+	github.com/josharian/intern v1.0.0 // indirect
-
+	github.com/json-iterator/go v1.1.12 // indirect
-replace k8s.io/metrics => k8s.io/metrics v0.0.0-20190805143318-16b07057415d
+	github.com/mailru/easyjson v0.7.6 // indirect
-
+	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
-replace k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.0.0-20190805142637-3b65bc4bb24f
+	github.com/moby/spdystream v0.2.0 // indirect
-
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
-replace k8s.io/code-generator => k8s.io/code-generator v0.0.0-20190612205613-18da4a14b22b
+	github.com/modern-go/reflect2 v1.0.2 // indirect
-
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
-replace k8s.io/client-go => k8s.io/client-go v0.0.0-20190805141520-2fe0317bcee0
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/prometheus/procfs v0.8.0 // indirect
+	github.com/uber/jaeger-client-go v2.25.0+incompatible // indirect
+	go.etcd.io/etcd/api/v3 v3.5.4 // indirect
+	go.etcd.io/etcd/client/pkg/v3 v3.5.4 // indirect
+	go.etcd.io/etcd/client/v3 v3.5.4 // indirect
+	go.opentelemetry.io/contrib v0.20.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp v0.20.0 // indirect
+	go.opentelemetry.io/otel/metric v0.20.0 // indirect
+	go.opentelemetry.io/otel/sdk/export/metric v0.20.0 // indirect
+	go.opentelemetry.io/otel/sdk/metric v0.20.0 // indirect
+	go.opentelemetry.io/proto/otlp v0.7.0 // indirect
+	go.uber.org/atomic v1.7.0 // indirect
+	go.uber.org/multierr v1.6.0 // indirect
+	go.uber.org/zap v1.21.0 // indirect
+	golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd // indirect
+	golang.org/x/net v0.7.0 // indirect
+	golang.org/x/oauth2 v0.5.0 // indirect
+	golang.org/x/term v0.5.0 // indirect
+	golang.org/x/text v0.7.0 // indirect
+	google.golang.org/api v0.57.0 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect
+	google.golang.org/grpc v1.47.0 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	k8s.io/apiextensions-apiserver v0.25.0 // indirect
+	k8s.io/component-base v0.25.0 // indirect
+	k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect
+	sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.32 // indirect
+	sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
+	sigs.k8s.io/yaml v1.3.0 // indirect
+)

hack/skaffold/virtual-kubelet/base.yml

@@ -56,6 +56,14 @@ rules:
   verbs:
   - create
   - patch
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - get
+  - create
+  - update
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding

hack/skaffold/virtual-kubelet/pod.yml

@@ -4,6 +4,8 @@ metadata:
   name: vkubelet-mock-0
 spec:
   containers:
+  - name: jaeger-tracing
+    image: jaegertracing/all-in-one:1.22
   - name: vkubelet-mock-0
     image: virtual-kubelet
     # "IfNotPresent" is used to prevent Minikube from trying to pull from the registry (and failing) in the first place.
@@ -23,18 +25,16 @@ spec:
     - --klog.logtostderr
     - --log-level
     - debug
+    - --trace-exporter
+    - jaeger
+    - --trace-sample-rate=always
     env:
+    - name: JAEGER_AGENT_ENDPOINT
+      value: localhost:6831
     - name: KUBELET_PORT
       value: "10250"
     - name: VKUBELET_POD_IP
       valueFrom:
         fieldRef:
           fieldPath: status.podIP
-    ports:
-    - name: metrics
-      containerPort: 10255
-    readinessProbe:
-      httpGet:
-        path: /stats/summary
-        port: metrics
   serviceAccountName: virtual-kubelet

hack/skaffold/virtual-kubelet/skaffold.yml

@@ -1,4 +1,4 @@
-apiVersion: skaffold/v1beta12
+apiVersion: skaffold/v2beta10
 kind: Config
 build:
   artifacts:

internal/expansion/LICENSE

@@ -0,0 +1,27 @@
+Copyright (c) 2012 The Go Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

internal/expansion/README.md

@@ -0,0 +1,4 @@
+Copied from
+https://github.com/kubernetes/kubernetes/tree/master/third_party/forked/golang/expansion .
+
+This is to eliminate a direct dependency on kubernetes/kubernetes.

internal/expansion/expand.go

@@ -0,0 +1,102 @@
+package expansion
+
+import (
+	"bytes"
+)
+
+const (
+	operator        = '$'
+	referenceOpener = '('
+	referenceCloser = ')'
+)
+
+// syntaxWrap returns the input string wrapped by the expansion syntax.
+func syntaxWrap(input string) string {
+	return string(operator) + string(referenceOpener) + input + string(referenceCloser)
+}
+
+// MappingFuncFor returns a mapping function for use with Expand that
+// implements the expansion semantics defined in the expansion spec; it
+// returns the input string wrapped in the expansion syntax if no mapping
+// for the input is found.
+func MappingFuncFor(context ...map[string]string) func(string) string {
+	return func(input string) string {
+		for _, vars := range context {
+			val, ok := vars[input]
+			if ok {
+				return val
+			}
+		}
+
+		return syntaxWrap(input)
+	}
+}
+
+// Expand replaces variable references in the input string according to
+// the expansion spec using the given mapping function to resolve the
+// values of variables.
+func Expand(input string, mapping func(string) string) string {
+	var buf bytes.Buffer
+	checkpoint := 0
+	for cursor := 0; cursor < len(input); cursor++ {
+		if input[cursor] == operator && cursor+1 < len(input) {
+			// Copy the portion of the input string since the last
+			// checkpoint into the buffer
+			buf.WriteString(input[checkpoint:cursor])
+
+			// Attempt to read the variable name as defined by the
+			// syntax from the input string
+			read, isVar, advance := tryReadVariableName(input[cursor+1:])
+
+			if isVar {
+				// We were able to read a variable name correctly;
+				// apply the mapping to the variable name and copy the
+				// bytes into the buffer
+				buf.WriteString(mapping(read))
+			} else {
+				// Not a variable name; copy the read bytes into the buffer
+				buf.WriteString(read)
+			}
+
+			// Advance the cursor in the input string to account for
+			// bytes consumed to read the variable name expression
+			cursor += advance
+
+			// Advance the checkpoint in the input string
+			checkpoint = cursor + 1
+		}
+	}
+
+	// Return the buffer and any remaining unwritten bytes in the
+	// input string.
+	return buf.String() + input[checkpoint:]
+}
+
+// tryReadVariableName attempts to read a variable name from the input
+// string and returns the content read from the input, whether that content
+// represents a variable name to perform mapping on, and the number of bytes
+// consumed in the input string.
+//
+// The input string is assumed not to contain the initial operator.
+func tryReadVariableName(input string) (string, bool, int) {
+	switch input[0] {
+	case operator:
+		// Escaped operator; return it.
+		return input[0:1], false, 1
+	case referenceOpener:
+		// Scan to expression closer
+		for i := 1; i < len(input); i++ {
+			if input[i] == referenceCloser {
+				return input[1:i], true, i + 1
+			}
+		}
+
+		// Incomplete reference; return it.
+		return string(operator) + string(referenceOpener), false, 1
+	default:
+		// Not the beginning of an expression, ie, an operator
+		// that doesn't begin an expression.  Return the operator
+		// and the first rune in the string.
+		return (string(operator) + string(input[0])), false, 1
+	}
+}

internal/expansion/expand_test.go

@@ -0,0 +1,281 @@
+package expansion
+
+import (
+	"testing"
+)
+
+func TestMapReference(t *testing.T) {
+	// We use a struct here instead of a map because we need mappings to happen in order.
+	// Go maps are randomized.
+	type envVar struct {
+		Name  string
+		Value string
+	}
+
+	envs := []envVar{
+		{"FOO", "bar"},
+		{"ZOO", "$(FOO)-1"},
+		{"BLU", "$(ZOO)-2"},
+	}
+
+	declaredEnv := map[string]string{
+		"FOO": "bar",
+		"ZOO": "$(FOO)-1",
+		"BLU": "$(ZOO)-2",
+	}
+
+	serviceEnv := map[string]string{}
+
+	mapping := MappingFuncFor(declaredEnv, serviceEnv)
+
+	for _, env := range envs {
+		declaredEnv[env.Name] = Expand(env.Value, mapping)
+	}
+
+	expectedEnv := map[string]string{
+		"FOO": "bar",
+		"ZOO": "bar-1",
+		"BLU": "bar-1-2",
+	}
+
+	for k, v := range expectedEnv {
+		if e, a := v, declaredEnv[k]; e != a {
+			t.Errorf("Expected %v, got %v", e, a)
+		} else {
+			delete(declaredEnv, k)
+		}
+	}
+
+	if len(declaredEnv) != 0 {
+		t.Errorf("Unexpected keys in declared env: %v", declaredEnv)
+	}
+}
+
+func TestMapping(t *testing.T) {
+	context := map[string]string{
+		"VAR_A":     "A",
+		"VAR_B":     "B",
+		"VAR_C":     "C",
+		"VAR_REF":   "$(VAR_A)",
+		"VAR_EMPTY": "",
+	}
+	mapping := MappingFuncFor(context)
+
+	doExpansionTest(t, mapping)
+}
+
+func TestMappingDual(t *testing.T) {
+	context := map[string]string{
+		"VAR_A":     "A",
+		"VAR_EMPTY": "",
+	}
+	context2 := map[string]string{
+		"VAR_B":   "B",
+		"VAR_C":   "C",
+		"VAR_REF": "$(VAR_A)",
+	}
+	mapping := MappingFuncFor(context, context2)
+
+	doExpansionTest(t, mapping)
+}
+
+func doExpansionTest(t *testing.T, mapping func(string) string) {
+	cases := []struct {
+		name     string
+		input    string
+		expected string
+	}{
+		{
+			name:     "whole string",
+			input:    "$(VAR_A)",
+			expected: "A",
+		},
+		{
+			name:     "repeat",
+			input:    "$(VAR_A)-$(VAR_A)",
+			expected: "A-A",
+		},
+		{
+			name:     "beginning",
+			input:    "$(VAR_A)-1",
+			expected: "A-1",
+		},
+		{
+			name:     "middle",
+			input:    "___$(VAR_B)___",
+			expected: "___B___",
+		},
+		{
+			name:     "end",
+			input:    "___$(VAR_C)",
+			expected: "___C",
+		},
+		{
+			name:     "compound",
+			input:    "$(VAR_A)_$(VAR_B)_$(VAR_C)",
+			expected: "A_B_C",
+		},
+		{
+			name:     "escape & expand",
+			input:    "$$(VAR_B)_$(VAR_A)",
+			expected: "$(VAR_B)_A",
+		},
+		{
+			name:     "compound escape",
+			input:    "$$(VAR_A)_$$(VAR_B)",
+			expected: "$(VAR_A)_$(VAR_B)",
+		},
+		{
+			name:     "mixed in escapes",
+			input:    "f000-$$VAR_A",
+			expected: "f000-$VAR_A",
+		},
+		{
+			name:     "backslash escape ignored",
+			input:    "foo\\$(VAR_C)bar",
+			expected: "foo\\Cbar",
+		},
+		{
+			name:     "backslash escape ignored",
+			input:    "foo\\\\$(VAR_C)bar",
+			expected: "foo\\\\Cbar",
+		},
+		{
+			name:     "lots of backslashes",
+			input:    "foo\\\\\\\\$(VAR_A)bar",
+			expected: "foo\\\\\\\\Abar",
+		},
+		{
+			name:     "nested var references",
+			input:    "$(VAR_A$(VAR_B))",
+			expected: "$(VAR_A$(VAR_B))",
+		},
+		{
+			name:     "nested var references second type",
+			input:    "$(VAR_A$(VAR_B)",
+			expected: "$(VAR_A$(VAR_B)",
+		},
+		{
+			name:     "value is a reference",
+			input:    "$(VAR_REF)",
+			expected: "$(VAR_A)",
+		},
+		{
+			name:     "value is a reference x 2",
+			input:    "%%$(VAR_REF)--$(VAR_REF)%%",
+			expected: "%%$(VAR_A)--$(VAR_A)%%",
+		},
+		{
+			name:     "empty var",
+			input:    "foo$(VAR_EMPTY)bar",
+			expected: "foobar",
+		},
+		{
+			name:     "unterminated expression",
+			input:    "foo$(VAR_Awhoops!",
+			expected: "foo$(VAR_Awhoops!",
+		},
+		{
+			name:     "expression without operator",
+			input:    "f00__(VAR_A)__",
+			expected: "f00__(VAR_A)__",
+		},
+		{
+			name:     "shell special vars pass through",
+			input:    "$?_boo_$!",
+			expected: "$?_boo_$!",
+		},
+		{
+			name:     "bare operators are ignored",
+			input:    "$VAR_A",
+			expected: "$VAR_A",
+		},
+		{
+			name:     "undefined vars are passed through",
+			input:    "$(VAR_DNE)",
+			expected: "$(VAR_DNE)",
+		},
+		{
+			name:     "multiple (even) operators, var undefined",
+			input:    "$$$$$$(BIG_MONEY)",
+			expected: "$$$(BIG_MONEY)",
+		},
+		{
+			name:     "multiple (even) operators, var defined",
+			input:    "$$$$$$(VAR_A)",
+			expected: "$$$(VAR_A)",
+		},
+		{
+			name:     "multiple (odd) operators, var undefined",
+			input:    "$$$$$$$(GOOD_ODDS)",
+			expected: "$$$$(GOOD_ODDS)",
+		},
+		{
+			name:     "multiple (odd) operators, var defined",
+			input:    "$$$$$$$(VAR_A)",
+			expected: "$$$A",
+		},
+		{
+			name:     "missing open expression",
+			input:    "$VAR_A)",
+			expected: "$VAR_A)",
+		},
+		{
+			name:     "shell syntax ignored",
+			input:    "${VAR_A}",
+			expected: "${VAR_A}",
+		},
+		{
+			name:     "trailing incomplete expression not consumed",
+			input:    "$(VAR_B)_______$(A",
+			expected: "B_______$(A",
+		},
+		{
+			name:     "trailing incomplete expression, no content, is not consumed",
+			input:    "$(VAR_C)_______$(",
+			expected: "C_______$(",
+		},
+		{
+			name:     "operator at end of input string is preserved",
+			input:    "$(VAR_A)foobarzab$",
+			expected: "Afoobarzab$",
+		},
+		{
+			name:     "shell escaped incomplete expr",
+			input:    "foo-\\$(VAR_A",
+			expected: "foo-\\$(VAR_A",
+		},
+		{
+			name:     "lots of $( in middle",
+			input:    "--$($($($($--",
+			expected: "--$($($($($--",
+		},
+		{
+			name:     "lots of $( in beginning",
+			input:    "$($($($($--foo$(",
+			expected: "$($($($($--foo$(",
+		},
+		{
+			name:     "lots of $( at end",
+			input:    "foo0--$($($($(",
+			expected: "foo0--$($($($(",
+		},
+		{
+			name:     "escaped operators in variable names are not escaped",
+			input:    "$(foo$$var)",
+			expected: "$(foo$$var)",
+		},
+		{
+			name:     "newline not expanded",
+			input:    "\n",
+			expected: "\n",
+		},
+	}
+
+	for _, tc := range cases {
+		expanded := Expand(tc.input, mapping)
+		if e, a := tc.expected, expanded; e != a {
+			t.Errorf("%v: expected %q, got %q", tc.name, e, a)
+		}
+	}
+}

internal/kubernetes/README.md

@@ -0,0 +1,3 @@
+These package are copied from upstream kubernetes.
+
+They are here to prevent a dep on the whole of kubernetes/kubernetes.

internal/kubernetes/remotecommand/attach.go

@@ -0,0 +1,79 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package remotecommand
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"time"
+
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	remotecommandconsts "k8s.io/apimachinery/pkg/util/remotecommand"
+	"k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/client-go/tools/remotecommand"
+	utilexec "k8s.io/utils/exec"
+)
+
+// Attacher knows how to attach to a container in a pod.
+type Attacher interface {
+	// AttachToContainer attaches to a container in the pod, copying data
+	// between in/out/err and the container's stdin/stdout/stderr.
+	AttachToContainer(name string, uid types.UID, container string, in io.Reader, out, err io.WriteCloser, tty bool, resize <-chan remotecommand.TerminalSize, timeout time.Duration) error
+}
+
+// ServeAttach handles requests to attach to a container. After
+// creating/receiving the required streams, it delegates the actual attachment
+// to the attacher.
+func ServeAttach(w http.ResponseWriter, req *http.Request, attacher Attacher, podName string, uid types.UID, container string, streamOpts *Options, idleTimeout, streamCreationTimeout time.Duration, supportedProtocols []string) {
+	ctx, ok := createStreams(req, w, streamOpts, supportedProtocols, idleTimeout, streamCreationTimeout)
+	if !ok {
+		// error is handled by createStreams
+		return
+	}
+	defer ctx.conn.Close()
+
+	err := attacher.AttachToContainer(podName, uid, container, ctx.stdinStream, ctx.stdoutStream, ctx.stderrStream, ctx.tty, ctx.resizeChan, 0)
+	if err != nil {
+		if exitErr, ok := err.(utilexec.ExitError); ok && exitErr.Exited() {
+			rc := exitErr.ExitStatus()
+			ctx.writeStatus(&apierrors.StatusError{ErrStatus: metav1.Status{
+				Status: metav1.StatusFailure,
+				Reason: remotecommandconsts.NonZeroExitCodeReason,
+				Details: &metav1.StatusDetails{
+					Causes: []metav1.StatusCause{
+						{
+							Type:    remotecommandconsts.ExitCodeCauseType,
+							Message: fmt.Sprintf("%d", rc),
+						},
+					},
+				},
+				Message: fmt.Sprintf("command terminated with non-zero exit code: %v", exitErr),
+			}})
+			return
+		}
+		err = fmt.Errorf("error attaching to container: %v", err)
+		runtime.HandleError(err)
+		ctx.writeStatus(apierrors.NewInternalError(err))
+		return
+	}
+	ctx.writeStatus(&apierrors.StatusError{ErrStatus: metav1.Status{
+		Status: metav1.StatusSuccess,
+	}})
+}

internal/kubernetes/remotecommand/exec.go

@@ -0,0 +1,79 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package remotecommand
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"time"
+
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	remotecommandconsts "k8s.io/apimachinery/pkg/util/remotecommand"
+	"k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/client-go/tools/remotecommand"
+	utilexec "k8s.io/utils/exec"
+)
+
+// Executor knows how to execute a command in a container in a pod.
+type Executor interface {
+	// ExecInContainer executes a command in a container in the pod, copying data
+	// between in/out/err and the container's stdin/stdout/stderr.
+	ExecInContainer(name string, uid types.UID, container string, cmd []string, in io.Reader, out, err io.WriteCloser, tty bool, resize <-chan remotecommand.TerminalSize, timeout time.Duration) error
+}
+
+// ServeExec handles requests to execute a command in a container. After
+// creating/receiving the required streams, it delegates the actual execution
+// to the executor.
+func ServeExec(w http.ResponseWriter, req *http.Request, executor Executor, podName string, uid types.UID, container string, cmd []string, streamOpts *Options, idleTimeout, streamCreationTimeout time.Duration, supportedProtocols []string) {
+	ctx, ok := createStreams(req, w, streamOpts, supportedProtocols, idleTimeout, streamCreationTimeout)
+	if !ok {
+		// error is handled by createStreams
+		return
+	}
+	defer ctx.conn.Close()
+
+	err := executor.ExecInContainer(podName, uid, container, cmd, ctx.stdinStream, ctx.stdoutStream, ctx.stderrStream, ctx.tty, ctx.resizeChan, 0)
+	if err != nil {
+		if exitErr, ok := err.(utilexec.ExitError); ok && exitErr.Exited() {
+			rc := exitErr.ExitStatus()
+			ctx.writeStatus(&apierrors.StatusError{ErrStatus: metav1.Status{
+				Status: metav1.StatusFailure,
+				Reason: remotecommandconsts.NonZeroExitCodeReason,
+				Details: &metav1.StatusDetails{
+					Causes: []metav1.StatusCause{
+						{
+							Type:    remotecommandconsts.ExitCodeCauseType,
+							Message: fmt.Sprintf("%d", rc),
+						},
+					},
+				},
+				Message: fmt.Sprintf("command terminated with non-zero exit code: %v", exitErr),
+			}})
+		} else {
+			err = fmt.Errorf("error executing command in container: %v", err)
+			runtime.HandleError(err)
+			ctx.writeStatus(apierrors.NewInternalError(err))
+		}
+	} else {
+		ctx.writeStatus(&apierrors.StatusError{ErrStatus: metav1.Status{
+			Status: metav1.StatusSuccess,
+		}})
+	}
+}

internal/kubernetes/remotecommand/httpstream.go

@@ -0,0 +1,447 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package remotecommand
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"time"
+
+	api "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/httpstream"
+	"k8s.io/apimachinery/pkg/util/httpstream/spdy"
+	remotecommandconsts "k8s.io/apimachinery/pkg/util/remotecommand"
+	"k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apiserver/pkg/util/wsstream"
+	"k8s.io/client-go/tools/remotecommand"
+
+	"k8s.io/klog/v2"
+)
+
+// Options contains details about which streams are required for
+// remote command execution.
+type Options struct {
+	Stdin  bool
+	Stdout bool
+	Stderr bool
+	TTY    bool
+}
+
+// NewOptions creates a new Options from the Request.
+func NewOptions(req *http.Request) (*Options, error) {
+	tty := req.FormValue(api.ExecTTYParam) == "1"
+	stdin := req.FormValue(api.ExecStdinParam) == "1"
+	stdout := req.FormValue(api.ExecStdoutParam) == "1"
+	stderr := req.FormValue(api.ExecStderrParam) == "1"
+	if tty && stderr {
+		// TODO: make this an error before we reach this method
+		klog.V(4).Infof("Access to exec with tty and stderr is not supported, bypassing stderr")
+		stderr = false
+	}
+
+	if !stdin && !stdout && !stderr {
+		return nil, fmt.Errorf("you must specify at least 1 of stdin, stdout, stderr")
+	}
+
+	return &Options{
+		Stdin:  stdin,
+		Stdout: stdout,
+		Stderr: stderr,
+		TTY:    tty,
+	}, nil
+}
+
+// context contains the connection and streams used when
+// forwarding an attach or execute session into a container.
+type context struct {
+	conn         io.Closer
+	stdinStream  io.ReadCloser
+	stdoutStream io.WriteCloser
+	stderrStream io.WriteCloser
+	writeStatus  func(status *apierrors.StatusError) error
+	resizeStream io.ReadCloser
+	resizeChan   chan remotecommand.TerminalSize
+	tty          bool
+}
+
+// streamAndReply holds both a Stream and a channel that is closed when the stream's reply frame is
+// enqueued. Consumers can wait for replySent to be closed prior to proceeding, to ensure that the
+// replyFrame is enqueued before the connection's goaway frame is sent (e.g. if a stream was
+// received and right after, the connection gets closed).
+type streamAndReply struct {
+	httpstream.Stream
+	replySent <-chan struct{}
+}
+
+// waitStreamReply waits until either replySent or stop is closed. If replySent is closed, it sends
+// an empty struct to the notify channel.
+func waitStreamReply(replySent <-chan struct{}, notify chan<- struct{}, stop <-chan struct{}) {
+	select {
+	case <-replySent:
+		notify <- struct{}{}
+	case <-stop:
+	}
+}
+
+func createStreams(req *http.Request, w http.ResponseWriter, opts *Options, supportedStreamProtocols []string, idleTimeout, streamCreationTimeout time.Duration) (*context, bool) {
+	var ctx *context
+	var ok bool
+	if wsstream.IsWebSocketRequest(req) {
+		ctx, ok = createWebSocketStreams(req, w, opts, idleTimeout)
+	} else {
+		ctx, ok = createHTTPStreamStreams(req, w, opts, supportedStreamProtocols, idleTimeout, streamCreationTimeout)
+	}
+	if !ok {
+		return nil, false
+	}
+
+	if ctx.resizeStream != nil {
+		ctx.resizeChan = make(chan remotecommand.TerminalSize)
+		go handleResizeEvents(ctx.resizeStream, ctx.resizeChan)
+	}
+
+	return ctx, true
+}
+
+func createHTTPStreamStreams(req *http.Request, w http.ResponseWriter, opts *Options, supportedStreamProtocols []string, idleTimeout, streamCreationTimeout time.Duration) (*context, bool) {
+	protocol, err := httpstream.Handshake(req, w, supportedStreamProtocols)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return nil, false
+	}
+
+	streamCh := make(chan streamAndReply)
+
+	upgrader := spdy.NewResponseUpgrader()
+	conn := upgrader.UpgradeResponse(w, req, func(stream httpstream.Stream, replySent <-chan struct{}) error {
+		streamCh <- streamAndReply{Stream: stream, replySent: replySent}
+		return nil
+	})
+	// from this point on, we can no longer call methods on response
+	if conn == nil {
+		// The upgrader is responsible for notifying the client of any errors that
+		// occurred during upgrading. All we can do is return here at this point
+		// if we weren't successful in upgrading.
+		return nil, false
+	}
+
+	conn.SetIdleTimeout(idleTimeout)
+
+	var handler protocolHandler
+	switch protocol {
+	case remotecommandconsts.StreamProtocolV4Name:
+		handler = &v4ProtocolHandler{}
+	case remotecommandconsts.StreamProtocolV3Name:
+		handler = &v3ProtocolHandler{}
+	case remotecommandconsts.StreamProtocolV2Name:
+		handler = &v2ProtocolHandler{}
+	case "":
+		klog.V(4).Infof("Client did not request protocol negotiation. Falling back to %q", remotecommandconsts.StreamProtocolV1Name)
+		fallthrough
+	case remotecommandconsts.StreamProtocolV1Name:
+		handler = &v1ProtocolHandler{}
+	}
+
+	// count the streams client asked for, starting with 1
+	expectedStreams := 1
+	if opts.Stdin {
+		expectedStreams++
+	}
+	if opts.Stdout {
+		expectedStreams++
+	}
+	if opts.Stderr {
+		expectedStreams++
+	}
+	if opts.TTY && handler.supportsTerminalResizing() {
+		expectedStreams++
+	}
+
+	expired := time.NewTimer(streamCreationTimeout)
+	defer expired.Stop()
+
+	ctx, err := handler.waitForStreams(streamCh, expectedStreams, expired.C)
+	if err != nil {
+		runtime.HandleError(err)
+		return nil, false
+	}
+
+	ctx.conn = conn
+	ctx.tty = opts.TTY
+
+	return ctx, true
+}
+
+type protocolHandler interface {
+	// waitForStreams waits for the expected streams or a timeout, returning a
+	// remoteCommandContext if all the streams were received, or an error if not.
+	waitForStreams(streams <-chan streamAndReply, expectedStreams int, expired <-chan time.Time) (*context, error)
+	// supportsTerminalResizing returns true if the protocol handler supports terminal resizing
+	supportsTerminalResizing() bool
+}
+
+// v4ProtocolHandler implements the V4 protocol version for streaming command execution. It only differs
+// in from v3 in the error stream format using an json-marshaled metav1.Status which carries
+// the process' exit code.
+type v4ProtocolHandler struct{}
+
+func (*v4ProtocolHandler) waitForStreams(streams <-chan streamAndReply, expectedStreams int, expired <-chan time.Time) (*context, error) {
+	ctx := &context{}
+	receivedStreams := 0
+	replyChan := make(chan struct{})
+	stop := make(chan struct{})
+	defer close(stop)
+WaitForStreams:
+	for {
+		select {
+		case stream := <-streams:
+			streamType := stream.Headers().Get(api.StreamType)
+			switch streamType {
+			case api.StreamTypeError:
+				ctx.writeStatus = v4WriteStatusFunc(stream) // write json errors
+				go waitStreamReply(stream.replySent, replyChan, stop)
+			case api.StreamTypeStdin:
+				ctx.stdinStream = stream
+				go waitStreamReply(stream.replySent, replyChan, stop)
+			case api.StreamTypeStdout:
+				ctx.stdoutStream = stream
+				go waitStreamReply(stream.replySent, replyChan, stop)
+			case api.StreamTypeStderr:
+				ctx.stderrStream = stream
+				go waitStreamReply(stream.replySent, replyChan, stop)
+			case api.StreamTypeResize:
+				ctx.resizeStream = stream
+				go waitStreamReply(stream.replySent, replyChan, stop)
+			default:
+				runtime.HandleError(fmt.Errorf("unexpected stream type: %q", streamType))
+			}
+		case <-replyChan:
+			receivedStreams++
+			if receivedStreams == expectedStreams {
+				break WaitForStreams
+			}
+		case <-expired:
+			// TODO find a way to return the error to the user. Maybe use a separate
+			// stream to report errors?
+			return nil, errors.New("timed out waiting for client to create streams")
+		}
+	}
+
+	return ctx, nil
+}
+
+// supportsTerminalResizing returns true because v4ProtocolHandler supports it
+func (*v4ProtocolHandler) supportsTerminalResizing() bool { return true }
+
+// v3ProtocolHandler implements the V3 protocol version for streaming command execution.
+type v3ProtocolHandler struct{}
+
+func (*v3ProtocolHandler) waitForStreams(streams <-chan streamAndReply, expectedStreams int, expired <-chan time.Time) (*context, error) {
+	ctx := &context{}
+	receivedStreams := 0
+	replyChan := make(chan struct{})
+	stop := make(chan struct{})
+	defer close(stop)
+WaitForStreams:
+	for {
+		select {
+		case stream := <-streams:
+			streamType := stream.Headers().Get(api.StreamType)
+			switch streamType {
+			case api.StreamTypeError:
+				ctx.writeStatus = v1WriteStatusFunc(stream)
+				go waitStreamReply(stream.replySent, replyChan, stop)
+			case api.StreamTypeStdin:
+				ctx.stdinStream = stream
+				go waitStreamReply(stream.replySent, replyChan, stop)
+			case api.StreamTypeStdout:
+				ctx.stdoutStream = stream
+				go waitStreamReply(stream.replySent, replyChan, stop)
+			case api.StreamTypeStderr:
+				ctx.stderrStream = stream
+				go waitStreamReply(stream.replySent, replyChan, stop)
+			case api.StreamTypeResize:
+				ctx.resizeStream = stream
+				go waitStreamReply(stream.replySent, replyChan, stop)
+			default:
+				runtime.HandleError(fmt.Errorf("unexpected stream type: %q", streamType))
+			}
+		case <-replyChan:
+			receivedStreams++
+			if receivedStreams == expectedStreams {
+				break WaitForStreams
+			}
+		case <-expired:
+			// TODO find a way to return the error to the user. Maybe use a separate
+			// stream to report errors?
+			return nil, errors.New("timed out waiting for client to create streams")
+		}
+	}
+
+	return ctx, nil
+}
+
+// supportsTerminalResizing returns true because v3ProtocolHandler supports it
+func (*v3ProtocolHandler) supportsTerminalResizing() bool { return true }
+
+// v2ProtocolHandler implements the V2 protocol version for streaming command execution.
+type v2ProtocolHandler struct{}
+
+func (*v2ProtocolHandler) waitForStreams(streams <-chan streamAndReply, expectedStreams int, expired <-chan time.Time) (*context, error) {
+	ctx := &context{}
+	receivedStreams := 0
+	replyChan := make(chan struct{})
+	stop := make(chan struct{})
+	defer close(stop)
+WaitForStreams:
+	for {
+		select {
+		case stream := <-streams:
+			streamType := stream.Headers().Get(api.StreamType)
+			switch streamType {
+			case api.StreamTypeError:
+				ctx.writeStatus = v1WriteStatusFunc(stream)
+				go waitStreamReply(stream.replySent, replyChan, stop)
+			case api.StreamTypeStdin:
+				ctx.stdinStream = stream
+				go waitStreamReply(stream.replySent, replyChan, stop)
+			case api.StreamTypeStdout:
+				ctx.stdoutStream = stream
+				go waitStreamReply(stream.replySent, replyChan, stop)
+			case api.StreamTypeStderr:
+				ctx.stderrStream = stream
+				go waitStreamReply(stream.replySent, replyChan, stop)
+			default:
+				runtime.HandleError(fmt.Errorf("unexpected stream type: %q", streamType))
+			}
+		case <-replyChan:
+			receivedStreams++
+			if receivedStreams == expectedStreams {
+				break WaitForStreams
+			}
+		case <-expired:
+			// TODO find a way to return the error to the user. Maybe use a separate
+			// stream to report errors?
+			return nil, errors.New("timed out waiting for client to create streams")
+		}
+	}
+
+	return ctx, nil
+}
+
+// supportsTerminalResizing returns false because v2ProtocolHandler doesn't support it.
+func (*v2ProtocolHandler) supportsTerminalResizing() bool { return false }
+
+// v1ProtocolHandler implements the V1 protocol version for streaming command execution.
+type v1ProtocolHandler struct{}
+
+func (*v1ProtocolHandler) waitForStreams(streams <-chan streamAndReply, expectedStreams int, expired <-chan time.Time) (*context, error) {
+	ctx := &context{}
+	receivedStreams := 0
+	replyChan := make(chan struct{})
+	stop := make(chan struct{})
+	defer close(stop)
+WaitForStreams:
+	for {
+		select {
+		case stream := <-streams:
+			streamType := stream.Headers().Get(api.StreamType)
+			switch streamType {
+			case api.StreamTypeError:
+				ctx.writeStatus = v1WriteStatusFunc(stream)
+
+				// This defer statement shouldn't be here, but due to previous refactoring, it ended up in
+				// here. This is what 1.0.x kubelets do, so we're retaining that behavior. This is fixed in
+				// the v2ProtocolHandler.
+				defer stream.Reset()
+
+				go waitStreamReply(stream.replySent, replyChan, stop)
+			case api.StreamTypeStdin:
+				ctx.stdinStream = stream
+				go waitStreamReply(stream.replySent, replyChan, stop)
+			case api.StreamTypeStdout:
+				ctx.stdoutStream = stream
+				go waitStreamReply(stream.replySent, replyChan, stop)
+			case api.StreamTypeStderr:
+				ctx.stderrStream = stream
+				go waitStreamReply(stream.replySent, replyChan, stop)
+			default:
+				runtime.HandleError(fmt.Errorf("unexpected stream type: %q", streamType))
+			}
+		case <-replyChan:
+			receivedStreams++
+			if receivedStreams == expectedStreams {
+				break WaitForStreams
+			}
+		case <-expired:
+			// TODO find a way to return the error to the user. Maybe use a separate
+			// stream to report errors?
+			return nil, errors.New("timed out waiting for client to create streams")
+		}
+	}
+
+	if ctx.stdinStream != nil {
+		ctx.stdinStream.Close()
+	}
+
+	return ctx, nil
+}
+
+// supportsTerminalResizing returns false because v1ProtocolHandler doesn't support it.
+func (*v1ProtocolHandler) supportsTerminalResizing() bool { return false }
+
+func handleResizeEvents(stream io.Reader, channel chan<- remotecommand.TerminalSize) {
+	defer runtime.HandleCrash()
+	defer close(channel)
+
+	decoder := json.NewDecoder(stream)
+	for {
+		size := remotecommand.TerminalSize{}
+		if err := decoder.Decode(&size); err != nil {
+			break
+		}
+		channel <- size
+	}
+}
+
+func v1WriteStatusFunc(stream io.Writer) func(status *apierrors.StatusError) error {
+	return func(status *apierrors.StatusError) error {
+		if status.Status().Status == metav1.StatusSuccess {
+			return nil // send error messages
+		}
+		_, err := stream.Write([]byte(status.Error()))
+		return err
+	}
+}
+
+// v4WriteStatusFunc returns a WriteStatusFunc that marshals a given api Status
+// as json in the error channel.
+func v4WriteStatusFunc(stream io.Writer) func(status *apierrors.StatusError) error {
+	return func(status *apierrors.StatusError) error {
+		bs, err := json.Marshal(status.Status())
+		if err != nil {
+			return err
+		}
+		_, err = stream.Write(bs)
+		return err
+	}
+}

internal/kubernetes/remotecommand/websocket.go

@@ -0,0 +1,132 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package remotecommand
+
+import (
+	"fmt"
+	"net/http"
+	"time"
+
+	"k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apiserver/pkg/server/httplog"
+	"k8s.io/apiserver/pkg/util/wsstream"
+)
+
+const (
+	stdinChannel = iota
+	stdoutChannel
+	stderrChannel
+	errorChannel
+	resizeChannel
+
+	preV4BinaryWebsocketProtocol = wsstream.ChannelWebSocketProtocol
+	preV4Base64WebsocketProtocol = wsstream.Base64ChannelWebSocketProtocol
+	v4BinaryWebsocketProtocol    = "v4." + wsstream.ChannelWebSocketProtocol
+	v4Base64WebsocketProtocol    = "v4." + wsstream.Base64ChannelWebSocketProtocol
+)
+
+// createChannels returns the standard channel types for a shell connection (STDIN 0, STDOUT 1, STDERR 2)
+// along with the approximate duplex value. It also creates the error (3) and resize (4) channels.
+func createChannels(opts *Options) []wsstream.ChannelType {
+	// open the requested channels, and always open the error channel
+	channels := make([]wsstream.ChannelType, 5)
+	channels[stdinChannel] = readChannel(opts.Stdin)
+	channels[stdoutChannel] = writeChannel(opts.Stdout)
+	channels[stderrChannel] = writeChannel(opts.Stderr)
+	channels[errorChannel] = wsstream.WriteChannel
+	channels[resizeChannel] = wsstream.ReadChannel
+	return channels
+}
+
+// readChannel returns wsstream.ReadChannel if real is true, or wsstream.IgnoreChannel.
+func readChannel(real bool) wsstream.ChannelType {
+	if real {
+		return wsstream.ReadChannel
+	}
+	return wsstream.IgnoreChannel
+}
+
+// writeChannel returns wsstream.WriteChannel if real is true, or wsstream.IgnoreChannel.
+func writeChannel(real bool) wsstream.ChannelType {
+	if real {
+		return wsstream.WriteChannel
+	}
+	return wsstream.IgnoreChannel
+}
+
+// createWebSocketStreams returns a context containing the websocket connection and
+// streams needed to perform an exec or an attach.
+func createWebSocketStreams(req *http.Request, w http.ResponseWriter, opts *Options, idleTimeout time.Duration) (*context, bool) {
+	channels := createChannels(opts)
+	conn := wsstream.NewConn(map[string]wsstream.ChannelProtocolConfig{
+		"": {
+			Binary:   true,
+			Channels: channels,
+		},
+		preV4BinaryWebsocketProtocol: {
+			Binary:   true,
+			Channels: channels,
+		},
+		preV4Base64WebsocketProtocol: {
+			Binary:   false,
+			Channels: channels,
+		},
+		v4BinaryWebsocketProtocol: {
+			Binary:   true,
+			Channels: channels,
+		},
+		v4Base64WebsocketProtocol: {
+			Binary:   false,
+			Channels: channels,
+		},
+	})
+	conn.SetIdleTimeout(idleTimeout)
+	negotiatedProtocol, streams, err := conn.Open(httplog.Unlogged(req, w), req)
+	if err != nil {
+		runtime.HandleError(fmt.Errorf("unable to upgrade websocket connection: %v", err))
+		return nil, false
+	}
+
+	// Send an empty message to the lowest writable channel to notify the client the connection is established
+	// TODO: make generic to SPDY and WebSockets and do it outside of this method?
+	switch {
+	case opts.Stdout:
+		streams[stdoutChannel].Write([]byte{})
+	case opts.Stderr:
+		streams[stderrChannel].Write([]byte{})
+	default:
+		streams[errorChannel].Write([]byte{})
+	}
+
+	ctx := &context{
+		conn:         conn,
+		stdinStream:  streams[stdinChannel],
+		stdoutStream: streams[stdoutChannel],
+		stderrStream: streams[stderrChannel],
+		tty:          opts.TTY,
+		resizeStream: streams[resizeChannel],
+	}
+
+	switch negotiatedProtocol {
+	case v4BinaryWebsocketProtocol, v4Base64WebsocketProtocol:
+		ctx.writeStatus = v4WriteStatusFunc(streams[errorChannel])
+	default:
+		ctx.writeStatus = v1WriteStatusFunc(streams[errorChannel])
+	}
+
+	return ctx, true
+}

internal/lock/monitor.go

@@ -0,0 +1,100 @@
+package lock
+
+import (
+	"sync"
+)
+
+// NewMonitorVariable instantiates an empty monitor variable
+func NewMonitorVariable() MonitorVariable {
+	mv := &monitorVariable{
+		versionInvalidationChannel: make(chan struct{}),
+	}
+	return mv
+}
+
+// MonitorVariable is a specific monitor variable which allows for channel-subscription to changes to
+// the internal value of the MonitorVariable.
+type MonitorVariable interface {
+	Set(value interface{})
+	Subscribe() Subscription
+}
+
+// Subscription is not concurrency safe. It must not be shared between multiple goroutines.
+type Subscription interface {
+	// On instantiation, if the value has been set, this will return a closed channel. Otherwise, it will follow the
+	// standard semantic, which is when the Monitor Variable is updated, this channel will close. The channel is updated
+	// based on reading Value(). Once a value is read, the channel returned will only be closed if a the Monitor Variable
+	// is set to a new value.
+	NewValueReady() <-chan struct{}
+	// Value returns a value object in a non-blocking fashion. This also means it may return an uninitialized value.
+	// If the monitor variable has not yet been set, the "Version" of the value will be 0.
+	Value() Value
+}
+
+// Value contains the last set value from Set(). If the value is unset the version will be 0, and the value will be
+// nil.
+type Value struct {
+	Value   interface{}
+	Version int64
+}
+
+type monitorVariable struct {
+	lock         sync.Mutex
+	currentValue interface{}
+	// 0 indicates uninitialized
+	currentVersion             int64
+	versionInvalidationChannel chan struct{}
+}
+
+func (m *monitorVariable) Set(newValue interface{}) {
+	m.lock.Lock()
+	defer m.lock.Unlock()
+	m.currentValue = newValue
+	m.currentVersion++
+	close(m.versionInvalidationChannel)
+	m.versionInvalidationChannel = make(chan struct{})
+}
+
+func (m *monitorVariable) Subscribe() Subscription {
+	m.lock.Lock()
+	defer m.lock.Unlock()
+	sub := &subscription{
+		mv: m,
+	}
+	if m.currentVersion > 0 {
+		// A value has been set. Set the first versionInvalidationChannel to a closed one.
+		closedCh := make(chan struct{})
+		close(closedCh)
+		sub.lastVersionReadInvalidationChannel = closedCh
+	} else {
+		// The value hasn't yet been initialized.
+		sub.lastVersionReadInvalidationChannel = m.versionInvalidationChannel
+	}
+
+	return sub
+}
+
+type subscription struct {
+	mv                                 *monitorVariable
+	lastVersionRead                    int64
+	lastVersionReadInvalidationChannel chan struct{}
+}
+
+func (s *subscription) NewValueReady() <-chan struct{} {
+	/* This lock could be finer grained (on just the subscription) */
+	s.mv.lock.Lock()
+	defer s.mv.lock.Unlock()
+	return s.lastVersionReadInvalidationChannel
+}
+
+func (s *subscription) Value() Value {
+	s.mv.lock.Lock()
+	defer s.mv.lock.Unlock()
+	val := Value{
+		Value:   s.mv.currentValue,
+		Version: s.mv.currentVersion,
+	}
+	s.lastVersionRead = s.mv.currentVersion
+	s.lastVersionReadInvalidationChannel = s.mv.versionInvalidationChannel
+	return val
+}

internal/lock/monitor_test.go

@@ -0,0 +1,111 @@
+package lock
+
+import (
+	"sync"
+	"testing"
+	"time"
+
+	"golang.org/x/sync/errgroup"
+	"k8s.io/apimachinery/pkg/util/sets"
+
+	"gotest.tools/assert"
+	is "gotest.tools/assert/cmp"
+)
+
+func TestMonitorUninitialized(t *testing.T) {
+	t.Parallel()
+	mv := NewMonitorVariable()
+	subscription := mv.Subscribe()
+	select {
+	case <-subscription.NewValueReady():
+		t.Fatalf("Received value update message: %v", subscription.Value())
+	case <-time.After(time.Second):
+	}
+}
+
+func TestGetUninitialized(t *testing.T) {
+	mv := NewMonitorVariable()
+	subscription := mv.Subscribe()
+	val := subscription.Value()
+	assert.Assert(t, is.Equal(val.Version, int64(0)))
+}
+
+func TestMonitorSetInitialVersionAfterListen(t *testing.T) {
+	mv := NewMonitorVariable()
+	subscription := mv.Subscribe()
+	go mv.Set("test")
+	<-subscription.NewValueReady()
+	assert.Assert(t, is.Equal(subscription.Value().Value, "test"))
+}
+
+func TestMonitorSetInitialVersionBeforeListen(t *testing.T) {
+	mv := NewMonitorVariable()
+	subscription := mv.Subscribe()
+	mv.Set("test")
+	<-subscription.NewValueReady()
+	assert.Assert(t, is.Equal(subscription.Value().Value, "test"))
+}
+
+func TestMonitorMultipleVersionsBlock(t *testing.T) {
+	t.Parallel()
+	mv := NewMonitorVariable()
+	subscription := mv.Subscribe()
+	mv.Set("test")
+	<-subscription.NewValueReady()
+	/* This should mark the "current" version as seen */
+	val := subscription.Value()
+	assert.Assert(t, is.Equal(val.Version, int64(1)))
+	select {
+	case <-subscription.NewValueReady():
+		t.Fatalf("Received value update message: %v", subscription.Value())
+	case <-time.After(time.Second):
+	}
+}
+func TestMonitorMultipleVersions(t *testing.T) {
+	t.Parallel()
+	lock := sync.Mutex{}
+	lock.Lock()
+	mv := NewMonitorVariable()
+	triggers := []int{}
+	ch := make(chan struct{}, 10)
+	go func() {
+		defer lock.Unlock()
+		subscription := mv.Subscribe()
+		for {
+			// Lint is wrong, we need to call the function each time to get a fresh channel.
+			<-subscription.NewValueReady()
+			val := subscription.Value()
+			triggers = append(triggers, val.Value.(int))
+			ch <- struct{}{}
+			if val.Value == 9 {
+				return
+			}
+		}
+	}()
+
+	for i := 0; i < 10; i++ {
+		mv.Set(i)
+		// Wait for the trigger to occur
+		<-ch
+	}
+
+	// Wait for the goroutine to finish
+	lock.Lock()
+	t.Logf("Saw %v triggers", triggers)
+	assert.Assert(t, is.Len(triggers, 10))
+	// Make sure we saw all 10 unique values
+	assert.Assert(t, is.Equal(sets.NewInt(triggers...).Len(), 10))
+}
+func TestMonitorMultipleSubscribers(t *testing.T) {
+	group := &errgroup.Group{}
+	mv := NewMonitorVariable()
+	for i := 0; i < 10; i++ {
+		sub := mv.Subscribe()
+		group.Go(func() error {
+			<-sub.NewValueReady()
+			return nil
+		})
+	}
+	mv.Set(1)
+	_ = group.Wait()
+}

internal/lockdeps/lockdeps.go

@@ -5,4 +5,5 @@ import (
 	// This is a dep that `go mod tidy` keeps removing, because it's a transitive dep that's pulled in via a test
 	// See: https://github.com/golang/go/issues/29702
 	_ "github.com/prometheus/client_golang/prometheus"
+	_ "golang.org/x/sys/unix"
 )

internal/manager/resource_test.go

@@ -123,7 +123,7 @@ func TestGetConfigMap(t *testing.T) {
 	}
 	value := configMap.Data["key-0"]
 	if value != "val-0" {
-		t.Fatal("got unexpected value", string(value))
+		t.Fatal("got unexpected value", value)
 	}
 
 	// Try to get a configmap that does not exist, and make sure we've got a "not found" error as a response.

internal/podutils/README.md

@@ -0,0 +1,15 @@
+Much of this is copied from k8s.io/kubernetes, even if it isn't a 1-1 copy of a
+file.  This exists so we do not have to import from k8s.io/kubernetes which is
+currently problematic.  Ideally most or all of this will go away and an upstream
+solution is found so that we can share an implementation with Kubelet without
+importing from k8s.io/kubernetes
+
+
+| filename | upstream location |
+|----------|-------------------|
+| envvars.go | https://github.com/kubernetes/kubernetes/blob/98d5dc5d36d34a7ee13368a7893dcb400ec4e566/pkg/kubelet/envvars/envvars.go#L32 |
+| helper.go#ConvertDownwardAPIFieldLabel | https://github.com/kubernetes/kubernetes/blob/98d5dc5d36d34a7ee13368a7893dcb400ec4e566/pkg/apis/core/pods/helpers.go#L65 |
+| helper.go#ExtractFieldPathAsString | https://github.com/kubernetes/kubernetes/blob/98d5dc5d36d34a7ee13368a7893dcb400ec4e566/pkg/fieldpath/fieldpath.go#L46 |
+| helper.go#SplitMaybeSubscriptedPath | https://github.com/kubernetes/kubernetes/blob/98d5dc5d36d34a7ee13368a7893dcb400ec4e566/pkg/fieldpath/fieldpath.go#L96 |
+| helper.go#FormatMap | https://github.com/kubernetes/kubernetes/blob/ea0764452222146c47ec826977f49d7001b0ea8c/pkg/fieldpath/fieldpath.go#L29 |
+| helper.go#IsServiceIPSet | https://github.com/kubernetes/kubernetes/blob/ea0764452222146c47ec826977f49d7001b0ea8c/pkg/apis/core/v1/helper/helpers.go#L139 |

internal/podutils/env.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package node
+package podutils
 
 import (
 	"context"
@@ -20,20 +20,16 @@ import (
 	"sort"
 	"strings"
 
+	"github.com/virtual-kubelet/virtual-kubelet/internal/expansion"
+	"github.com/virtual-kubelet/virtual-kubelet/internal/manager"
+	"github.com/virtual-kubelet/virtual-kubelet/log"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/sets"
 	apivalidation "k8s.io/apimachinery/pkg/util/validation"
 	"k8s.io/client-go/tools/record"
-	podshelper "k8s.io/kubernetes/pkg/apis/core/pods"
+	"k8s.io/utils/pointer"
-	v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper"
-	fieldpath "k8s.io/kubernetes/pkg/fieldpath"
-	"k8s.io/kubernetes/pkg/kubelet/envvars"
-	"k8s.io/kubernetes/third_party/forked/golang/expansion"
-
-	"github.com/virtual-kubelet/virtual-kubelet/internal/manager"
-	"github.com/virtual-kubelet/virtual-kubelet/log"
 )
 
 const (
@@ -71,9 +67,8 @@ const (
 
 var masterServices = sets.NewString("kubernetes")
 
-// populateEnvironmentVariables populates the environment of each container (and init container) in the specified pod.
+// PopulateEnvironmentVariables populates the environment of each container (and init container) in the specified pod.
-// TODO Make this the single exported function of a "pkg/environment" package in the future.
+func PopulateEnvironmentVariables(ctx context.Context, pod *corev1.Pod, rm *manager.ResourceManager, recorder record.EventRecorder) error {
-func populateEnvironmentVariables(ctx context.Context, pod *corev1.Pod, rm *manager.ResourceManager, recorder record.EventRecorder) error {
 
 	// Populate each init container's environment.
 	for idx := range pod.Spec.InitContainers {
@@ -109,7 +104,7 @@ func populateContainerEnvironment(ctx context.Context, pod *corev1.Pod, containe
 	// https://github.com/kubernetes/kubernetes/blob/v1.13.1/pkg/kubelet/kubelet_pods.go#L557-L558
 	container.EnvFrom = []corev1.EnvFromSource{}
 
-	res := make([]corev1.EnvVar, 0)
+	res := make([]corev1.EnvVar, 0, len(tmpEnv))
 
 	for key, val := range tmpEnv {
 		res = append(res, corev1.EnvVar{
@@ -140,7 +135,7 @@ func getServiceEnvVarMap(rm *manager.ResourceManager, ns string, enableServiceLi
 	for i := range services {
 		service := services[i]
 		// ignore services where ClusterIP is "None" or empty
-		if !v1helper.IsServiceIPSet(service) {
+		if !IsServiceIPSet(service) {
 			continue
 		}
 		serviceName := service.Name
@@ -163,7 +158,7 @@ func getServiceEnvVarMap(rm *manager.ResourceManager, ns string, enableServiceLi
 		mappedServices = append(mappedServices, serviceMap[key])
 	}
 
-	for _, e := range envvars.FromServices(mappedServices) {
+	for _, e := range FromServices(mappedServices) {
 		m[e.Name] = e.Value
 	}
 	return m, nil
@@ -172,7 +167,7 @@ func getServiceEnvVarMap(rm *manager.ResourceManager, ns string, enableServiceLi
 // makeEnvironmentMapBasedOnEnvFrom returns a map representing the resolved environment of the specified container after being populated from the entries in the ".envFrom" field.
 func makeEnvironmentMapBasedOnEnvFrom(ctx context.Context, pod *corev1.Pod, container *corev1.Container, rm *manager.ResourceManager, recorder record.EventRecorder) (map[string]string, error) {
 	// Create a map to hold the resulting environment.
-	res := make(map[string]string, 0)
+	res := make(map[string]string)
 	// Iterate over "envFrom" references in order to populate the environment.
 loop:
 	for _, envFrom := range container.EnvFrom {
@@ -293,7 +288,6 @@ loop:
 
 // makeEnvironmentMap returns a map representing the resolved environment of the specified container after being populated from the entries in the ".env" and ".envFrom" field.
 func makeEnvironmentMap(ctx context.Context, pod *corev1.Pod, container *corev1.Container, rm *manager.ResourceManager, recorder record.EventRecorder, res map[string]string) error {
-
 	// TODO If pod.Spec.EnableServiceLinks is nil then fail as per 1.14 kubelet.
 	enableServiceLinks := corev1.DefaultEnableServiceLinks
 	if pod.Spec.EnableServiceLinks != nil {
@@ -315,136 +309,13 @@ func makeEnvironmentMap(ctx context.Context, pod *corev1.Pod, container *corev1.
 	mappingFunc := expansion.MappingFuncFor(res, svcEnv)
 
 	// Iterate over environment variables in order to populate the map.
-loop:
 	for _, env := range container.Env {
-		switch {
+		val, err := getEnvironmentVariableValue(ctx, &env, mappingFunc, pod, container, rm, recorder)
-		// Handle values that have been directly provided.
+		if err != nil {
-		case env.Value != "":
+			return err
-			// Expand variable references
+		}
-			res[env.Name] = expansion.Expand(env.Value, mappingFunc)
+		if val != nil {
-			continue loop
+			res[env.Name] = *val
-		// Handle population from a configmap key.
-		case env.ValueFrom != nil && env.ValueFrom.ConfigMapKeyRef != nil:
-			// The environment variable must be set from a configmap.
-			vf := env.ValueFrom.ConfigMapKeyRef
-			// Check whether the key reference is optional.
-			// This will control whether we fail when unable to read the requested key.
-			optional := vf != nil && vf.Optional != nil && *vf.Optional
-			// Try to grab the referenced configmap.
-			m, err := rm.GetConfigMap(vf.Name, pod.Namespace)
-			if err != nil {
-				// We couldn't fetch the configmap.
-				// However, if the key reference is optional we should not fail.
-				if optional {
-					if errors.IsNotFound(err) {
-						recorder.Eventf(pod, corev1.EventTypeWarning, ReasonOptionalConfigMapNotFound, "skipping optional envvar %q: configmap %q not found", env.Name, vf.Name)
-					} else {
-						log.G(ctx).Warnf("failed to read configmap %q: %v", vf.Name, err)
-						recorder.Eventf(pod, corev1.EventTypeWarning, ReasonFailedToReadOptionalConfigMap, "skipping optional envvar %q: failed to read configmap %q", env.Name, vf.Name)
-					}
-					// Continue on to the next reference.
-					continue loop
-				}
-				// At this point we know the key reference is mandatory.
-				// Hence, we should return a meaningful error.
-				if errors.IsNotFound(err) {
-					recorder.Eventf(pod, corev1.EventTypeWarning, ReasonMandatoryConfigMapNotFound, "configmap %q not found", vf.Name)
-					return fmt.Errorf("configmap %q not found", vf.Name)
-				}
-				recorder.Eventf(pod, corev1.EventTypeWarning, ReasonFailedToReadMandatoryConfigMap, "failed to read configmap %q", vf.Name)
-				return fmt.Errorf("failed to read configmap %q: %v", vf.Name, err)
-			}
-			// At this point we have successfully fetched the target configmap.
-			// We must now try to grab the requested key.
-			var (
-				keyExists bool
-				keyValue  string
-			)
-			if keyValue, keyExists = m.Data[vf.Key]; !keyExists {
-				// The requested key does not exist.
-				// However, we should not fail if the key reference is optional.
-				if optional {
-					// Continue on to the next reference.
-					recorder.Eventf(pod, corev1.EventTypeWarning, ReasonOptionalConfigMapKeyNotFound, "skipping optional envvar %q: key %q does not exist in configmap %q", env.Name, vf.Key, vf.Name)
-					continue loop
-				}
-				// At this point we know the key reference is mandatory.
-				// Hence, we should fail.
-				recorder.Eventf(pod, corev1.EventTypeWarning, ReasonMandatoryConfigMapKeyNotFound, "key %q does not exist in configmap %q", vf.Key, vf.Name)
-				return fmt.Errorf("configmap %q doesn't contain the %q key required by pod %s", vf.Name, vf.Key, pod.Name)
-			}
-			// Populate the environment variable and continue on to the next reference.
-			res[env.Name] = keyValue
-			continue loop
-		// Handle population from a secret key.
-		case env.ValueFrom != nil && env.ValueFrom.SecretKeyRef != nil:
-			vf := env.ValueFrom.SecretKeyRef
-			// Check whether the key reference is optional.
-			// This will control whether we fail when unable to read the requested key.
-			optional := vf != nil && vf.Optional != nil && *vf.Optional
-			// Try to grab the referenced secret.
-			s, err := rm.GetSecret(vf.Name, pod.Namespace)
-			if err != nil {
-				// We couldn't fetch the secret.
-				// However, if the key reference is optional we should not fail.
-				if optional {
-					if errors.IsNotFound(err) {
-						recorder.Eventf(pod, corev1.EventTypeWarning, ReasonOptionalSecretNotFound, "skipping optional envvar %q: secret %q not found", env.Name, vf.Name)
-					} else {
-						log.G(ctx).Warnf("failed to read secret %q: %v", vf.Name, err)
-						recorder.Eventf(pod, corev1.EventTypeWarning, ReasonFailedToReadOptionalSecret, "skipping optional envvar %q: failed to read secret %q", env.Name, vf.Name)
-					}
-					// Continue on to the next reference.
-					continue loop
-				}
-				// At this point we know the key reference is mandatory.
-				// Hence, we should return a meaningful error.
-				if errors.IsNotFound(err) {
-					recorder.Eventf(pod, corev1.EventTypeWarning, ReasonMandatorySecretNotFound, "secret %q not found", vf.Name)
-					return fmt.Errorf("secret %q not found", vf.Name)
-				}
-				recorder.Eventf(pod, corev1.EventTypeWarning, ReasonFailedToReadMandatorySecret, "failed to read secret %q", vf.Name)
-				return fmt.Errorf("failed to read secret %q: %v", vf.Name, err)
-			}
-			// At this point we have successfully fetched the target secret.
-			// We must now try to grab the requested key.
-			var (
-				keyExists bool
-				keyValue  []byte
-			)
-			if keyValue, keyExists = s.Data[vf.Key]; !keyExists {
-				// The requested key does not exist.
-				// However, we should not fail if the key reference is optional.
-				if optional {
-					// Continue on to the next reference.
-					recorder.Eventf(pod, corev1.EventTypeWarning, ReasonOptionalSecretKeyNotFound, "skipping optional envvar %q: key %q does not exist in secret %q", env.Name, vf.Key, vf.Name)
-					continue loop
-				}
-				// At this point we know the key reference is mandatory.
-				// Hence, we should fail.
-				recorder.Eventf(pod, corev1.EventTypeWarning, ReasonMandatorySecretKeyNotFound, "key %q does not exist in secret %q", vf.Key, vf.Name)
-				return fmt.Errorf("secret %q doesn't contain the %q key required by pod %s", vf.Name, vf.Key, pod.Name)
-			}
-			// Populate the environment variable and continue on to the next reference.
-			res[env.Name] = string(keyValue)
-			continue loop
-		// Handle population from a field (downward API).
-		case env.ValueFrom != nil && env.ValueFrom.FieldRef != nil:
-			// https://github.com/virtual-kubelet/virtual-kubelet/issues/123
-			vf := env.ValueFrom.FieldRef
-
-			runtimeVal, err := podFieldSelectorRuntimeValue(vf, pod)
-			if err != nil {
-				return err
-			}
-
-			res[env.Name] = runtimeVal
-
-			continue loop
-		// Handle population from a resource request/limit.
-		case env.ValueFrom != nil && env.ValueFrom.ResourceFieldRef != nil:
-			// TODO Implement populating resource requests.
-			continue loop
 		}
 	}
 
@@ -458,10 +329,160 @@ loop:
 	return nil
 }
 
+func getEnvironmentVariableValue(ctx context.Context, env *corev1.EnvVar, mappingFunc func(string) string, pod *corev1.Pod, container *corev1.Container, rm *manager.ResourceManager, recorder record.EventRecorder) (*string, error) {
+	if env.ValueFrom != nil {
+		return getEnvironmentVariableValueWithValueFrom(ctx, env, mappingFunc, pod, container, rm, recorder)
+	}
+	// Handle values that have been directly provided after expanding variable references.
+	return pointer.String(expansion.Expand(env.Value, mappingFunc)), nil
+}
+
+func getEnvironmentVariableValueWithValueFrom(ctx context.Context, env *corev1.EnvVar, mappingFunc func(string) string, pod *corev1.Pod, container *corev1.Container, rm *manager.ResourceManager, recorder record.EventRecorder) (*string, error) {
+	// Handle population from a configmap key.
+	if env.ValueFrom.ConfigMapKeyRef != nil {
+		return getEnvironmentVariableValueWithValueFromConfigMapKeyRef(ctx, env, mappingFunc, pod, container, rm, recorder)
+	}
+
+	// Handle population from a secret key.
+	if env.ValueFrom.SecretKeyRef != nil {
+		return getEnvironmentVariableValueWithValueFromSecretKeyRef(ctx, env, mappingFunc, pod, container, rm, recorder)
+	}
+
+	// Handle population from a field (downward API).
+	if env.ValueFrom.FieldRef != nil {
+		return getEnvironmentVariableValueWithValueFromFieldRef(ctx, env, mappingFunc, pod, container, rm, recorder)
+	}
+	if env.ValueFrom.ResourceFieldRef != nil {
+		// TODO Implement populating resource requests.
+		return nil, nil
+	}
+
+	log.G(ctx).WithField("env", env).Error("Unhandled environment variable with non-nil env.ValueFrom, do not know how to populate")
+	return nil, nil
+}
+
+func getEnvironmentVariableValueWithValueFromConfigMapKeyRef(ctx context.Context, env *corev1.EnvVar, mappingFunc func(string) string, pod *corev1.Pod, container *corev1.Container, rm *manager.ResourceManager, recorder record.EventRecorder) (*string, error) {
+	// The environment variable must be set from a configmap.
+	vf := env.ValueFrom.ConfigMapKeyRef
+	// Check whether the key reference is optional.
+	// This will control whether we fail when unable to read the requested key.
+	optional := vf != nil && vf.Optional != nil && *vf.Optional
+	// Try to grab the referenced configmap.
+	m, err := rm.GetConfigMap(vf.Name, pod.Namespace)
+	if err != nil {
+		// We couldn't fetch the configmap.
+		// However, if the key reference is optional we should not fail.
+		if optional {
+			if errors.IsNotFound(err) {
+				recorder.Eventf(pod, corev1.EventTypeWarning, ReasonOptionalConfigMapNotFound, "skipping optional envvar %q: configmap %q not found", env.Name, vf.Name)
+			} else {
+				log.G(ctx).Warnf("failed to read configmap %q: %v", vf.Name, err)
+				recorder.Eventf(pod, corev1.EventTypeWarning, ReasonFailedToReadOptionalConfigMap, "skipping optional envvar %q: failed to read configmap %q", env.Name, vf.Name)
+			}
+			// Continue on to the next reference.
+			return nil, nil
+		}
+		// At this point we know the key reference is mandatory.
+		// Hence, we should return a meaningful error.
+		if errors.IsNotFound(err) {
+			recorder.Eventf(pod, corev1.EventTypeWarning, ReasonMandatoryConfigMapNotFound, "configmap %q not found", vf.Name)
+			return nil, fmt.Errorf("configmap %q not found", vf.Name)
+		}
+		recorder.Eventf(pod, corev1.EventTypeWarning, ReasonFailedToReadMandatoryConfigMap, "failed to read configmap %q", vf.Name)
+		return nil, fmt.Errorf("failed to read configmap %q: %v", vf.Name, err)
+	}
+	// At this point we have successfully fetched the target configmap.
+	// We must now try to grab the requested key.
+	var (
+		keyExists bool
+		keyValue  string
+	)
+	if keyValue, keyExists = m.Data[vf.Key]; !keyExists {
+		// The requested key does not exist.
+		// However, we should not fail if the key reference is optional.
+		if optional {
+			// Continue on to the next reference.
+			recorder.Eventf(pod, corev1.EventTypeWarning, ReasonOptionalConfigMapKeyNotFound, "skipping optional envvar %q: key %q does not exist in configmap %q", env.Name, vf.Key, vf.Name)
+			return nil, nil
+		}
+		// At this point we know the key reference is mandatory.
+		// Hence, we should fail.
+		recorder.Eventf(pod, corev1.EventTypeWarning, ReasonMandatoryConfigMapKeyNotFound, "key %q does not exist in configmap %q", vf.Key, vf.Name)
+		return nil, fmt.Errorf("configmap %q doesn't contain the %q key required by pod %s", vf.Name, vf.Key, pod.Name)
+	}
+	// Populate the environment variable and continue on to the next reference.
+	return pointer.String(keyValue), nil
+}
+
+func getEnvironmentVariableValueWithValueFromSecretKeyRef(ctx context.Context, env *corev1.EnvVar, mappingFunc func(string) string, pod *corev1.Pod, container *corev1.Container, rm *manager.ResourceManager, recorder record.EventRecorder) (*string, error) {
+	vf := env.ValueFrom.SecretKeyRef
+	// Check whether the key reference is optional.
+	// This will control whether we fail when unable to read the requested key.
+	optional := vf != nil && vf.Optional != nil && *vf.Optional
+	// Try to grab the referenced secret.
+	s, err := rm.GetSecret(vf.Name, pod.Namespace)
+	if err != nil {
+		// We couldn't fetch the secret.
+		// However, if the key reference is optional we should not fail.
+		if optional {
+			if errors.IsNotFound(err) {
+				recorder.Eventf(pod, corev1.EventTypeWarning, ReasonOptionalSecretNotFound, "skipping optional envvar %q: secret %q not found", env.Name, vf.Name)
+			} else {
+				log.G(ctx).Warnf("failed to read secret %q: %v", vf.Name, err)
+				recorder.Eventf(pod, corev1.EventTypeWarning, ReasonFailedToReadOptionalSecret, "skipping optional envvar %q: failed to read secret %q", env.Name, vf.Name)
+			}
+			// Continue on to the next reference.
+			return nil, nil
+		}
+		// At this point we know the key reference is mandatory.
+		// Hence, we should return a meaningful error.
+		if errors.IsNotFound(err) {
+			recorder.Eventf(pod, corev1.EventTypeWarning, ReasonMandatorySecretNotFound, "secret %q not found", vf.Name)
+			return nil, fmt.Errorf("secret %q not found", vf.Name)
+		}
+		recorder.Eventf(pod, corev1.EventTypeWarning, ReasonFailedToReadMandatorySecret, "failed to read secret %q", vf.Name)
+		return nil, fmt.Errorf("failed to read secret %q: %v", vf.Name, err)
+	}
+	// At this point we have successfully fetched the target secret.
+	// We must now try to grab the requested key.
+	var (
+		keyExists bool
+		keyValue  []byte
+	)
+	if keyValue, keyExists = s.Data[vf.Key]; !keyExists {
+		// The requested key does not exist.
+		// However, we should not fail if the key reference is optional.
+		if optional {
+			// Continue on to the next reference.
+			recorder.Eventf(pod, corev1.EventTypeWarning, ReasonOptionalSecretKeyNotFound, "skipping optional envvar %q: key %q does not exist in secret %q", env.Name, vf.Key, vf.Name)
+			return nil, nil
+		}
+		// At this point we know the key reference is mandatory.
+		// Hence, we should fail.
+		recorder.Eventf(pod, corev1.EventTypeWarning, ReasonMandatorySecretKeyNotFound, "key %q does not exist in secret %q", vf.Key, vf.Name)
+		return nil, fmt.Errorf("secret %q doesn't contain the %q key required by pod %s", vf.Name, vf.Key, pod.Name)
+	}
+	// Populate the environment variable and continue on to the next reference.
+	return pointer.String(string(keyValue)), nil
+}
+
+// Handle population from a field (downward API).
+func getEnvironmentVariableValueWithValueFromFieldRef(ctx context.Context, env *corev1.EnvVar, mappingFunc func(string) string, pod *corev1.Pod, container *corev1.Container, rm *manager.ResourceManager, recorder record.EventRecorder) (*string, error) {
+	// https://github.com/virtual-kubelet/virtual-kubelet/issues/123
+	vf := env.ValueFrom.FieldRef
+
+	runtimeVal, err := podFieldSelectorRuntimeValue(vf, pod)
+	if err != nil {
+		return nil, err
+	}
+
+	return pointer.String(runtimeVal), nil
+}
+
 // podFieldSelectorRuntimeValue returns the runtime value of the given
 // selector for a pod.
 func podFieldSelectorRuntimeValue(fs *corev1.ObjectFieldSelector, pod *corev1.Pod) (string, error) {
-	internalFieldPath, _, err := podshelper.ConvertDownwardAPIFieldLabel(fs.APIVersion, fs.FieldPath, "")
+	internalFieldPath, _, err := ConvertDownwardAPIFieldLabel(fs.APIVersion, fs.FieldPath, "")
 	if err != nil {
 		return "", err
 	}
@@ -472,5 +493,5 @@ func podFieldSelectorRuntimeValue(fs *corev1.ObjectFieldSelector, pod *corev1.Po
 		return pod.Spec.ServiceAccountName, nil
 
 	}
-	return fieldpath.ExtractFieldPathAsString(pod, internalFieldPath)
+	return ExtractFieldPathAsString(pod, internalFieldPath)
 }

internal/podutils/env_internal_test.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package node
+package podutils
 
 import (
 	"context"
@@ -215,7 +215,7 @@ func TestPopulatePodWithInitContainersUsingEnv(t *testing.T) {
 	}
 
 	// Populate the pod's environment.
-	err := populateEnvironmentVariables(context.Background(), pod, rm, er)
+	err := PopulateEnvironmentVariables(context.Background(), pod, rm, er)
 	assert.Check(t, err)
 
 	// Make sure that all the containers' environments contain all the expected keys and values.
@@ -375,7 +375,7 @@ func TestPopulatePodWithInitContainersUsingEnvWithFieldRef(t *testing.T) {
 	}
 
 	// Populate the pod's environment.
-	err := populateEnvironmentVariables(context.Background(), pod, rm, er)
+	err := PopulateEnvironmentVariables(context.Background(), pod, rm, er)
 	assert.NilError(t, err)
 
 	// Make sure that all the containers' environments contain all the expected keys and values.
@@ -491,7 +491,7 @@ func TestPopulatePodWithInitContainersUsingEnvFrom(t *testing.T) {
 	}
 
 	// Populate the pod's environment.
-	err := populateEnvironmentVariables(context.Background(), pod, rm, er)
+	err := PopulateEnvironmentVariables(context.Background(), pod, rm, er)
 	assert.Check(t, err)
 
 	// Make sure that all the containers' environments contain all the expected keys and values.
@@ -632,7 +632,7 @@ func TestEnvFromConfigMapAndSecretWithInvalidKeys(t *testing.T) {
 	}
 
 	// Populate the pods's environment.
-	err := populateEnvironmentVariables(context.Background(), pod, rm, er)
+	err := PopulateEnvironmentVariables(context.Background(), pod, rm, er)
 	assert.Check(t, err)
 
 	// Make sure that the container's environment has two variables (corresponding to the single valid key in both the configmap and the secret).
@@ -663,7 +663,7 @@ func TestEnvFromConfigMapAndSecretWithInvalidKeys(t *testing.T) {
 }
 
 // TestEnvOverridesEnvFrom populates the environment of a container from a configmap, and from another configmap's key with a "conflicting" key.
-// Then, it checks that the value of the "conflicting" key has been correctly overriden.
+// Then, it checks that the value of the "conflicting" key has been correctly overridden.
 func TestEnvOverridesEnvFrom(t *testing.T) {
 	rm := testutil.FakeResourceManager(configMap3)
 	er := testutil.FakeEventRecorder(defaultEventRecorderBufferSize)
@@ -672,7 +672,7 @@ func TestEnvOverridesEnvFrom(t *testing.T) {
 	override := "__override__"
 
 	// Create a pod object having a single container.
-	// The container's environment is to be populated from a configmap, and later overriden with a value provided directly.
+	// The container's environment is to be populated from a configmap, and later overridden with a value provided directly.
 	pod := &corev1.Pod{
 		ObjectMeta: metav1.ObjectMeta{
 			Namespace: namespace,
@@ -703,7 +703,7 @@ func TestEnvOverridesEnvFrom(t *testing.T) {
 	}
 
 	// Populate the pods's environment.
-	err := populateEnvironmentVariables(context.Background(), pod, rm, er)
+	err := PopulateEnvironmentVariables(context.Background(), pod, rm, er)
 	assert.Check(t, err)
 
 	// Make sure that the container's environment contains all the expected keys and values.
@@ -780,7 +780,7 @@ func TestEnvFromInexistentConfigMaps(t *testing.T) {
 	}
 
 	// Populate the pods's environment.
-	err := populateEnvironmentVariables(context.Background(), pod, rm, er)
+	err := PopulateEnvironmentVariables(context.Background(), pod, rm, er)
 	assert.Check(t, is.ErrorContains(err, ""))
 
 	// Make sure that two events have been recorded with the correct reason and message.
@@ -837,7 +837,7 @@ func TestEnvFromInexistentSecrets(t *testing.T) {
 	}
 
 	// Populate the pods's environment.
-	err := populateEnvironmentVariables(context.Background(), pod, rm, er)
+	err := PopulateEnvironmentVariables(context.Background(), pod, rm, er)
 	assert.Check(t, is.ErrorContains(err, ""))
 
 	// Make sure that two events have been recorded with the correct reason and message.
@@ -877,7 +877,7 @@ func TestEnvReferencingInexistentConfigMapKey(t *testing.T) {
 									},
 									Key: "key",
 									// This scenario has been observed before https://github.com/virtual-kubelet/virtual-kubelet/issues/444#issuecomment-449611851.
-									// A nil value of optional means "mandatory", hence we should expect "populateEnvironmentVariables" to return an error.
+									// A nil value of optional means "mandatory", hence we should expect "PopulateEnvironmentVariables" to return an error.
 									Optional: nil,
 								},
 							},
@@ -890,7 +890,7 @@ func TestEnvReferencingInexistentConfigMapKey(t *testing.T) {
 	}
 
 	// Populate the pods's environment.
-	err := populateEnvironmentVariables(context.Background(), pod, rm, er)
+	err := PopulateEnvironmentVariables(context.Background(), pod, rm, er)
 	assert.Check(t, is.ErrorContains(err, ""))
 
 	// Make sure that two events have been recorded with the correct reason and message.
@@ -927,7 +927,7 @@ func TestEnvReferencingInexistentSecretKey(t *testing.T) {
 									},
 									Key: "key",
 									// This scenario has been observed before https://github.com/virtual-kubelet/virtual-kubelet/issues/444#issuecomment-449611851.
-									// A nil value of optional means "mandatory", hence we should expect "populateEnvironmentVariables" to return an error.
+									// A nil value of optional means "mandatory", hence we should expect "PopulateEnvironmentVariables" to return an error.
 									Optional: nil,
 								},
 							},
@@ -940,7 +940,7 @@ func TestEnvReferencingInexistentSecretKey(t *testing.T) {
 	}
 
 	// Populate the pods's environment.
-	err := populateEnvironmentVariables(context.Background(), pod, rm, er)
+	err := PopulateEnvironmentVariables(context.Background(), pod, rm, er)
 	assert.Check(t, is.ErrorContains(err, ""))
 
 	// Make sure that two events have been recorded with the correct reason and message.
@@ -1023,7 +1023,7 @@ func TestServiceEnvVar(t *testing.T) {
 	for _, tc := range testCases {
 		pod.Spec.EnableServiceLinks = tc.enableServiceLinks
 
-		err := populateEnvironmentVariables(context.Background(), pod, rm, er)
+		err := PopulateEnvironmentVariables(context.Background(), pod, rm, er)
 		assert.NilError(t, err, "[%s]", tc.name)
 		assert.Check(t, is.DeepEqual(pod.Spec.Containers[0].Env, tc.expectedEnvs, sortOpt))
 	}
@@ -1066,7 +1066,7 @@ func TestComposingEnv(t *testing.T) {
 	}
 
 	// Populate the pods's environment.
-	err := populateEnvironmentVariables(context.Background(), pod, rm, er)
+	err := PopulateEnvironmentVariables(context.Background(), pod, rm, er)
 	assert.Check(t, err)
 
 	// Make sure that the container's environment contains all the expected keys and values.
@@ -1090,3 +1090,45 @@ func TestComposingEnv(t *testing.T) {
 	// Make sure that no events have been recorded.
 	assert.Check(t, is.Len(er.Events, 0))
 }
+
+// TestEmptyEnvVar tests that env var can be have the value ""
+func TestEmptyEnvVar(t *testing.T) {
+	rm := testutil.FakeResourceManager()
+	er := testutil.FakeEventRecorder(defaultEventRecorderBufferSize)
+
+	// Create a pod object having a single container.
+	// The container's third environment variable is composed of the previous two.
+	pod := &corev1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: namespace,
+			Name:      "pod-0",
+		},
+		Spec: corev1.PodSpec{
+			Containers: []corev1.Container{
+				{
+					Env: []corev1.EnvVar{
+						{
+							Name: envVarName1,
+						},
+					},
+				},
+			},
+		},
+	}
+
+	// Populate the pods's environment.
+	err := PopulateEnvironmentVariables(context.Background(), pod, rm, er)
+	assert.Check(t, err)
+
+	// Make sure that the container's environment contains all the expected keys and values.
+	assert.Check(t, is.DeepEqual(pod.Spec.Containers[0].Env, []corev1.EnvVar{
+		{
+			Name: envVarName1,
+		},
+	},
+		sortOpt,
+	))
+
+	// Make sure that no events have been recorded.
+	assert.Check(t, is.Len(er.Events, 0))
+}

internal/podutils/envvars.go

@@ -0,0 +1,112 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package podutils
+
+import (
+	"fmt"
+	"net"
+	"strconv"
+	"strings"
+
+	v1 "k8s.io/api/core/v1"
+)
+
+// FromServices builds environment variables that a container is started with,
+// which tell the container where to find the services it may need, which are
+// provided as an argument.
+func FromServices(services []*v1.Service) []v1.EnvVar {
+	var result []v1.EnvVar
+	for i := range services {
+		service := services[i]
+
+		// ignore services where ClusterIP is "None" or empty
+		// the services passed to this method should be pre-filtered
+		// only services that have the cluster IP set should be included here
+		if !IsServiceIPSet(service) {
+			continue
+		}
+
+		// Host
+		name := makeEnvVariableName(service.Name) + "_SERVICE_HOST"
+		result = append(result, v1.EnvVar{Name: name, Value: service.Spec.ClusterIP})
+		// First port - give it the backwards-compatible name
+		name = makeEnvVariableName(service.Name) + "_SERVICE_PORT"
+		result = append(result, v1.EnvVar{Name: name, Value: strconv.Itoa(int(service.Spec.Ports[0].Port))})
+		// All named ports (only the first may be unnamed, checked in validation)
+		for i := range service.Spec.Ports {
+			sp := &service.Spec.Ports[i]
+			if sp.Name != "" {
+				pn := name + "_" + makeEnvVariableName(sp.Name)
+				result = append(result, v1.EnvVar{Name: pn, Value: strconv.Itoa(int(sp.Port))})
+			}
+		}
+		// Docker-compatible vars.
+		result = append(result, makeLinkVariables(service)...)
+	}
+	return result
+}
+
+func makeEnvVariableName(str string) string {
+	// TODO: If we simplify to "all names are DNS1123Subdomains" this
+	// will need two tweaks:
+	//   1) Handle leading digits
+	//   2) Handle dots
+	return strings.ToUpper(strings.Replace(str, "-", "_", -1))
+}
+
+func makeLinkVariables(service *v1.Service) []v1.EnvVar {
+	prefix := makeEnvVariableName(service.Name)
+	all := []v1.EnvVar{}
+	for i := range service.Spec.Ports {
+		sp := &service.Spec.Ports[i]
+
+		protocol := string(v1.ProtocolTCP)
+		if sp.Protocol != "" {
+			protocol = string(sp.Protocol)
+		}
+
+		hostPort := net.JoinHostPort(service.Spec.ClusterIP, strconv.Itoa(int(sp.Port)))
+
+		if i == 0 {
+			// Docker special-cases the first port.
+			all = append(all, v1.EnvVar{
+				Name:  prefix + "_PORT",
+				Value: fmt.Sprintf("%s://%s", strings.ToLower(protocol), hostPort),
+			})
+		}
+		portPrefix := fmt.Sprintf("%s_PORT_%d_%s", prefix, sp.Port, strings.ToUpper(protocol))
+		all = append(all, []v1.EnvVar{
+			{
+				Name:  portPrefix,
+				Value: fmt.Sprintf("%s://%s", strings.ToLower(protocol), hostPort),
+			},
+			{
+				Name:  portPrefix + "_PROTO",
+				Value: strings.ToLower(protocol),
+			},
+			{
+				Name:  portPrefix + "_PORT",
+				Value: strconv.Itoa(int(sp.Port)),
+			},
+			{
+				Name:  portPrefix + "_ADDR",
+				Value: service.Spec.ClusterIP,
+			},
+		}...)
+	}
+	return all
+}

internal/podutils/helper.go

@@ -0,0 +1,156 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package podutils
+
+import (
+	"fmt"
+	"strings"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/meta"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apimachinery/pkg/util/validation"
+)
+
+// ConvertDownwardAPIFieldLabel converts the specified downward API field label
+// and its value in the pod of the specified version to the internal version,
+// and returns the converted label and value. This function returns an error if
+// the conversion fails.
+func ConvertDownwardAPIFieldLabel(version, label, value string) (string, string, error) {
+	if version != "v1" {
+		return "", "", fmt.Errorf("unsupported pod version: %s", version)
+	}
+
+	if path, _, ok := SplitMaybeSubscriptedPath(label); ok {
+		switch path {
+		case "metadata.annotations", "metadata.labels":
+			return label, value, nil
+		default:
+			return "", "", fmt.Errorf("field label does not support subscript: %s", label)
+		}
+	}
+
+	switch label {
+	case "metadata.annotations",
+		"metadata.labels",
+		"metadata.name",
+		"metadata.namespace",
+		"metadata.uid",
+		"spec.nodeName",
+		"spec.restartPolicy",
+		"spec.serviceAccountName",
+		"spec.schedulerName",
+		"status.phase",
+		"status.hostIP",
+		"status.podIP",
+		"status.podIPs":
+		return label, value, nil
+	// This is for backwards compatibility with old v1 clients which send spec.host
+	case "spec.host":
+		return "spec.nodeName", value, nil
+	default:
+		return "", "", fmt.Errorf("field label not supported: %s", label)
+	}
+}
+
+// ExtractFieldPathAsString extracts the field from the given object
+// and returns it as a string.  The object must be a pointer to an
+// API type.
+func ExtractFieldPathAsString(obj interface{}, fieldPath string) (string, error) {
+	accessor, err := meta.Accessor(obj)
+	if err != nil {
+		return "", err
+	}
+
+	if path, subscript, ok := SplitMaybeSubscriptedPath(fieldPath); ok {
+		switch path {
+		case "metadata.annotations":
+			if errs := validation.IsQualifiedName(strings.ToLower(subscript)); len(errs) != 0 {
+				return "", fmt.Errorf("invalid key subscript in %s: %s", fieldPath, strings.Join(errs, ";"))
+			}
+			return accessor.GetAnnotations()[subscript], nil
+		case "metadata.labels":
+			if errs := validation.IsQualifiedName(subscript); len(errs) != 0 {
+				return "", fmt.Errorf("invalid key subscript in %s: %s", fieldPath, strings.Join(errs, ";"))
+			}
+			return accessor.GetLabels()[subscript], nil
+		default:
+			return "", fmt.Errorf("fieldPath %q does not support subscript", fieldPath)
+		}
+	}
+
+	switch fieldPath {
+	case "metadata.annotations":
+		return FormatMap(accessor.GetAnnotations()), nil
+	case "metadata.labels":
+		return FormatMap(accessor.GetLabels()), nil
+	case "metadata.name":
+		return accessor.GetName(), nil
+	case "metadata.namespace":
+		return accessor.GetNamespace(), nil
+	case "metadata.uid":
+		return string(accessor.GetUID()), nil
+	}
+
+	return "", fmt.Errorf("unsupported fieldPath: %v", fieldPath)
+}
+
+// SplitMaybeSubscriptedPath checks whether the specified fieldPath is
+// subscripted, and
+//  - if yes, this function splits the fieldPath into path and subscript, and
+//    returns (path, subscript, true).
+//  - if no, this function returns (fieldPath, "", false).
+//
+// Example inputs and outputs:
+//  - "metadata.annotations['myKey']" --> ("metadata.annotations", "myKey", true)
+//  - "metadata.annotations['a[b]c']" --> ("metadata.annotations", "a[b]c", true)
+//  - "metadata.labels['']"           --> ("metadata.labels", "", true)
+//  - "metadata.labels"               --> ("metadata.labels", "", false)
+func SplitMaybeSubscriptedPath(fieldPath string) (string, string, bool) {
+	if !strings.HasSuffix(fieldPath, "']") {
+		return fieldPath, "", false
+	}
+	s := strings.TrimSuffix(fieldPath, "']")
+	parts := strings.SplitN(s, "['", 2)
+	if len(parts) < 2 {
+		return fieldPath, "", false
+	}
+	if len(parts[0]) == 0 {
+		return fieldPath, "", false
+	}
+	return parts[0], parts[1], true
+}
+
+// FormatMap formats map[string]string to a string.
+func FormatMap(m map[string]string) (fmtStr string) {
+	// output with keys in sorted order to provide stable output
+	keys := sets.NewString()
+	for key := range m {
+		keys.Insert(key)
+	}
+	for _, key := range keys.List() {
+		fmtStr += fmt.Sprintf("%v=%q\n", key, m[key])
+	}
+	fmtStr = strings.TrimSuffix(fmtStr, "\n")
+
+	return
+}
+
+// IsServiceIPSet aims to check if the service's ClusterIP is set or not the objective is not to perform validation here
+func IsServiceIPSet(service *corev1.Service) bool {
+	return service.Spec.ClusterIP != corev1.ClusterIPNone && service.Spec.ClusterIP != ""
+}

internal/queue/queue.go

@@ -0,0 +1,502 @@
+// Copyright © 2017 The virtual-kubelet authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package queue
+
+import (
+	"container/list"
+	"context"
+	"fmt"
+	"sync"
+	"time"
+
+	pkgerrors "github.com/pkg/errors"
+	"github.com/virtual-kubelet/virtual-kubelet/log"
+	"github.com/virtual-kubelet/virtual-kubelet/trace"
+	"golang.org/x/sync/semaphore"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/util/workqueue"
+	"k8s.io/utils/clock"
+)
+
+const (
+	// MaxRetries is the number of times we try to process a given key before permanently forgetting it.
+	MaxRetries = 20
+)
+
+// ShouldRetryFunc is a mechanism to have a custom retry policy
+type ShouldRetryFunc func(ctx context.Context, key string, timesTried int, originallyAdded time.Time, err error) (*time.Duration, error)
+
+// ItemHandler is a callback that handles a single key on the Queue
+type ItemHandler func(ctx context.Context, key string) error
+
+// Queue implements a wrapper around workqueue with native VK instrumentation
+type Queue struct {
+	// clock is used for testing
+	clock clock.Clock
+	// lock protects running, and the items list / map
+	lock    sync.Mutex
+	running bool
+	name    string
+	handler ItemHandler
+
+	ratelimiter workqueue.RateLimiter
+	// items are items that are marked dirty waiting for processing.
+	items *list.List
+	// itemInQueue is a map of (string) key -> item while it is in the items list
+	itemsInQueue map[string]*list.Element
+	// itemsBeingProcessed is a map of (string) key -> item once it has been moved
+	itemsBeingProcessed map[string]*queueItem
+	// Wait for next semaphore is an exclusive (1 item) lock that is taken every time items is checked to see if there
+	// is an item in queue for work
+	waitForNextItemSemaphore *semaphore.Weighted
+
+	// wakeup
+	wakeupCh chan struct{}
+
+	retryFunc ShouldRetryFunc
+}
+
+type queueItem struct {
+	key                    string
+	plannedToStartWorkAt   time.Time
+	redirtiedAt            time.Time
+	redirtiedWithRatelimit bool
+	forget                 bool
+	requeues               int
+
+	// Debugging information only
+	originallyAdded     time.Time
+	addedViaRedirty     bool
+	delayedViaRateLimit *time.Duration
+}
+
+func (item *queueItem) String() string {
+	return fmt.Sprintf("<plannedToStartWorkAt:%s key: %s>", item.plannedToStartWorkAt.String(), item.key)
+}
+
+// New creates a queue
+//
+// It expects to get a item rate limiter, and a friendly name which is used in logs, and in the internal kubernetes
+// metrics. If retryFunc is nil, the default retry function.
+func New(ratelimiter workqueue.RateLimiter, name string, handler ItemHandler, retryFunc ShouldRetryFunc) *Queue {
+	if retryFunc == nil {
+		retryFunc = DefaultRetryFunc
+	}
+	return &Queue{
+		clock:                    clock.RealClock{},
+		name:                     name,
+		ratelimiter:              ratelimiter,
+		items:                    list.New(),
+		itemsBeingProcessed:      make(map[string]*queueItem),
+		itemsInQueue:             make(map[string]*list.Element),
+		handler:                  handler,
+		wakeupCh:                 make(chan struct{}, 1),
+		waitForNextItemSemaphore: semaphore.NewWeighted(1),
+		retryFunc:                retryFunc,
+	}
+}
+
+// Enqueue enqueues the key in a rate limited fashion
+func (q *Queue) Enqueue(ctx context.Context, key string) {
+	q.lock.Lock()
+	defer q.lock.Unlock()
+
+	q.insert(ctx, key, true, nil)
+}
+
+// EnqueueWithoutRateLimit enqueues the key without a rate limit
+func (q *Queue) EnqueueWithoutRateLimit(ctx context.Context, key string) {
+	q.lock.Lock()
+	defer q.lock.Unlock()
+
+	q.insert(ctx, key, false, nil)
+}
+
+// Forget forgets the key
+func (q *Queue) Forget(ctx context.Context, key string) {
+	q.lock.Lock()
+	defer q.lock.Unlock()
+	ctx, span := trace.StartSpan(ctx, "Forget")
+	defer span.End()
+
+	ctx = span.WithFields(ctx, map[string]interface{}{
+		"queue": q.name,
+		"key":   key,
+	})
+
+	if item, ok := q.itemsInQueue[key]; ok {
+		span.WithField(ctx, "status", "itemInQueue")
+		delete(q.itemsInQueue, key)
+		q.items.Remove(item)
+		return
+	}
+
+	if qi, ok := q.itemsBeingProcessed[key]; ok {
+		span.WithField(ctx, "status", "itemBeingProcessed")
+		qi.forget = true
+		return
+	}
+	span.WithField(ctx, "status", "notfound")
+}
+
+func durationDeref(duration *time.Duration, def time.Duration) time.Duration {
+	if duration == nil {
+		return def
+	}
+
+	return *duration
+}
+
+// insert inserts a new item to be processed at time time. It will not further delay items if when is later than the
+// original time the item was scheduled to be processed. If when is earlier, it will "bring it forward"
+// If ratelimit is specified, and delay is nil, then the ratelimiter's delay (return from When function) will be used
+// If ratelimit is specified, and the delay is non-nil, then the delay value will be used
+// If ratelimit is false, then only delay is used to schedule the work. If delay is nil, it will be considered 0.
+func (q *Queue) insert(ctx context.Context, key string, ratelimit bool, delay *time.Duration) *queueItem {
+	ctx, span := trace.StartSpan(ctx, "insert")
+	defer span.End()
+
+	ctx = span.WithFields(ctx, map[string]interface{}{
+		"queue":     q.name,
+		"key":       key,
+		"ratelimit": ratelimit,
+	})
+	if delay == nil {
+		ctx = span.WithField(ctx, "delay", "nil")
+	} else {
+		ctx = span.WithField(ctx, "delay", delay.String())
+	}
+
+	defer func() {
+		select {
+		case q.wakeupCh <- struct{}{}:
+		default:
+		}
+	}()
+
+	// First see if the item is already being processed
+	if item, ok := q.itemsBeingProcessed[key]; ok {
+		span.WithField(ctx, "status", "itemsBeingProcessed")
+		when := q.clock.Now().Add(durationDeref(delay, 0))
+		// Is the item already been redirtied?
+		if item.redirtiedAt.IsZero() {
+			item.redirtiedAt = when
+			item.redirtiedWithRatelimit = ratelimit
+		} else if when.Before(item.redirtiedAt) {
+			item.redirtiedAt = when
+			item.redirtiedWithRatelimit = ratelimit
+		}
+		item.forget = false
+		return item
+	}
+
+	// Is the item already in the queue?
+	if item, ok := q.itemsInQueue[key]; ok {
+		span.WithField(ctx, "status", "itemsInQueue")
+		qi := item.Value.(*queueItem)
+		when := q.clock.Now().Add(durationDeref(delay, 0))
+		q.adjustPosition(qi, item, when)
+		return qi
+	}
+
+	span.WithField(ctx, "status", "added")
+	now := q.clock.Now()
+	val := &queueItem{
+		key:                  key,
+		plannedToStartWorkAt: now,
+		originallyAdded:      now,
+	}
+
+	if ratelimit {
+		actualDelay := q.ratelimiter.When(key)
+		// Check if delay is overridden
+		if delay != nil {
+			actualDelay = *delay
+		}
+		span.WithField(ctx, "delay", actualDelay.String())
+		val.plannedToStartWorkAt = val.plannedToStartWorkAt.Add(actualDelay)
+		val.delayedViaRateLimit = &actualDelay
+	} else {
+		val.plannedToStartWorkAt = val.plannedToStartWorkAt.Add(durationDeref(delay, 0))
+	}
+
+	for item := q.items.Back(); item != nil; item = item.Prev() {
+		qi := item.Value.(*queueItem)
+		if qi.plannedToStartWorkAt.Before(val.plannedToStartWorkAt) {
+			q.itemsInQueue[key] = q.items.InsertAfter(val, item)
+			return val
+		}
+	}
+
+	q.itemsInQueue[key] = q.items.PushFront(val)
+	return val
+}
+
+func (q *Queue) adjustPosition(qi *queueItem, element *list.Element, when time.Time) {
+	if when.After(qi.plannedToStartWorkAt) {
+		// The item has already been delayed appropriately
+		return
+	}
+
+	qi.plannedToStartWorkAt = when
+	for prev := element.Prev(); prev != nil; prev = prev.Prev() {
+		item := prev.Value.(*queueItem)
+		// does this item plan to start work *before* the new time? If so add it
+		if item.plannedToStartWorkAt.Before(when) {
+			q.items.MoveAfter(element, prev)
+			return
+		}
+	}
+
+	q.items.MoveToFront(element)
+}
+
+// EnqueueWithoutRateLimitWithDelay enqueues without rate limiting, but work will not start for this given delay period
+func (q *Queue) EnqueueWithoutRateLimitWithDelay(ctx context.Context, key string, after time.Duration) {
+	q.lock.Lock()
+	defer q.lock.Unlock()
+	q.insert(ctx, key, false, &after)
+}
+
+// Empty returns if the queue has no items in it
+//
+// It should only be used for debugging.
+func (q *Queue) Empty() bool {
+	return q.Len() == 0
+}
+
+// Len includes items that are in the queue, and are being processed
+func (q *Queue) Len() int {
+	q.lock.Lock()
+	defer q.lock.Unlock()
+	if q.items.Len() != len(q.itemsInQueue) {
+		panic("Internally inconsistent state")
+	}
+
+	return q.items.Len() + len(q.itemsBeingProcessed)
+}
+
+// Run starts the workers
+//
+// It blocks until context is cancelled, and all of the workers exit.
+func (q *Queue) Run(ctx context.Context, workers int) {
+	if workers <= 0 {
+		panic(fmt.Sprintf("Workers must be greater than 0, got: %d", workers))
+	}
+
+	q.lock.Lock()
+	if q.running {
+		panic(fmt.Sprintf("Queue %s is already running", q.name))
+	}
+	q.running = true
+	q.lock.Unlock()
+	defer func() {
+		q.lock.Lock()
+		defer q.lock.Unlock()
+		q.running = false
+	}()
+
+	// Make sure all workers are stopped before we finish up.
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	group := &wait.Group{}
+	for i := 0; i < workers; i++ {
+		// This is required because i is referencing a mutable variable and that's running in a separate goroutine
+		idx := i
+		group.StartWithContext(ctx, func(ctx context.Context) {
+			q.worker(ctx, idx)
+		})
+	}
+	defer group.Wait()
+	<-ctx.Done()
+}
+
+func (q *Queue) worker(ctx context.Context, i int) {
+	ctx = log.WithLogger(ctx, log.G(ctx).WithFields(map[string]interface{}{
+		"workerId": i,
+		"queue":    q.name,
+	}))
+	for q.handleQueueItem(ctx) {
+	}
+}
+
+func (q *Queue) getNextItem(ctx context.Context) (*queueItem, error) {
+	if err := q.waitForNextItemSemaphore.Acquire(ctx, 1); err != nil {
+		return nil, err
+	}
+	defer q.waitForNextItemSemaphore.Release(1)
+
+	for {
+		q.lock.Lock()
+		element := q.items.Front()
+		if element == nil {
+			// Wait for the next item
+			q.lock.Unlock()
+			select {
+			case <-ctx.Done():
+				return nil, ctx.Err()
+			case <-q.wakeupCh:
+			}
+		} else {
+			qi := element.Value.(*queueItem)
+			timeUntilProcessing := time.Until(qi.plannedToStartWorkAt)
+
+			// Do we need to sleep? If not, let's party.
+			if timeUntilProcessing <= 0 {
+				q.itemsBeingProcessed[qi.key] = qi
+				q.items.Remove(element)
+				delete(q.itemsInQueue, qi.key)
+				q.lock.Unlock()
+				return qi, nil
+			}
+
+			q.lock.Unlock()
+			if err := func() error {
+				timer := q.clock.NewTimer(timeUntilProcessing)
+				defer timer.Stop()
+				select {
+				case <-timer.C():
+				case <-ctx.Done():
+					return ctx.Err()
+				case <-q.wakeupCh:
+				}
+				return nil
+			}(); err != nil {
+				return nil, err
+			}
+		}
+	}
+}
+
+// handleQueueItem handles a single item
+//
+// A return value of "false" indicates that further processing should be stopped.
+func (q *Queue) handleQueueItem(ctx context.Context) bool {
+	ctx, span := trace.StartSpan(ctx, "handleQueueItem")
+	defer span.End()
+
+	qi, err := q.getNextItem(ctx)
+	if err != nil {
+		span.SetStatus(err)
+		return false
+	}
+
+	// We expect strings to come off the work Queue.
+	// These are of the form namespace/name.
+	// We do this as the delayed nature of the work Queue means the items in the informer cache may actually be more u
+	// to date that when the item was initially put onto the workqueue.
+	ctx = span.WithField(ctx, "key", qi.key)
+	log.G(ctx).Debug("Got Queue object")
+
+	err = q.handleQueueItemObject(ctx, qi)
+	if err != nil {
+		// We've actually hit an error, so we set the span's status based on the error.
+		span.SetStatus(err)
+		log.G(ctx).WithError(err).Error("Error processing Queue item")
+		return true
+	}
+	log.G(ctx).Debug("Processed Queue item")
+
+	return true
+}
+
+func (q *Queue) handleQueueItemObject(ctx context.Context, qi *queueItem) error {
+	// This is a separate function / span, because the handleQueueItem span is the time spent waiting for the object
+	// plus the time spend handling the object. Instead, this function / span is scoped to a single object.
+	ctx, span := trace.StartSpan(ctx, "handleQueueItemObject")
+	defer span.End()
+
+	ctx = span.WithFields(ctx, map[string]interface{}{
+		"requeues":        qi.requeues,
+		"originallyAdded": qi.originallyAdded.String(),
+		"addedViaRedirty": qi.addedViaRedirty,
+		"plannedForWork":  qi.plannedToStartWorkAt.String(),
+	})
+
+	if qi.delayedViaRateLimit != nil {
+		ctx = span.WithField(ctx, "delayedViaRateLimit", qi.delayedViaRateLimit.String())
+	}
+
+	// Add the current key as an attribute to the current span.
+	ctx = span.WithField(ctx, "key", qi.key)
+	// Run the syncHandler, passing it the namespace/name string of the Pod resource to be synced.
+	err := q.handler(ctx, qi.key)
+
+	q.lock.Lock()
+	defer q.lock.Unlock()
+
+	delete(q.itemsBeingProcessed, qi.key)
+	if qi.forget {
+		q.ratelimiter.Forget(qi.key)
+		log.G(ctx).WithError(err).Warnf("forgetting %q as told to forget while in progress", qi.key)
+		return nil
+	}
+
+	if err != nil {
+		ctx = span.WithField(ctx, "error", err.Error())
+		var delay *time.Duration
+
+		// Stash the original error for logging below
+		originalError := err
+		delay, err = q.retryFunc(ctx, qi.key, qi.requeues+1, qi.originallyAdded, err)
+		if err == nil {
+			// Put the item back on the work Queue to handle any transient errors.
+			log.G(ctx).WithError(originalError).Warnf("requeuing %q due to failed sync", qi.key)
+			newQI := q.insert(ctx, qi.key, true, delay)
+			newQI.requeues = qi.requeues + 1
+			newQI.originallyAdded = qi.originallyAdded
+
+			return nil
+		}
+		if !qi.redirtiedAt.IsZero() {
+			err = fmt.Errorf("temporarily (requeued) forgetting %q due to: %w", qi.key, err)
+		} else {
+			err = fmt.Errorf("forgetting %q due to: %w", qi.key, err)
+		}
+	}
+
+	// We've exceeded the maximum retries or we were successful.
+	q.ratelimiter.Forget(qi.key)
+	if !qi.redirtiedAt.IsZero() {
+		delay := time.Until(qi.redirtiedAt)
+		newQI := q.insert(ctx, qi.key, qi.redirtiedWithRatelimit, &delay)
+		newQI.addedViaRedirty = true
+	}
+
+	span.SetStatus(err)
+	return err
+}
+
+func (q *Queue) String() string {
+	q.lock.Lock()
+	defer q.lock.Unlock()
+
+	items := make([]string, 0, q.items.Len())
+
+	for next := q.items.Front(); next != nil; next = next.Next() {
+		items = append(items, next.Value.(*queueItem).String())
+	}
+	return fmt.Sprintf("<items:%s>", items)
+}
+
+// DefaultRetryFunc is the default function used for retries by the queue subsystem.
+func DefaultRetryFunc(ctx context.Context, key string, timesTried int, originallyAdded time.Time, err error) (*time.Duration, error) {
+	if timesTried < MaxRetries {
+		return nil, nil
+	}
+
+	return nil, pkgerrors.Wrapf(err, "maximum retries (%d) reached", MaxRetries)
+}

internal/queue/queue_test.go

@@ -0,0 +1,510 @@
+package queue
+
+import (
+	"context"
+	"errors"
+	"strconv"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/sirupsen/logrus"
+	"github.com/virtual-kubelet/virtual-kubelet/log"
+	logruslogger "github.com/virtual-kubelet/virtual-kubelet/log/logrus"
+	"golang.org/x/time/rate"
+	"gotest.tools/assert"
+	is "gotest.tools/assert/cmp"
+	"k8s.io/client-go/util/workqueue"
+	"k8s.io/utils/clock"
+)
+
+func durationPtr(d time.Duration) *time.Duration {
+	return &d
+}
+
+func TestQueueMaxRetries(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	logger := logrus.New()
+	logger.SetLevel(logrus.DebugLevel)
+	ctx = log.WithLogger(ctx, logruslogger.FromLogrus(logrus.NewEntry(logger)))
+	n := 0
+	knownErr := errors.New("Testing error")
+	handler := func(ctx context.Context, key string) error {
+		n++
+		return knownErr
+	}
+	wq := New(workqueue.NewMaxOfRateLimiter(
+		// The default upper bound is 1000 seconds. Let's not use that.
+		workqueue.NewItemExponentialFailureRateLimiter(5*time.Millisecond, 10*time.Millisecond),
+		&workqueue.BucketRateLimiter{Limiter: rate.NewLimiter(rate.Limit(10), 100)},
+	), t.Name(), handler, nil)
+	wq.Enqueue(context.TODO(), "test")
+
+	for n < MaxRetries {
+		assert.Assert(t, wq.handleQueueItem(ctx))
+	}
+
+	assert.Assert(t, is.Equal(n, MaxRetries))
+	assert.Assert(t, is.Equal(0, wq.Len()))
+}
+
+func TestQueueCustomRetries(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	logger := logrus.New()
+	logger.SetLevel(logrus.DebugLevel)
+	ctx = log.WithLogger(ctx, logruslogger.FromLogrus(logrus.NewEntry(logger)))
+	n := 0
+	errorSeen := 0
+	retryTestError := errors.New("Error should be retried every 10 milliseconds")
+	handler := func(ctx context.Context, key string) error {
+		if key == "retrytest" {
+			n++
+			return retryTestError
+		}
+		return errors.New("Unknown error")
+	}
+
+	shouldRetryFunc := func(ctx context.Context, key string, timesTried int, originallyAdded time.Time, err error) (*time.Duration, error) {
+		var sleepTime *time.Duration
+		if errors.Is(err, retryTestError) {
+			errorSeen++
+			sleepTime = durationPtr(10 * time.Millisecond)
+		}
+		_, retErr := DefaultRetryFunc(ctx, key, timesTried, originallyAdded, err)
+		return sleepTime, retErr
+	}
+
+	wq := New(&workqueue.BucketRateLimiter{Limiter: rate.NewLimiter(rate.Limit(1000), 1000)}, t.Name(), handler, shouldRetryFunc)
+
+	timeTaken := func(key string) time.Duration {
+		start := time.Now()
+		wq.Enqueue(context.TODO(), key)
+		for i := 0; i < MaxRetries; i++ {
+			assert.Assert(t, wq.handleQueueItem(ctx))
+		}
+		return time.Since(start)
+	}
+
+	unknownTime := timeTaken("unknown")
+	assert.Assert(t, n == 0)
+	assert.Assert(t, unknownTime < 10*time.Millisecond)
+
+	retrytestTime := timeTaken("retrytest")
+	assert.Assert(t, is.Equal(n, MaxRetries))
+	assert.Assert(t, is.Equal(errorSeen, MaxRetries))
+
+	assert.Assert(t, is.Equal(0, wq.Len()))
+	assert.Assert(t, retrytestTime > 10*time.Millisecond*time.Duration(n-1))
+	assert.Assert(t, retrytestTime < 2*10*time.Millisecond*time.Duration(n-1))
+}
+
+func TestForget(t *testing.T) {
+	t.Parallel()
+	handler := func(ctx context.Context, key string) error {
+		panic("Should never be called")
+	}
+	wq := New(workqueue.DefaultItemBasedRateLimiter(), t.Name(), handler, nil)
+
+	wq.Forget(context.TODO(), "val")
+	assert.Assert(t, is.Equal(0, wq.Len()))
+
+	v := "test"
+	wq.EnqueueWithoutRateLimit(context.TODO(), v)
+	assert.Assert(t, is.Equal(1, wq.Len()))
+}
+
+func TestQueueEmpty(t *testing.T) {
+	t.Parallel()
+	ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
+	defer cancel()
+
+	q := New(workqueue.DefaultItemBasedRateLimiter(), t.Name(), func(ctx context.Context, key string) error {
+		return nil
+	}, nil)
+
+	item, err := q.getNextItem(ctx)
+	assert.Error(t, err, context.DeadlineExceeded.Error())
+	assert.Assert(t, is.Nil(item))
+}
+
+func TestQueueItemNoSleep(t *testing.T) {
+	t.Parallel()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 1000*time.Millisecond)
+	defer cancel()
+
+	q := New(workqueue.DefaultItemBasedRateLimiter(), t.Name(), func(ctx context.Context, key string) error {
+		return nil
+	}, nil)
+
+	q.lock.Lock()
+	q.insert(ctx, "foo", false, durationPtr(-1*time.Hour))
+	q.insert(ctx, "bar", false, durationPtr(-1*time.Hour))
+	q.lock.Unlock()
+
+	item, err := q.getNextItem(ctx)
+	assert.NilError(t, err)
+	assert.Assert(t, is.Equal(item.key, "foo"))
+
+	item, err = q.getNextItem(ctx)
+	assert.NilError(t, err)
+	assert.Assert(t, is.Equal(item.key, "bar"))
+}
+
+func TestQueueItemSleep(t *testing.T) {
+	t.Parallel()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 1000*time.Millisecond)
+	defer cancel()
+
+	q := New(workqueue.DefaultItemBasedRateLimiter(), t.Name(), func(ctx context.Context, key string) error {
+		return nil
+	}, nil)
+	q.lock.Lock()
+	q.insert(ctx, "foo", false, durationPtr(100*time.Millisecond))
+	q.insert(ctx, "bar", false, durationPtr(100*time.Millisecond))
+	q.lock.Unlock()
+
+	item, err := q.getNextItem(ctx)
+	assert.NilError(t, err)
+	assert.Assert(t, is.Equal(item.key, "foo"))
+}
+
+func TestQueueBackgroundAdd(t *testing.T) {
+	t.Parallel()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5000*time.Millisecond)
+	defer cancel()
+
+	q := New(workqueue.DefaultItemBasedRateLimiter(), t.Name(), func(ctx context.Context, key string) error {
+		return nil
+	}, nil)
+	start := time.Now()
+	time.AfterFunc(100*time.Millisecond, func() {
+		q.lock.Lock()
+		defer q.lock.Unlock()
+		q.insert(ctx, "foo", false, nil)
+	})
+
+	item, err := q.getNextItem(ctx)
+	assert.NilError(t, err)
+	assert.Assert(t, is.Equal(item.key, "foo"))
+	assert.Assert(t, time.Since(start) > 100*time.Millisecond)
+}
+
+func TestQueueBackgroundAdvance(t *testing.T) {
+	t.Parallel()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5000*time.Millisecond)
+	defer cancel()
+
+	q := New(workqueue.DefaultItemBasedRateLimiter(), t.Name(), func(ctx context.Context, key string) error {
+		return nil
+	}, nil)
+	start := time.Now()
+	q.lock.Lock()
+	q.insert(ctx, "foo", false, durationPtr(10*time.Second))
+	q.lock.Unlock()
+
+	time.AfterFunc(200*time.Millisecond, func() {
+		q.lock.Lock()
+		defer q.lock.Unlock()
+		q.insert(ctx, "foo", false, nil)
+	})
+
+	item, err := q.getNextItem(ctx)
+	assert.NilError(t, err)
+	assert.Assert(t, is.Equal(item.key, "foo"))
+	assert.Assert(t, time.Since(start) > 200*time.Millisecond)
+	assert.Assert(t, time.Since(start) < 5*time.Second)
+}
+
+func TestQueueRedirty(t *testing.T) {
+	t.Parallel()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5000*time.Millisecond)
+	defer cancel()
+
+	var times int64
+	var q *Queue
+	q = New(workqueue.DefaultItemBasedRateLimiter(), t.Name(), func(ctx context.Context, key string) error {
+		assert.Assert(t, is.Equal(key, "foo"))
+		if atomic.AddInt64(&times, 1) == 1 {
+			q.EnqueueWithoutRateLimit(context.TODO(), "foo")
+		} else {
+			cancel()
+		}
+		return nil
+	}, nil)
+
+	q.EnqueueWithoutRateLimit(context.TODO(), "foo")
+	q.Run(ctx, 1)
+	for !q.Empty() {
+		time.Sleep(100 * time.Millisecond)
+	}
+	assert.Assert(t, is.Equal(atomic.LoadInt64(&times), int64(2)))
+}
+
+func TestHeapConcurrency(t *testing.T) {
+	t.Parallel()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5000*time.Millisecond)
+	defer cancel()
+
+	start := time.Now()
+	seen := sync.Map{}
+	q := New(workqueue.DefaultItemBasedRateLimiter(), t.Name(), func(ctx context.Context, key string) error {
+		seen.Store(key, struct{}{})
+		time.Sleep(time.Second)
+		return nil
+	}, nil)
+	for i := 0; i < 20; i++ {
+		q.EnqueueWithoutRateLimit(context.TODO(), strconv.Itoa(i))
+	}
+
+	assert.Assert(t, q.Len() == 20)
+
+	go q.Run(ctx, 20)
+	for q.Len() > 0 {
+		time.Sleep(100 * time.Millisecond)
+	}
+
+	for i := 0; i < 20; i++ {
+		_, ok := seen.Load(strconv.Itoa(i))
+		assert.Assert(t, ok, "Did not observe: %d", i)
+	}
+	assert.Assert(t, time.Since(start) < 5*time.Second)
+}
+
+func checkConsistency(t *testing.T, q *Queue) {
+	q.lock.Lock()
+	defer q.lock.Unlock()
+
+	for next := q.items.Front(); next != nil && next.Next() != nil; next = next.Next() {
+		qi := next.Value.(*queueItem)
+		qiNext := next.Next().Value.(*queueItem)
+		assert.Assert(t, qi.plannedToStartWorkAt.Before(qiNext.plannedToStartWorkAt) || qi.plannedToStartWorkAt.Equal(qiNext.plannedToStartWorkAt))
+	}
+}
+
+func TestHeapOrder(t *testing.T) {
+	q := New(workqueue.DefaultItemBasedRateLimiter(), t.Name(), func(ctx context.Context, key string) error {
+		return nil
+	}, nil)
+	q.clock = nonmovingClock{}
+
+	q.EnqueueWithoutRateLimitWithDelay(context.TODO(), "a", 1000)
+	q.EnqueueWithoutRateLimitWithDelay(context.TODO(), "b", 2000)
+	q.EnqueueWithoutRateLimitWithDelay(context.TODO(), "c", 3000)
+	q.EnqueueWithoutRateLimitWithDelay(context.TODO(), "d", 4000)
+	q.EnqueueWithoutRateLimitWithDelay(context.TODO(), "e", 5000)
+	checkConsistency(t, q)
+	t.Logf("%v", q)
+	q.EnqueueWithoutRateLimitWithDelay(context.TODO(), "d", 1000)
+	checkConsistency(t, q)
+	t.Logf("%v", q)
+	q.EnqueueWithoutRateLimitWithDelay(context.TODO(), "c", 1001)
+	checkConsistency(t, q)
+	t.Logf("%v", q)
+	q.EnqueueWithoutRateLimitWithDelay(context.TODO(), "e", 999)
+	checkConsistency(t, q)
+	t.Logf("%v", q)
+}
+
+type rateLimitWrapper struct {
+	addedMap     sync.Map
+	forgottenMap sync.Map
+	rl           workqueue.RateLimiter
+}
+
+func (r *rateLimitWrapper) When(item interface{}) time.Duration {
+	if _, ok := r.forgottenMap.Load(item); ok {
+		r.forgottenMap.Delete(item)
+		// Reset the added map
+		r.addedMap.Store(item, 1)
+	} else {
+		actual, loaded := r.addedMap.LoadOrStore(item, 1)
+		if loaded {
+			r.addedMap.Store(item, actual.(int)+1)
+		}
+	}
+
+	return r.rl.When(item)
+}
+
+func (r *rateLimitWrapper) Forget(item interface{}) {
+	r.forgottenMap.Store(item, struct{}{})
+	r.rl.Forget(item)
+}
+
+func (r *rateLimitWrapper) NumRequeues(item interface{}) int {
+	return r.rl.NumRequeues(item)
+}
+
+func TestRateLimiter(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 5000*time.Millisecond)
+	defer cancel()
+
+	syncMap := sync.Map{}
+	syncMap.Store("foo", 0)
+	syncMap.Store("bar", 0)
+	syncMap.Store("baz", 0)
+	syncMap.Store("quux", 0)
+
+	start := time.Now()
+	ratelimiter := &rateLimitWrapper{
+		rl: workqueue.NewItemFastSlowRateLimiter(1*time.Millisecond, 100*time.Millisecond, 1),
+	}
+
+	q := New(ratelimiter, t.Name(), func(ctx context.Context, key string) error {
+		oldValue, _ := syncMap.Load(key)
+		syncMap.Store(key, oldValue.(int)+1)
+		if oldValue.(int) < 9 {
+			return errors.New("test")
+		}
+		return nil
+	}, nil)
+
+	enqueued := 0
+	syncMap.Range(func(key, value interface{}) bool {
+		enqueued++
+		q.Enqueue(context.TODO(), key.(string))
+		return true
+	})
+
+	assert.Assert(t, enqueued == 4)
+	go q.Run(ctx, 10)
+
+	incomplete := true
+	for incomplete {
+		time.Sleep(10 * time.Millisecond)
+		incomplete = false
+		// Wait for all items to finish processing.
+		syncMap.Range(func(key, value interface{}) bool {
+			if value.(int) < 10 {
+				incomplete = true
+			}
+			return true
+		})
+	}
+
+	// Make sure there were ~9 "slow" rate limits per item, and 1 fast
+	assert.Assert(t, time.Since(start) > 9*100*time.Millisecond)
+	// Make sure we didn't go off the deep end.
+	assert.Assert(t, time.Since(start) < 2*9*100*time.Millisecond)
+
+	// Make sure each item was seen. And Forgotten.
+	syncMap.Range(func(key, value interface{}) bool {
+		_, ok := ratelimiter.forgottenMap.Load(key)
+		assert.Assert(t, ok, "%s in forgotten map", key)
+		val, ok := ratelimiter.addedMap.Load(key)
+		assert.Assert(t, ok, "%s in added map", key)
+		assert.Assert(t, val == 10)
+		return true
+	})
+
+	q.lock.Lock()
+	defer q.lock.Unlock()
+	assert.Assert(t, len(q.itemsInQueue) == 0)
+	assert.Assert(t, len(q.itemsBeingProcessed) == 0)
+	assert.Assert(t, q.items.Len() == 0)
+
+}
+
+func TestQueueForgetInProgress(t *testing.T) {
+	t.Parallel()
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	var times int64
+	var q *Queue
+	q = New(workqueue.DefaultItemBasedRateLimiter(), t.Name(), func(ctx context.Context, key string) error {
+		assert.Assert(t, is.Equal(key, "foo"))
+		atomic.AddInt64(&times, 1)
+		q.Forget(context.TODO(), key)
+		return errors.New("test")
+	}, nil)
+
+	q.EnqueueWithoutRateLimit(context.TODO(), "foo")
+	go q.Run(ctx, 1)
+	for !q.Empty() {
+		time.Sleep(100 * time.Millisecond)
+	}
+	assert.Assert(t, is.Equal(atomic.LoadInt64(&times), int64(1)))
+}
+
+func TestQueueForgetBeforeStart(t *testing.T) {
+	t.Parallel()
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	q := New(workqueue.DefaultItemBasedRateLimiter(), t.Name(), func(ctx context.Context, key string) error {
+		panic("shouldn't be called")
+	}, nil)
+
+	q.EnqueueWithoutRateLimit(context.TODO(), "foo")
+	q.Forget(context.TODO(), "foo")
+	go q.Run(ctx, 1)
+	for !q.Empty() {
+		time.Sleep(100 * time.Millisecond)
+	}
+}
+
+func TestQueueMoveItem(t *testing.T) {
+	t.Parallel()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	q := New(workqueue.DefaultItemBasedRateLimiter(), t.Name(), func(ctx context.Context, key string) error {
+		panic("shouldn't be called")
+	}, nil)
+	q.clock = nonmovingClock{}
+
+	q.insert(ctx, "foo", false, durationPtr(3000))
+	q.insert(ctx, "bar", false, durationPtr(2000))
+	q.insert(ctx, "baz", false, durationPtr(1000))
+	checkConsistency(t, q)
+	t.Log(q)
+
+	q.insert(ctx, "foo", false, durationPtr(2000))
+	checkConsistency(t, q)
+	t.Log(q)
+
+	q.insert(ctx, "foo", false, durationPtr(1999))
+	checkConsistency(t, q)
+	t.Log(q)
+
+	q.insert(ctx, "foo", false, durationPtr(999))
+	checkConsistency(t, q)
+	t.Log(q)
+}
+
+type nonmovingClock struct {
+}
+
+func (n nonmovingClock) Now() time.Time {
+	return time.Time{}
+}
+
+func (n nonmovingClock) Since(t time.Time) time.Duration {
+	return n.Now().Sub(t)
+}
+
+func (n nonmovingClock) After(d time.Duration) <-chan time.Time {
+	panic("implement me")
+}
+
+func (n nonmovingClock) NewTimer(d time.Duration) clock.Timer {
+	panic("implement me")
+}
+
+func (n nonmovingClock) Sleep(d time.Duration) {
+	panic("implement me")
+}
+
+func (n nonmovingClock) Tick(d time.Duration) <-chan time.Time {
+	panic("implement me")
+}

internal/test/e2e/framework/node.go

@@ -16,23 +16,24 @@ import (
 // WaitUntilNodeCondition establishes a watch on the vk node.
 // Then, it waits for the specified condition function to be verified.
 func (f *Framework) WaitUntilNodeCondition(fn watch.ConditionFunc) error {
+	// Watch for updates to the Pod resource until fn is satisfied, or until the timeout is reached.
+	ctx, cancel := context.WithTimeout(context.Background(), f.WatchTimeout)
+	defer cancel()
+
 	// Create a field selector that matches the specified Pod resource.
 	fs := fields.OneTermEqualSelector("metadata.name", f.NodeName).String()
 	// Create a ListWatch so we can receive events for the matched Pod resource.
 	lw := &cache.ListWatch{
 		ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
 			options.FieldSelector = fs
-			return f.KubeClient.CoreV1().Nodes().List(options)
+			return f.KubeClient.CoreV1().Nodes().List(ctx, options)
 		},
 		WatchFunc: func(options metav1.ListOptions) (watchapi.Interface, error) {
 			options.FieldSelector = fs
-			return f.KubeClient.CoreV1().Nodes().Watch(options)
+			return f.KubeClient.CoreV1().Nodes().Watch(ctx, options)
 		},
 	}
 
-	// Watch for updates to the Pod resource until fn is satisfied, or until the timeout is reached.
-	ctx, cancel := context.WithTimeout(context.Background(), f.WatchTimeout)
-	defer cancel()
 	last, err := watch.UntilWithSync(ctx, lw, &corev1.Node{}, nil, fn)
 	if err != nil {
 		return err
@@ -44,17 +45,17 @@ func (f *Framework) WaitUntilNodeCondition(fn watch.ConditionFunc) error {
 }
 
 // DeleteNode deletes the vk node used by the framework
-func (f *Framework) DeleteNode() error {
+func (f *Framework) DeleteNode(ctx context.Context) error {
 	var gracePeriod int64
 	propagation := metav1.DeletePropagationBackground
 	opts := metav1.DeleteOptions{
 		PropagationPolicy:  &propagation,
 		GracePeriodSeconds: &gracePeriod,
 	}
-	return f.KubeClient.CoreV1().Nodes().Delete(f.NodeName, &opts)
+	return f.KubeClient.CoreV1().Nodes().Delete(ctx, f.NodeName, opts)
 }
 
 // GetNode gets the vk nodeused by the framework
-func (f *Framework) GetNode() (*corev1.Node, error) {
+func (f *Framework) GetNode(ctx context.Context) (*corev1.Node, error) {
-	return f.KubeClient.CoreV1().Nodes().Get(f.NodeName, metav1.GetOptions{})
+	return f.KubeClient.CoreV1().Nodes().Get(ctx, f.NodeName, metav1.GetOptions{})
 }

internal/test/e2e/framework/pod.go

@@ -6,13 +6,13 @@ import (
 	"strings"
 
 	corev1 "k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/runtime"
 	watchapi "k8s.io/apimachinery/pkg/watch"
 	"k8s.io/client-go/tools/cache"
 	"k8s.io/client-go/tools/watch"
-	podutil "k8s.io/kubernetes/pkg/api/v1/pod"
 )
 
 // CreateDummyPodObjectWithPrefix creates a dujmmy pod object using the specified prefix as the value of .metadata.generateName.
@@ -47,21 +47,21 @@ func (f *Framework) CreateDummyPodObjectWithPrefix(testName string, prefix strin
 }
 
 // CreatePod creates the specified pod in the Kubernetes API.
-func (f *Framework) CreatePod(pod *corev1.Pod) (*corev1.Pod, error) {
+func (f *Framework) CreatePod(ctx context.Context, pod *corev1.Pod) (*corev1.Pod, error) {
-	return f.KubeClient.CoreV1().Pods(f.Namespace).Create(pod)
+	return f.KubeClient.CoreV1().Pods(f.Namespace).Create(ctx, pod, metav1.CreateOptions{})
 }
 
 // DeletePod deletes the pod with the specified name and namespace in the Kubernetes API using the default grace period.
-func (f *Framework) DeletePod(namespace, name string) error {
+func (f *Framework) DeletePod(ctx context.Context, namespace, name string) error {
-	return f.KubeClient.CoreV1().Pods(namespace).Delete(name, &metav1.DeleteOptions{})
+	return f.KubeClient.CoreV1().Pods(namespace).Delete(ctx, name, metav1.DeleteOptions{})
 }
 
 // DeletePodImmediately forcibly deletes the pod with the specified name and namespace in the Kubernetes API.
 // This is equivalent to running "kubectl delete --force --grace-period 0 --namespace <namespace> pod <name>".
-func (f *Framework) DeletePodImmediately(namespace, name string) error {
+func (f *Framework) DeletePodImmediately(ctx context.Context, namespace, name string) error {
 	grace := int64(0)
 	propagation := metav1.DeletePropagationBackground
-	return f.KubeClient.CoreV1().Pods(namespace).Delete(name, &metav1.DeleteOptions{
+	return f.KubeClient.CoreV1().Pods(namespace).Delete(ctx, name, metav1.DeleteOptions{
 		GracePeriodSeconds: &grace,
 		PropagationPolicy:  &propagation,
 	})
@@ -70,22 +70,22 @@ func (f *Framework) DeletePodImmediately(namespace, name string) error {
 // WaitUntilPodCondition establishes a watch on the pod with the specified name and namespace.
 // Then, it waits for the specified condition function to be verified.
 func (f *Framework) WaitUntilPodCondition(namespace, name string, fn watch.ConditionFunc) (*corev1.Pod, error) {
+	// Watch for updates to the Pod resource until fn is satisfied, or until the timeout is reached.
+	ctx, cfn := context.WithTimeout(context.Background(), f.WatchTimeout)
+	defer cfn()
 	// Create a field selector that matches the specified Pod resource.
 	fs := fields.ParseSelectorOrDie(fmt.Sprintf("metadata.namespace==%s,metadata.name==%s", namespace, name))
 	// Create a ListWatch so we can receive events for the matched Pod resource.
 	lw := &cache.ListWatch{
 		ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
 			options.FieldSelector = fs.String()
-			return f.KubeClient.CoreV1().Pods(namespace).List(options)
+			return f.KubeClient.CoreV1().Pods(namespace).List(ctx, options)
 		},
 		WatchFunc: func(options metav1.ListOptions) (watchapi.Interface, error) {
 			options.FieldSelector = fs.String()
-			return f.KubeClient.CoreV1().Pods(namespace).Watch(options)
+			return f.KubeClient.CoreV1().Pods(namespace).Watch(ctx, options)
 		},
 	}
-	// Watch for updates to the Pod resource until fn is satisfied, or until the timeout is reached.
-	ctx, cfn := context.WithTimeout(context.Background(), f.WatchTimeout)
-	defer cfn()
 	last, err := watch.UntilWithSync(ctx, lw, &corev1.Pod{}, nil, fn)
 	if err != nil {
 		return nil, err
@@ -101,10 +101,20 @@ func (f *Framework) WaitUntilPodCondition(namespace, name string, fn watch.Condi
 func (f *Framework) WaitUntilPodReady(namespace, name string) (*corev1.Pod, error) {
 	return f.WaitUntilPodCondition(namespace, name, func(event watchapi.Event) (bool, error) {
 		pod := event.Object.(*corev1.Pod)
-		return pod.Status.Phase == corev1.PodRunning && podutil.IsPodReady(pod) && pod.Status.PodIP != "", nil
+		return pod.Status.Phase == corev1.PodRunning && IsPodReady(pod) && pod.Status.PodIP != "", nil
 	})
 }
 
+// IsPodReady returns true if a pod is ready.
+func IsPodReady(pod *v1.Pod) bool {
+	for _, cond := range pod.Status.Conditions {
+		if cond.Type == v1.PodReady && cond.Status == v1.ConditionTrue {
+			return true
+		}
+	}
+	return false
+}
+
 // WaitUntilPodDeleted blocks until the pod with the specified name and namespace is deleted from apiserver.
 func (f *Framework) WaitUntilPodDeleted(namespace, name string) (*corev1.Pod, error) {
 	return f.WaitUntilPodCondition(namespace, name, func(event watchapi.Event) (bool, error) {
@@ -129,22 +139,24 @@ func (f *Framework) WaitUntilPodInPhase(namespace, name string, phases ...corev1
 // WaitUntilPodEventWithReason establishes a watch on events involving the specified pod.
 // Then, it waits for an event with the specified reason to be created/updated.
 func (f *Framework) WaitUntilPodEventWithReason(pod *corev1.Pod, reason string) error {
+	// Watch for updates to the Event resource until fn is satisfied, or until the timeout is reached.
+	ctx, cfn := context.WithTimeout(context.Background(), f.WatchTimeout)
+	defer cfn()
+
 	// Create a field selector that matches Event resources involving the specified pod.
 	fs := fields.ParseSelectorOrDie(fmt.Sprintf("involvedObject.kind==Pod,involvedObject.uid==%s", pod.UID))
 	// Create a ListWatch so we can receive events for the matched Event resource.
 	lw := &cache.ListWatch{
 		ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
 			options.FieldSelector = fs.String()
-			return f.KubeClient.CoreV1().Events(pod.Namespace).List(options)
+			return f.KubeClient.CoreV1().Events(pod.Namespace).List(ctx, options)
 		},
 		WatchFunc: func(options metav1.ListOptions) (watchapi.Interface, error) {
 			options.FieldSelector = fs.String()
-			return f.KubeClient.CoreV1().Events(pod.Namespace).Watch(options)
+			return f.KubeClient.CoreV1().Events(pod.Namespace).Watch(ctx, options)
 		},
 	}
-	// Watch for updates to the Event resource until fn is satisfied, or until the timeout is reached.
+
-	ctx, cfn := context.WithTimeout(context.Background(), f.WatchTimeout)
-	defer cfn()
 	last, err := watch.UntilWithSync(ctx, lw, &corev1.Event{}, nil, func(event watchapi.Event) (b bool, e error) {
 		switch event.Type {
 		case watchapi.Error:
@@ -164,8 +176,8 @@ func (f *Framework) WaitUntilPodEventWithReason(pod *corev1.Pod, reason string)
 	return nil
 }
 
-// GetRunningPods gets the running pods from the provider of the virtual kubelet
+// GetRunningPodsFromProvider gets the running pods from the provider of the virtual kubelet
-func (f *Framework) GetRunningPods() (*corev1.PodList, error) {
+func (f *Framework) GetRunningPodsFromProvider(ctx context.Context) (*corev1.PodList, error) {
 	result := &corev1.PodList{}
 
 	err := f.KubeClient.CoreV1().
@@ -175,7 +187,24 @@ func (f *Framework) GetRunningPods() (*corev1.PodList, error) {
 		Name(f.NodeName).
 		SubResource("proxy").
 		Suffix("runningpods/").
-		Do().
+		Do(ctx).
+		Into(result)
+
+	return result, err
+}
+
+// GetRunningPodsFromKubernetes gets the running pods from the provider of the virtual kubelet
+func (f *Framework) GetRunningPodsFromKubernetes(ctx context.Context) (*corev1.PodList, error) {
+	result := &corev1.PodList{}
+
+	err := f.KubeClient.CoreV1().
+		RESTClient().
+		Get().
+		Resource("nodes").
+		Name(f.NodeName).
+		SubResource("proxy").
+		Suffix("pods").
+		Do(ctx).
 		Into(result)
 
 	return result, err

internal/test/e2e/framework/stats.go

@@ -1,15 +1,16 @@
 package framework
 
 import (
+	"context"
 	"encoding/json"
-	"strconv"
 
+	api "github.com/virtual-kubelet/virtual-kubelet/node/api"
+	stats "github.com/virtual-kubelet/virtual-kubelet/node/api/statsv1alpha1"
 	"k8s.io/apimachinery/pkg/util/net"
-	stats "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
 )
 
 // GetStatsSummary queries the /stats/summary endpoint of the virtual-kubelet and returns the Summary object obtained as a response.
-func (f *Framework) GetStatsSummary() (*stats.Summary, error) {
+func (f *Framework) GetStatsSummary(ctx context.Context) (*stats.Summary, error) {
 	// Query the /stats/summary endpoint.
 	b, err := f.KubeClient.CoreV1().
 		RESTClient().
@@ -17,8 +18,8 @@ func (f *Framework) GetStatsSummary() (*stats.Summary, error) {
 		Namespace(f.Namespace).
 		Resource("pods").
 		SubResource("proxy").
-		Name(net.JoinSchemeNamePort("http", f.NodeName, strconv.Itoa(10255))).
+		Name(net.JoinSchemeNamePort("https", f.NodeName, "10250")).
-		Suffix("/stats/summary").DoRaw()
+		Suffix("/stats/summary").DoRaw(ctx)
 	if err != nil {
 		return nil, err
 	}
@@ -29,3 +30,21 @@ func (f *Framework) GetStatsSummary() (*stats.Summary, error) {
 	}
 	return res, nil
 }
+
+// GetStatsSummary queries the /metrics/resource endpoint of the virtual-kubelet and returns the Summary object obtained as a response.
+func (f *Framework) GetMetricsResource(ctx context.Context) ([]byte, error) {
+	// Query the /stats/summary endpoint.
+	b, err := f.KubeClient.CoreV1().
+		RESTClient().
+		Get().
+		Namespace(f.Namespace).
+		Resource("pods").
+		SubResource("proxy").
+		Name(net.JoinSchemeNamePort("https", f.NodeName, "10250")).
+		Suffix(api.MetricsResourceRouteSuffix).DoRaw(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	return b, nil
+}

internal/test/e2e/main_test.go

@@ -23,6 +23,12 @@ var (
 	nodeName   string
 )
 
+// go1.13 compatibility cf. https://github.com/golang/go/issues/31859
+var _ = func() bool {
+	testing.Init()
+	return true
+}()
+
 func init() {
 	flag.StringVar(&kubeconfig, "kubeconfig", "", "path to the kubeconfig file to use when running the test suite outside a kubernetes cluster")
 	flag.StringVar(&namespace, "namespace", defaultNamespace, "the name of the kubernetes namespace to use for running the test suite (i.e. where to create pods)")

internal/test/suite/suite.go

@@ -21,7 +21,7 @@ type ShouldSkipTestFunc func(string) bool
 
 // TestSuite contains methods that defines the lifecycle of a test suite
 type TestSuite interface {
-	Setup()
+	Setup(t *testing.T)
 	Teardown()
 }
 
@@ -39,7 +39,7 @@ type testCase struct {
 func Run(t *testing.T, ts TestSuite) {
 	defer failOnPanic(t)
 
-	ts.Setup()
+	ts.Setup(t)
 	defer ts.Teardown()
 
 	// The implementation below is based on https://github.com/stretchr/testify

internal/test/suite/suite_test.go

@@ -20,7 +20,7 @@ type basicTestSuite struct {
 	testsRan        []string
 }
 
-func (bts *basicTestSuite) Setup() {
+func (bts *basicTestSuite) Setup(t *testing.T) {
 	bts.setupCount++
 }
 

internal/test/util/provider.go

@@ -0,0 +1 @@
+package util