Add trailing newline to stats response

This was just an annoyance when working with the API from a terminal.

Signed-off-by: Brian Goff <cpuguy83@gmail.com>

.circleci/config.yml

@@ -1,150 +0,0 @@
-version: 2.0
-jobs:
-  validate:
-    resource_class: xlarge
-    docker:
-      - image: circleci/golang:1.15
-        environment:
-          GO111MODULE: "on"
-          GOPROXY: https://proxy.golang.org
-    working_directory: /go/src/github.com/virtual-kubelet/virtual-kubelet
-    steps:
-      - checkout
-      - restore_cache:
-          keys:
-            - validate-{{ checksum "go.mod" }}-{{ checksum "go.sum" }}
-      - run:
-          name: go vet
-          command: V=1 CI=1 make vet
-      - run:
-          name: Lint
-          command: make lint
-      - run:
-          name: Dependencies
-          command: scripts/validate/gomod.sh
-      - save_cache:
-          key: validate-{{ checksum "go.mod" }}-{{ checksum "go.sum" }}
-          paths:
-            - "/go/pkg/mod"
-
-  test:
-    resource_class: xlarge
-    docker:
-      - image: circleci/golang:1.15
-        environment:
-          GO111MODULE: "on"
-    working_directory: /go/src/github.com/virtual-kubelet/virtual-kubelet
-    steps:
-      - checkout
-      - restore_cache:
-          keys:
-            - test-{{ checksum "go.mod" }}-{{ checksum "go.sum" }}
-      - run:
-          name: Build
-          command: V=1 make build
-      - run:
-          name: Tests
-          command: V=1 CI=1 make test envtest
-      - save_cache:
-          key: test-{{ checksum "go.mod" }}-{{ checksum "go.sum" }}
-          paths:
-            - "/go/pkg/mod"
-
-  e2e:
-    machine:
-      image: ubuntu-1604:202010-01
-    working_directory: /home/circleci/go/src/github.com/virtual-kubelet/virtual-kubelet
-    environment:
-      CHANGE_MINIKUBE_NONE_USER: true
-      GOPATH: /home/circleci/go
-      KUBECONFIG: /home/circleci/.kube/config
-      KUBERNETES_VERSION: v1.20.1
-      MINIKUBE_HOME: /home/circleci
-      MINIKUBE_VERSION: v1.16.0
-      MINIKUBE_WANTUPDATENOTIFICATION: false
-      MINIKUBE_WANTREPORTERRORPROMPT: false
-      SKAFFOLD_VERSION: v1.17.2
-      GO111MODULE: "on"
-    steps:
-      - checkout
-      - run:
-          name: Install kubectl
-          command: |
-            curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/${KUBERNETES_VERSION}/bin/linux/amd64/kubectl
-            chmod +x kubectl
-            sudo mv kubectl /usr/local/bin/
-            mkdir -p ${HOME}/.kube
-            touch ${HOME}/.kube/config
-      - run:
-          name: Install Skaffold
-          command: |
-            curl -Lo skaffold https://storage.googleapis.com/skaffold/releases/${SKAFFOLD_VERSION}/skaffold-linux-amd64
-            chmod +x skaffold
-            sudo mv skaffold /usr/local/bin/
-      - run:
-          name: Install Minikube dependencies
-          command: |
-            sudo apt-get update && sudo apt-get install -y apt-transport-https curl
-            curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -
-            cat <<EOF | sudo tee /etc/apt/sources.list.d/kubernetes.list
-            deb https://apt.kubernetes.io/ kubernetes-xenial main
-            EOF
-            sudo apt-get update
-            sudo apt-get install -y kubelet # systemd unit is disabled
-      - run:
-          name: Install Minikube
-          command: |
-            curl -Lo minikube https://storage.googleapis.com/minikube/releases/${MINIKUBE_VERSION}/minikube-linux-amd64
-            chmod +x minikube
-            sudo mv minikube /usr/local/bin/
-      - run:
-          name: Start Minikube
-          command: |
-            sudo -E minikube start --vm-driver=none --cpus 2 --memory 2048 --kubernetes-version=${KUBERNETES_VERSION}
-      - run:
-          name: Wait for Minikube
-          command: |
-            JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}';
-            until kubectl get nodes -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do
-              sleep 1;
-            done
-      - run:
-          name: Watch pods
-          command: kubectl get pods -o json --watch
-          background: true
-      - run:
-          name: Watch nodes
-          command: kubectl get nodes -o json --watch
-          background: true
-      - restore_cache:
-          keys:
-            - e2e-{{ checksum "go.mod" }}-{{ checksum "go.sum" }}-2
-      - run:
-          name: Run the end-to-end test suite
-          command: |
-            mkdir $HOME/.go
-            export PATH=$HOME/.go/bin:${PATH}
-            curl -fsSL -o "/tmp/go.tar.gz" "https://dl.google.com/go/go1.15.6.linux-amd64.tar.gz"
-            tar -C $HOME/.go --strip-components=1 -xzf "/tmp/go.tar.gz"
-            go version
-            make e2e
-      - save_cache:
-          key: e2e-{{ checksum "go.mod" }}-{{ checksum "go.sum" }}-2
-          paths:
-            - "/home/circleci/go/pkg/mod"
-      - run:
-          name: Collect logs on failure from vkubelet-mock-0
-          command: |
-            kubectl logs vkubelet-mock-0
-          when: on_fail
-
-workflows:
-  version: 2
-  validate_and_test:
-    jobs:
-      - validate
-      - test
-      - e2e:
-          requires:
-            - validate
-            - test

.dockerignore

@@ -1,4 +1,6 @@
 .vscode
 private.env
 *.private.*
-providers/azurebatch/deployment/
+providers/azurebatch/deployment/
+Dockerfile
+.dockerignore

.github/dependabot.yml

@@ -0,0 +1,10 @@
+version: 2
+updates:
+  - package-ecosystem: "gomod"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"

.github/workflows/ci.yml

@@ -0,0 +1,115 @@
+name: CI
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+
+permissions:
+  contents: read
+
+env:
+  GO_VERSION: "1.23"
+
+jobs:
+  lint:
+    name: Lint
+    runs-on: ubuntu-22.04
+    timeout-minutes: 20
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GO_VERSION }}
+          cache: false
+      - uses: actions/checkout@v4
+      - uses: golangci/golangci-lint-action@v8
+        with:
+          version: v2.1
+          args: --timeout=15m --config=.golangci.yml
+          skip-cache: true
+
+  unit-tests:
+    name: Unit Tests
+    runs-on: ubuntu-22.04
+    timeout-minutes: 20
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - uses: actions/checkout@v4
+      - name: Run Tests
+        run: make test
+
+  env-tests:
+    name: Envtest Tests
+    runs-on: ubuntu-22.04
+    timeout-minutes: 10
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - uses: actions/checkout@v4
+      - name: Run Tests
+        run: make envtest
+
+  e2e:
+    name: E2E
+    runs-on: ubuntu-22.04
+    timeout-minutes: 20
+    env:
+      CHANGE_MINIKUBE_NONE_USER: true
+      KUBERNETES_VERSION: v1.31
+      MINIKUBE_HOME: /home/runner
+      MINIKUBE_VERSION: v1.34.0
+      MINIKUBE_WANTUPDATENOTIFICATION: false
+      MINIKUBE_WANTREPORTERRORPROMPT: false
+      SKAFFOLD_VERSION: v2.13.2
+      GO111MODULE: "on"
+
+    steps:
+      - uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Install Skaffold
+        run: |
+          curl -sLo skaffold https://storage.googleapis.com/skaffold/releases/${SKAFFOLD_VERSION}/skaffold-linux-amd64
+          chmod +x skaffold
+          sudo mv skaffold /usr/local/bin/
+          echo /usr/local/bin >> $GITHUB_PATH
+      - name: Install Minikube dependencies
+        run: |
+          sudo apt-get update && sudo apt-get install -y apt-transport-https curl
+          echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/${KUBERNETES_VERSION}/deb/ /" | sudo tee /etc/apt/sources.list.d/kubernetes.list
+          curl -fsSL https://pkgs.k8s.io/core:/stable:/${KUBERNETES_VERSION}/deb/Release.key | sudo gpg --no-tty --yes --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
+          sudo apt-get update
+          sudo apt-get remove -y containerd.io containerd
+          sudo apt-get install -y kubectl docker.io
+      - name: Install Minikube
+        run: |
+          curl -sLo minikube https://storage.googleapis.com/minikube/releases/${MINIKUBE_VERSION}/minikube-linux-amd64
+          chmod +x minikube
+          sudo mv minikube /usr/local/bin/
+      - name: Start Minikube
+        run: |
+          sudo usermod -aG docker $USER && newgrp docker
+          minikube start --vm-driver=docker --cpus 2 --memory 2048 --kubernetes-version=${KUBERNETES_VERSION}
+      - name: Wait for Minikube
+        run: |
+          JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}';
+          until kubectl get nodes -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do
+            sleep 1;
+          done
+      - name: Run Tests
+        run: make e2e

.github/workflows/codeql-analysis.yml

@@ -1,56 +1,59 @@
 name: "CodeQL"
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 on:
   push:
-    branches: [ master ]
-  pull_request:
-    # The branches below must be a subset of the branches above
-    branches: [ master ]
+    branches: [master]
   schedule:
-    - cron: '19 18 * * 3'
+    - cron: "19 18 * * 3"
 
 jobs:
   analyze:
     name: Analyze
     runs-on: ubuntu-latest
+    permissions:
+      security-events: write
 
     strategy:
       fail-fast: false
       matrix:
-        language: [ 'go' ]
+        language: ["go"]
         # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
         # Learn more:
         # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
 
     steps:
-    - name: Checkout repository
-      uses: actions/checkout@v2
+      - name: Checkout repository
+        uses: actions/checkout@v4
 
-    # Initializes the CodeQL tools for scanning.
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
-      with:
-        languages: ${{ matrix.language }}
-        # If you wish to specify custom queries, you can do so here or in a config file.
-        # By default, queries listed here will override any specified in a config file.
-        # Prefix the list here with "+" to use these queries and those in the config file.
-        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+      # Initializes the CodeQL tools for scanning.
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v3
+        with:
+          languages: ${{ matrix.language }}
+          # If you wish to specify custom queries, you can do so here or in a config file.
+          # By default, queries listed here will override any specified in a config file.
+          # Prefix the list here with "+" to use these queries and those in the config file.
+          # queries: ./path/to/local/query, your-org/your-repo/queries@main
 
-    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
-    # If this step fails, then you should remove it and run the build manually (see below)
-    - name: Autobuild
-      uses: github/codeql-action/autobuild@v1
+      # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+      # If this step fails, then you should remove it and run the build manually (see below)
+      - name: Autobuild
+        uses: github/codeql-action/autobuild@v3
 
-    # ℹ️ Command-line programs to run using the OS shell.
-    # 📚 https://git.io/JvXDl
+      # ℹ️ Command-line programs to run using the OS shell.
+      # 📚 https://git.io/JvXDl
 
-    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
-    #    and modify them (or add more) to build your code if your project
-    #    uses a compiled language
+      # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+      #    and modify them (or add more) to build your code if your project
+      #    uses a compiled language
 
-    #- run: |
-    #   make bootstrap
-    #   make release
+      #- run: |
+      #   make bootstrap
+      #   make release
 
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v3

.gitignore

@@ -30,6 +30,9 @@ credentials.json
 # Test loganalytics file
 loganalytics.json
 
+# Envtest
+.envtest/
+
 # VS Code files
 .vscode/
 
@@ -41,3 +44,5 @@ loganalytics.json
 **/terraform-provider-kubernetes
 **/*.tfstate*
 debug
+
+vendor/

.golangci.bck.yml

@@ -0,0 +1,34 @@
+issues:
+  exclude-use-default: false
+  exclude:
+    # EXC0001 errcheck: Almost all programs ignore errors on these functions and in most cases it's ok
+    - Error return value of .((os\.)?std(out|err)\..*|.*Close|.*Flush|os\.Remove(All)?|.*print(f|ln)?|os\.(Un)?Setenv). (is not checked|Errors unhandled)
+
+  exclude-dirs:
+    # This directory contains copy code from upstream kubernetes/kubernetes, skip it.
+    - internal/kubernetes
+    # This is mostly copied from upstream, rather than fixing that code here just ignore the errors.
+    - internal/podutils
+
+linters:
+  enable:
+    - errcheck
+    - staticcheck
+    - unconvert
+    - gofmt
+    - goimports
+    - ineffassign
+    - govet
+    - unused
+    - misspell
+    - gosec
+    - copyloopvar # Checks for pointers to enclosing loop variables
+    - tenv # Detects using os.Setenv instead of t.Setenv since Go 1.17
+    - lll
+
+linters-settings:
+  gosec:
+    excludes:
+      - G304 # Potential file inclusion via variable
+  lll:
+    line-length: 200

.golangci.yml

@@ -1,29 +1,51 @@
-linter-settings:
-  lll:
-    line-length: 200
-
-run:
-  skip-dirs:
-    # This directory contains copy code from upstream kubernetes/kubernetes, skip it.
-    - internal/kubernetes
-
+version: "2"
 linters:
   enable:
-    - errcheck
-    - govet
-    - ineffassign
-    - golint
-    - goconst
-    - goimports
-    - unused
-    - varcheck
-    - deadcode
+    - copyloopvar
+    - gosec
+    - lll
     - misspell
-    - nolintlint
-    - gocritic
-
+    - unconvert
+  settings:
+    gosec:
+      excludes:
+        - G304
+    lll:
+      line-length: 200
+  exclusions:
+    generated: lax
+    rules:
+      - path: (.+)\.go$
+        text: Error return value of .((os\.)?std(out|err)\..*|.*Close|.*Flush|os\.Remove(All)?|.*print(f|ln)?|os\.(Un)?Setenv). (is not checked|Errors unhandled)
+    paths:
+      - internal/kubernetes
+      - internal/podutils
+      - third_party$
+      - builtin$
+      - examples$
+formatters:
+  enable:
+    - gofmt
+    - goimports
+  exclusions:
+    generated: lax
+    paths:
+      - internal/kubernetes
+      - internal/podutils
+      - third_party$
+      - builtin$
+      - examples$
 issues:
-  exclude-use-default: false
-  exclude:
-    # EXC0001 errcheck: Almost all programs ignore errors on these functions and in most cases it's ok
-    - Error return value of .((os\.)?std(out|err)\..*|.*Close|.*Flush|os\.Remove(All)?|.*print(f|ln)?|os\.(Un)?Setenv). is not checked
+  # Maximum issues count per one linter.
+  # Set to 0 to disable.
+  # Default: 50
+  max-issues-per-linter: 0
+
+  # Maximum count of issues with the same text.
+  # Set to 0 to disable.
+  # Default: 3
+  max-same-issues: 0
+
+  # Make issues output unique by line.
+  # Default: true
+  uniq-by-line: false

Dockerfile

@@ -1,4 +1,6 @@
-FROM golang:1.15 as builder
+ARG GOLANG_CI_LINT_VERSION
+
+FROM golang:1.23 as builder
 ENV PATH /go/bin:/usr/local/go/bin:$PATH
 ENV GOPATH /go
 COPY . /go/src/github.com/virtual-kubelet/virtual-kubelet
@@ -7,6 +9,22 @@ ARG BUILD_TAGS=""
 RUN make VK_BUILD_TAGS="${BUILD_TAGS}" build
 RUN cp bin/virtual-kubelet /usr/bin/virtual-kubelet
 
+FROM golangci/golangci-lint:${GOLANG_CI_LINT_VERSION} as lint
+WORKDIR /app
+COPY go.mod ./
+COPY go.sum ./
+RUN \
+    --mount=type=cache,target=/root/.cache/go-build \
+    --mount=type=cache,target=/go/pkg/mod \
+    go mod download
+COPY . .
+ARG OUT_FORMAT
+RUN \
+    --mount=type=cache,target=/root/.cache/go-build \
+    --mount=type=cache,target=/go/pkg/mod \
+    --mount=type=cache,target=/root/.cache/golangci-lint \
+    golangci-lint run -v --out-format="${OUT_FORMAT:-colored-line-number}"
+
 FROM scratch
 COPY --from=builder /usr/bin/virtual-kubelet /usr/bin/virtual-kubelet
 COPY --from=builder /etc/ssl/certs/ /etc/ssl/certs

Makefile

@@ -5,6 +5,8 @@ exec := $(DOCKER_IMAGE)
 github_repo := virtual-kubelet/virtual-kubelet
 binary := virtual-kubelet
 
+GOTEST ?= go test $(if $V,-v)
+
 export GO111MODULE ?= on
 
 include Makefile.e2e
@@ -71,36 +73,28 @@ vet:
 	@echo "go vet'ing..."
 ifndef CI
 	@echo "go vet'ing Outside CI..."
-	go vet $(allpackages)
+	go vet $(TESTDIRS)
 else
 	@echo "go vet'ing in CI..."
 	mkdir -p test
-	( go vet $(allpackages); echo $$? ) | \
+	( go vet $(TESTDIRS); echo $$? ) | \
        tee test/vet.txt | sed '$$ d'; exit $$(tail -1 test/vet.txt)
 endif
 
 test:
-ifndef CI
-	@echo "Testing..."
-	go test  $(if $V,-v) $(allpackages)
-else
-	@echo "Testing in CI..."
-	mkdir -p test
-	( GODEBUG=cgocheck=2 go test -timeout=9m -v $(allpackages); echo $$? ) | \
-       tee test/output.txt | sed '$$ d'; exit $$(tail -1 test/output.txt)
-endif
+	$(GOTEST) $(TESTDIRS)
 
 list:
 	@echo "List..."
-	@echo $(allpackages)
+	@echo $(TESTDIRS)
 
 cover: gocovmerge
 	@echo "Coverage Report..."
 	@echo "NOTE: make cover does not exit 1 on failure, don't use it to check for tests success!"
 	rm -f .GOPATH/cover/*.out cover/all.merged
 	$(if $V,@echo "-- go test -coverpkg=./... -coverprofile=cover/... ./...")
-	@for MOD in $(allpackages); do \
-        go test -coverpkg=`echo $(allpackages)|tr " " ","` \
+	@for MOD in $(TESTDIRS); do \
+        go test -coverpkg=`echo $(TESTDIRS)|tr " " ","` \
             -coverprofile=cover/unit-`echo $$MOD|tr "/" "_"`.out \
             $$MOD 2>&1 | grep -v "no packages being tested depend on"; \
     done
@@ -142,11 +136,7 @@ VERSION          := $(shell git describe --tags --always --dirty="-dev")
 DATE             := $(shell date -u '+%Y-%m-%d-%H:%M UTC')
 VERSION_FLAGS    := -ldflags='-X "main.buildVersion=$(VERSION)" -X "main.buildTime=$(DATE)"'
 
-# assuming go 1.9 here!!
-_allpackages = $(shell go list ./...)
-
-# memoize allpackages, so that it's executed only once and only if used
-allpackages = $(if $(__allpackages),,$(eval __allpackages := $$(_allpackages)))$(__allpackages)
+TESTDIRS ?= ./...
 
 .PHONY: goimports
 goimports: $(gobin_tool)
@@ -175,24 +165,27 @@ authors:
 	rm -f NEWAUTHORS
 	rm -f GITAUTHORS
 
-checksums_2.3.1.txt:
-	curl -o checksums_2.3.1.txt -L https://github.com/kubernetes-sigs/kubebuilder/releases/download/v2.3.1/checksums.txt
-kubebuilder_2.3.1_${TEST_OS}_${TEST_ARCH}.tar.gz:
-	curl -C - -O -L https://github.com/kubernetes-sigs/kubebuilder/releases/download/v2.3.1/kubebuilder_2.3.1_${TEST_OS}_${TEST_ARCH}.tar.gz
-kubebuilder_2.3.1_${TEST_OS}_${TEST_ARCH}: kubebuilder_2.3.1_${TEST_OS}_${TEST_ARCH}.tar.gz checksums_2.3.1.txt
-	sha256sum -c --ignore-missing checksums_2.3.1.txt
-	tar -xvf kubebuilder_2.3.1_${TEST_OS}_${TEST_ARCH}.tar.gz
+SETUP_ENVTEST_VERSION ?= v0.0.0-20250604165838-d6126d850224
+ENVTEST_K8S_VERSION := 1.31.x
+
+ENVTEST ?= go run sigs.k8s.io/controller-runtime/tools/setup-envtest@$(SETUP_ENVTEST_VERSION)
+ENVTEST_DIR ?= $(shell pwd)/.envtest
+export KUBEBUILDER_ASSETS ?= $(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(ENVTEST_DIR) -p path)
 
 .PHONY: envtest
-envtest: kubebuilder_2.3.1_${TEST_OS}_${TEST_ARCH}
+envtest:
 	# You can add klog flags for debugging, like: -klog.v=10 -klog.logtostderr
 	# klogv2 flags just wraps our existing logrus.
-	KUBEBUILDER_ASSETS=$(PWD)/kubebuilder_2.3.1_${TEST_OS}_${TEST_ARCH}/bin go test -v -run=TestEnvtest ./node -envtest=true
+	$(GOTEST) -run=TestEnvtest ./node -envtest=true
 
 .PHONY: fmt
 fmt:
 	goimports -w $(shell go list -f '{{.Dir}}' ./...)
 
+
+export GOLANG_CI_LINT_VERSION ?= v1.49.0
+DOCKER_BUILD ?= docker buildx build
+
 .PHONY: lint
-lint: $(gobin_tool)
-	gobin -run github.com/golangci/golangci-lint/cmd/golangci-lint@v1.33.0 run ./...
+lint:
+	$(DOCKER_BUILD) --target=lint --build-arg GOLANG_CI_LINT_VERSION --build-arg OUT_FORMAT .

Makefile.e2e

@@ -1,7 +1,17 @@
+
+# skaffold checks for kubectl context
+# For minikube, docker-for-desktop and docker-desktop the context matches above names
+# If one wants to use kind, kind gives context names based on the cluster-name
+# But as of now they match the following syntax: kind-*
+# The first check verifies that this is a kind kubernetes context
+# Second check verifies the other ones
 .PHONY: skaffold.validate
 skaffold.validate: kubectl_context := $(shell kubectl config current-context)
 skaffold.validate:
-	@if [[ ! "minikube,docker-for-desktop,docker-desktop" =~ .*"$(kubectl_context)".* ]]; then \
+
+	@if [[ "$(kubectl_context)" =~ .*"kind".* ]]; then \
+		true; \
+	elif [[ ! "minikube,docker-for-desktop,docker-desktop" =~ .*"$(kubectl_context)".* ]]; then \
 		echo current-context is [$(kubectl_context)]. Must be one of [minikube,docker-for-desktop,docker-desktop]; \
 		false; \
 	fi
@@ -39,7 +49,7 @@ e2e: NODE_NAME := vkubelet-mock-0
 e2e: export VK_BUILD_TAGS += mock_provider
 e2e: e2e.clean bin/e2e/virtual-kubelet skaffold/run
 	@echo Running tests...
-	cd $(PWD)/internal/test/e2e && go test -v -timeout 5m -tags e2e ./... \
+	cd $(PWD)/internal/test/e2e && $(GOTEST) -timeout 5m -tags e2e ./... \
 		-kubeconfig=$(KUBECONFIG) \
 		-namespace=$(NAMESPACE) \
 		-node-name=$(NODE_NAME)

README.md

@@ -1,5 +1,7 @@
 # Virtual Kubelet
 
+[![Go Reference](https://pkg.go.dev/badge/github.com/virtual-kubelet/virtual-kubelet.svg)](https://pkg.go.dev/github.com/virtual-kubelet/virtual-kubelet)
+
 Virtual Kubelet is an open source [Kubernetes kubelet](https://kubernetes.io/docs/reference/generated/kubelet/)
 implementation that masquerades as a kubelet for the purposes of connecting Kubernetes to other APIs.
 This allows the nodes to be backed by other services like ACI, AWS Fargate, [IoT Edge](https://github.com/Azure/iot-edge-virtual-kubelet-provider), [Tensile Kube](https://github.com/virtual-kubelet/tensile-kube) etc. The primary scenario for VK is enabling the extension of the Kubernetes API into serverless container platforms like ACI and Fargate, though we are open to others. However, it should be noted that VK is explicitly not intended to be an alternative to Kubernetes federation.
@@ -23,6 +25,8 @@ The best description is "Kubernetes API on top, programmable back."
     + [AWS Fargate Provider](#aws-fargate-provider)
     + [Elotl Kip](#elotl-kip)
 	+ [HashiCorp Nomad](#hashicorp-nomad-provider)
+    + [InterLink](#interlink-provider)
+    + [Liqo](#liqo-provider)
     + [OpenStack Zun](#openstack-zun-provider)
     + [Tensile Kube Provider](#tensile-kube-provider)
     + [Adding a New Provider via the Provider Interface](#adding-a-new-provider-via-the-provider-interface)
@@ -46,7 +50,7 @@ project to build a custom Kubernetes node agent.
 See godoc for up to date instructions on consuming this project:
 https://godoc.org/github.com/virtual-kubelet/virtual-kubelet
 
-There are implementations available for several provides (listed above), see
+There are implementations available for [several providers](#providers), see
 those repos for details on how to deploy.
 
 ## Current Features
@@ -134,6 +138,20 @@ would on a Kubernetes node.
 
 For detailed instructions, follow the guide [here](https://github.com/virtual-kubelet/nomad/blob/master/README.md).
 
+### Interlink Provider
+
+[interLink](https://intertwin-eu.github.io/interLink/) provides an abstraction for the execution of a Kubernetes pod on any remote resource that has the capability to manage a container's execution lifecycle. The use cases that drove the initial development of the tool are the Slurm-powered HPC centers, regardless the plugin based design is enabling several additional use cases to provide Kubernetes-API based access to infrastracture that cannot host a Kubelet processes. 
+InterLink is a Virtual Kubelet provider that can manage container lifecycle through a well defined API specification, allowing for any resource provider to be integrated with a simple http server and a handful of methods.
+In other words, this is an attempt to streamline the process of creating custom Virtual Kubelet providers, avoiding the need for any resource provider to implement its own version of a Kubelet workflow, which would require having some domain expertise in the Kubernetes internals.
+
+For detailed instruction, follow the guide [here](https://intertwin-eu.github.io/interLink/docs/category/guides).
+
+### Liqo Provider
+
+[Liqo](https://liqo.io) implements a provider for Virtual Kubelet designed to transparently offload pods and services to "peered" Kubernetes remote cluster. Liqo is capable of discovering neighbor clusters (using DNS, mDNS) and "peer" with them, or in other words, establish a relationship to share part of the cluster resources. When a cluster has established a peering, a new instance of the Liqo Virtual Kubelet is spawned to seamlessly extend the capacity of the cluster, by providing an abstraction of the resources of the remote cluster. The provider combined with the Liqo network fabric extends the cluster networking by enabling Pod-to-Pod traffic and multi-cluster east-west services, supporting endpoints on both clusters.
+
+For detailed instruction, follow the guide [here](https://github.com/liqotech/liqo/blob/master/README.md)
+
 ### OpenStack Zun Provider
 
 OpenStack [Zun](https://docs.openstack.org/zun/latest/) provider for Virtual Kubelet connects
@@ -162,12 +180,12 @@ performing the neccessary actions.
 
 There are 3 main interfaces:
 
-#### PodLifecylceHandler
+#### PodLifecycleHandler
 
 When pods are created, updated, or deleted from Kubernetes, these methods are
 called to handle those actions.
 
-[godoc#PodLifecylceHandler](https://godoc.org/github.com/virtual-kubelet/virtual-kubelet/node#PodLifecycleHandler)
+[godoc#PodLifecycleHandler](https://godoc.org/github.com/virtual-kubelet/virtual-kubelet/node#PodLifecycleHandler)
 
 ```go
 type PodLifecycleHandler interface {
@@ -264,6 +282,11 @@ One of the roles of a Kubelet is to accept requests from the API server for
 things like `kubectl logs` and  `kubectl exec`. Helpers for setting this up are
 provided [here](https://godoc.org/github.com/virtual-kubelet/virtual-kubelet/node/api)
 
+#### Scrape Pod metrics
+
+If you want to use HPA(Horizontal Pod Autoscaler) in your cluster, the provider should implement the `GetStatsSummary` function. Then metrics-server will be able to get the metrics of the pods on virtual-kubelet. Otherwise, you may see `No metrics for pod ` on metrics-server, which means the metrics of the pods on virtual-kubelet are not collected.
+
+
 ## Testing
 
 ### Unit tests
@@ -293,9 +316,8 @@ Enable the ServiceNodeExclusion flag, by modifying the Controller Manager manife
 Virtual Kubelet follows the [CNCF Code of Conduct](https://github.com/cncf/foundation/blob/master/code-of-conduct.md).
 Sign the [CNCF CLA](https://github.com/kubernetes/community/blob/master/CLA.md) to be able to make Pull Requests to this repo.
 
-Monthly Virtual Kubelet Office Hours are held at 10am PST on the last Thursday of every month in this [zoom meeting room](https://zoom.us/j/94701509915). Check out the calendar [here](https://calendar.google.com/calendar?cid=bjRtbGMxYWNtNXR0NXQ1a2hqZmRkNTRncGNAZ3JvdXAuY2FsZW5kYXIuZ29vZ2xlLmNvbQ).
+Monthly Virtual Kubelet Office Hours are held at 10am PST on the second Thursday of every month in this [zoom meeting room](https://zoom.us/j/94701509915). Check out the calendar [here](https://calendar.google.com/calendar/embed?src=b119ced62134053de07d6c261b50d21ebde0da54f4163f5771b60ecf906e8b90%40group.calendar.google.com&ctz=America%2FLos_Angeles).
 
 Our google drive with design specifications and meeting notes are [here](https://drive.google.com/drive/folders/19Ndu11WBCCBDowo9CrrGUHoIfd2L8Ueg?usp=sharing).
 
-We also have a community slack channel named virtual-kubelet in the Kubernetes slack. You can also connect with the Virtual Kubelet community via the [mailing list](virtualkubelet-dev@lists.cncf.io).
-
+We also have a community slack channel named virtual-kubelet in the Kubernetes slack. You can also connect with the Virtual Kubelet community via the [mailing list](https://lists.cncf.io/g/virtualkubelet-dev).

charts/README.md

@@ -1,14 +0,0 @@
-# The Virtual Kubelet Helm chart
-
-Each version of Virtual Kubelet has a dedicated [Helm](https://helm.sh) chart. Those charts are served as static assets directly from GitHub.
-
-## The `index.yaml` file
-
-This subdirectory has an `index.yaml` file, which is necessary for it to act as a Helm chart repository. To re-generate the `index.yaml` file (assuming that you have Helm installed):
-
-```shell
-cd /path/to/virtual-kubelet
-helm repo index charts
-```
-
-The `index.yaml` then needs to be committed to Git and merged to `master`.

charts/index.yaml

@@ -1,231 +0,0 @@
-apiVersion: v1
-entries:
-  virtual-kubelet:
-  - appVersion: "0.5"
-    created: 2019-01-28T11:18:23.622097-08:00
-    description: A Helm chart to install virtual kubelet inside a Kubernetes cluster.
-    digest: b4af3cc07e1a914b862ebcd05facbf155b16e098a138fcd7f5dc8ac715aabfa2
-    icon: https://avatars2.githubusercontent.com/u/34250142
-    maintainers:
-    - email: junjiez@microsoft.com
-      name: Robbie Zhang
-    name: virtual-kubelet
-    sources:
-    - https://github.com/virtual-kubelet/virtual-kubelet
-    urls:
-    - virtual-kubelet-0.5.0.tgz
-    version: 0.5.0
-  - appVersion: "0.5"
-    created: 2019-01-28T11:18:23.62762-08:00
-    description: A Helm chart to install virtual kubelet inside a Kubernetes cluster.
-    digest: b4af3cc07e1a914b862ebcd05facbf155b16e098a138fcd7f5dc8ac715aabfa2
-    icon: https://avatars2.githubusercontent.com/u/34250142
-    maintainers:
-    - email: junjiez@microsoft.com
-      name: Robbie Zhang
-    name: virtual-kubelet
-    sources:
-    - https://github.com/virtual-kubelet/virtual-kubelet
-    urls:
-    - virtual-kubelet-latest.tgz
-    version: 0.5.0
-  - appVersion: "0.4"
-    created: 2019-01-28T11:18:23.62156-08:00
-    description: A Helm chart to install virtual kubelet inside a Kubernetes cluster.
-    digest: ae4b8be9d69129f1002ea2228848ec790ea1280d9ff0f7dd99d0d6e3e13922f2
-    icon: https://avatars2.githubusercontent.com/u/34250142
-    maintainers:
-    - email: junjiez@microsoft.com
-      name: Robbie Zhang
-    name: virtual-kubelet
-    sources:
-    - https://github.com/virtual-kubelet/virtual-kubelet
-    urls:
-    - virtual-kubelet-0.4.0.tgz
-    version: 0.4.0
-  - appVersion: "0.3"
-    created: 2019-01-28T11:18:23.620905-08:00
-    description: A Helm chart to install virtual kubelet inside a Kubernetes cluster.
-    digest: f472f181c0724420d4f12218f8fc7f65d09fa02a8292f418d1537bf71231d643
-    icon: https://avatars2.githubusercontent.com/u/34250142
-    maintainers:
-    - email: junjiez@microsoft.com
-      name: Robbie Zhang
-    name: virtual-kubelet
-    sources:
-    - https://github.com/virtual-kubelet/virtual-kubelet
-    urls:
-    - virtual-kubelet-0.3.0.tgz
-    version: 0.3.0
-  - appVersion: "0.3"
-    created: 2019-01-28T11:18:23.6202-08:00
-    description: A Helm chart to install virtual kubelet inside a Kubernetes cluster.
-    digest: 5a2d0a269620ffe49498686161e853f49761ca4f905577fe1b8e552ab85fcaca
-    icon: https://avatars2.githubusercontent.com/u/34250142
-    maintainers:
-    - email: junjiez@microsoft.com
-      name: Robbie Zhang
-    name: virtual-kubelet
-    sources:
-    - https://github.com/virtual-kubelet/virtual-kubelet
-    urls:
-    - virtual-kubelet-0.2.0.tgz
-    version: 0.2.0
-  - created: 2019-01-28T11:18:23.619614-08:00
-    description: a Helm chart to install virtual kubelet inside a Kubernetes cluster.
-    digest: be2778949548cfb0cebe638cbe044d89045eb34ffc8d62180b86ff2f0f30b323
-    maintainers:
-    - email: junjiez@microsoft.com
-      name: Robbie Zhang
-    name: virtual-kubelet
-    sources:
-    - https://github.com/virtual-kubelet/virtual-kubelet
-    urls:
-    - virtual-kubelet-0.1.3.tgz
-    version: 0.1.3
-  - created: 2019-01-28T11:18:23.618974-08:00
-    description: a Helm chart to install virtual kubelet inside a Kubernetes cluster.
-    digest: b8519c66766d06a68671a2b8940cf44cccdf9321f144dead543f62d16325331e
-    maintainers:
-    - email: junjiez@microsoft.com
-      name: Robbie Zhang
-    name: virtual-kubelet
-    sources:
-    - https://github.com/virtual-kubelet/virtual-kubelet
-    urls:
-    - virtual-kubelet-0.1.2.tgz
-    version: 0.1.2
-  - created: 2019-01-28T11:18:23.618406-08:00
-    description: a Helm chart to install virtual kubelet inside a Kubernetes cluster.
-    digest: b15b3d5acde2cc264b5e8624c3bafcc249440c662ae353ef7012493eb0b6abf6
-    maintainers:
-    - email: junjiez@microsoft.com
-      name: Robbie Zhang
-    name: virtual-kubelet
-    sources:
-    - https://github.com/virtual-kubelet/virtual-kubelet
-    urls:
-    - virtual-kubelet-0.1.1.tgz
-    version: 0.1.1
-  - created: 2019-01-28T11:18:23.617865-08:00
-    description: a Helm chart to install virtual kubelet inside a Kubernetes cluster.
-    digest: 22c60ad2f7ea71abf58d65e52b91c2b9feca1ad6a15091321f7f12e5c43ecf81
-    maintainers:
-    - email: junjiez@microsoft.com
-      name: Robbie Zhang
-    name: virtual-kubelet
-    sources:
-    - https://github.com/virtual-kubelet/virtual-kubelet
-    urls:
-    - virtual-kubelet-0.1.0.tgz
-    version: 0.1.0
-  virtual-kubelet-for-aks:
-  - created: 2019-01-28T11:18:23.622632-08:00
-    description: a Helm chart to install virtual kubelet in an AKS or ACS cluster.
-    digest: 345cda5aeb537129e1ecc3d23c0cf47651454f672694a4402a7bbb5d3f3a91ba
-    maintainers:
-    - email: junjiez@microsoft.com
-      name: Robbie Zhang
-    name: virtual-kubelet-for-aks
-    sources:
-    - https://github.com/virtual-kubelet/virtual-kubelet
-    urls:
-    - virtual-kubelet-for-aks-0.1.10.tgz
-    version: 0.1.10
-  - created: 2019-01-28T11:18:23.627017-08:00
-    description: a Helm chart to install virtual kubelet in an AKS or ACS cluster.
-    digest: 345cda5aeb537129e1ecc3d23c0cf47651454f672694a4402a7bbb5d3f3a91ba
-    maintainers:
-    - email: junjiez@microsoft.com
-      name: Robbie Zhang
-    name: virtual-kubelet-for-aks
-    sources:
-    - https://github.com/virtual-kubelet/virtual-kubelet
-    urls:
-    - virtual-kubelet-for-aks-latest.tgz
-    version: 0.1.10
-  - created: 2019-01-28T11:18:23.626524-08:00
-    description: a Helm chart to install virtual kubelet in an AKS or ACS cluster.
-    digest: 4a30821657ff4ef4522060c9905e5659656c035126a7a9e650dd9bb54621b9eb
-    maintainers:
-    - email: junjiez@microsoft.com
-      name: Robbie Zhang
-    name: virtual-kubelet-for-aks
-    sources:
-    - https://github.com/virtual-kubelet/virtual-kubelet
-    urls:
-    - virtual-kubelet-for-aks-0.1.9.tgz
-    version: 0.1.9
-  - created: 2019-01-28T11:18:23.626081-08:00
-    description: a Helm chart to install virtual kubelet in an AKS or ACS cluster.
-    digest: 18d988bfb4d24b3674c6c690c0445b85cf3969471cfce74f7c49e87483cb497d
-    maintainers:
-    - email: junjiez@microsoft.com
-      name: Robbie Zhang
-    name: virtual-kubelet-for-aks
-    sources:
-    - https://github.com/virtual-kubelet/virtual-kubelet
-    urls:
-    - virtual-kubelet-for-aks-0.1.8.tgz
-    version: 0.1.8
-  - created: 2019-01-28T11:18:23.625442-08:00
-    description: a Helm chart to install virtual kubelet in an AKS or ACS cluster.
-    digest: 21c0719fe330981a170810ee4c3e84375106c176de08894827c8208a66f52112
-    maintainers:
-    - email: junjiez@microsoft.com
-      name: Robbie Zhang
-    name: virtual-kubelet-for-aks
-    sources:
-    - https://github.com/virtual-kubelet/virtual-kubelet
-    urls:
-    - virtual-kubelet-for-aks-0.1.7.tgz
-    version: 0.1.7
-  - created: 2019-01-28T11:18:23.624524-08:00
-    description: a Helm chart to install virtual kubelet in an AKS or ACS cluster.
-    digest: b5e72d03f04113a46350fa800135a67f5af9b4ffe3d6650bdeebd271b885e5aa
-    maintainers:
-    - email: junjiez@microsoft.com
-      name: Robbie Zhang
-    name: virtual-kubelet-for-aks
-    sources:
-    - https://github.com/virtual-kubelet/virtual-kubelet
-    urls:
-    - virtual-kubelet-for-aks-0.1.6.tgz
-    version: 0.1.6
-  - created: 2019-01-28T11:18:23.624071-08:00
-    description: a Helm chart to install virtual kubelet in an AKS or ACS cluster.
-    digest: 5b04abcc63dcc71b5ad25c8368ad1b114accf721e9c475c5a7f962a48fb0ed65
-    maintainers:
-    - email: junjiez@microsoft.com
-      name: Robbie Zhang
-    name: virtual-kubelet-for-aks
-    sources:
-    - https://github.com/virtual-kubelet/virtual-kubelet
-    urls:
-    - virtual-kubelet-for-aks-0.1.5.tgz
-    version: 0.1.5
-  - created: 2019-01-28T11:18:23.623547-08:00
-    description: a Helm chart to install virtual kubelet in an AKS or ACS cluster.
-    digest: b639126041dc4f3d0307f6a678d22021ba149f28612eb0bfc3903c75cf1ea414
-    maintainers:
-    - email: junjiez@microsoft.com
-      name: Robbie Zhang
-    name: virtual-kubelet-for-aks
-    sources:
-    - https://github.com/virtual-kubelet/virtual-kubelet
-    urls:
-    - virtual-kubelet-for-aks-0.1.4.tgz
-    version: 0.1.4
-  - created: 2019-01-28T11:18:23.623097-08:00
-    description: a Helm chart to install virtual kubelet in an AKS or ACS cluster.
-    digest: 1b428fd99c667482681c83b61753e7327b85ec2644ef4fa0e9366492a0e73cd7
-    maintainers:
-    - email: junjiez@microsoft.com
-      name: Robbie Zhang
-    name: virtual-kubelet-for-aks
-    sources:
-    - https://github.com/virtual-kubelet/virtual-kubelet
-    urls:
-    - virtual-kubelet-for-aks-0.1.3.tgz
-    version: 0.1.3
-generated: 2019-01-28T11:18:23.615432-08:00

charts/virtual-kubelet-for-aks/Chart.yaml

@@ -1,8 +0,0 @@
-name: virtual-kubelet-for-aks
-version: 0.1.10
-description: a Helm chart to install virtual kubelet in an AKS or ACS cluster.
-sources:
-  - https://github.com/virtual-kubelet/virtual-kubelet
-maintainers:
-  - name: Robbie Zhang
-    email: junjiez@microsoft.com

charts/virtual-kubelet-for-aks/templates/NOTES.txt

@@ -1,12 +0,0 @@
-The virtual kubelet is getting deployed on your cluster.
-
-To verify that virtual kubelet has started, run:
-
-  kubectl --namespace={{ .Release.Namespace }} get pods -l "app={{ template "fullname" . }}"
-
-{{- if (not .Values.env.apiserverCert) and (not .Values.env.apiserverKey) }}
-
-Note: 
-TLS key pair not provided for VK HTTP listener. A key pair was generated for you. This generated key pair is not suitable for production use.
-
-{{- end }}

charts/virtual-kubelet-for-aks/templates/_helpers.tpl

@@ -1,16 +0,0 @@
-{{/* vim: set filetype=mustache: */}}
-{{/*
-Expand the name of the chart.
-*/}}
-{{- define "name" -}}
-{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
-{{- end -}}
-
-{{/*
-Create a default fully qualified app name.
-We truncate at 24 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
-*/}}
-{{- define "fullname" -}}
-{{- $name := default .Chart.Name .Values.nameOverride -}}
-{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
-{{- end -}}

charts/virtual-kubelet-for-aks/templates/clusterrolebinding.yaml

@@ -1,14 +0,0 @@
-{{ if .Values.rbac.install }}
-apiVersion: "rbac.authorization.k8s.io/{{ .Values.rbac.apiVersion }}"
-kind: ClusterRoleBinding
-metadata:
-  name: {{ template "fullname" . }}
-subjects:
-- kind: ServiceAccount
-  name: {{ template "fullname" . }}
-  namespace: {{ .Release.Namespace }}
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: {{ .Values.rbac.roleRef }}
-{{ end }}

charts/virtual-kubelet-for-aks/templates/deployment.yaml

@@ -1,79 +0,0 @@
-apiVersion: extensions/v1beta1
-kind: Deployment
-metadata:
-  name: {{ template "fullname" . }}
-spec:
-  replicas: 1
-  template:
-    metadata:
-      labels:
-        app: {{ template "fullname" . }}
-    spec:
-      containers:
-      - name: {{ template "fullname" . }}
-        image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
-        imagePullPolicy: {{ .Values.image.pullPolicy }}
-        env:
-        - name: KUBELET_PORT
-          value: "10250"
-        - name: ACS_CREDENTIAL_LOCATION
-          value: /etc/acs/azure.json
-        - name: AZURE_TENANT_ID
-          value: {{ .Values.env.azureTenantId }}
-        - name: AZURE_SUBSCRIPTION_ID
-          value: {{ .Values.env.azureSubscriptionId }}
-        - name: AZURE_CLIENT_ID
-          value: {{ .Values.env.azureClientId }}
-        - name: AZURE_CLIENT_SECRET
-          valueFrom:
-            secretKeyRef:
-              name: {{ template "fullname" . }}
-              key: clientSecret
-        - name: ACI_RESOURCE_GROUP
-          value: {{ .Values.env.aciResourceGroup }}
-        - name: ACI_REGION
-          value: {{ default "westus" .Values.env.aciRegion }}
-        - name: APISERVER_CERT_LOCATION
-          value: /etc/virtual-kubelet/cert.pem
-        - name: APISERVER_KEY_LOCATION
-          value: /etc/virtual-kubelet/key.pem
-        - name: VKUBELET_POD_IP
-          valueFrom:
-            fieldRef:
-              fieldPath: status.podIP
-        - name: ACI_EXTRA_USER_AGENT
-          value: {{ printf "helm-chart/aks/%s/%s" .Chart.Name .Chart.Version }}
-        - name: ACI_SUBNET_NAME
-          value: {{ .Values.env.aciVnetSubnetName }}
-        - name: ACI_SUBNET_CIDR
-          value: {{ .Values.env.aciVnetSubnetCIDR }}
-        - name: MASTER_URI
-          value: {{ .Values.env.masterUri }}
-        - name: CLUSTER_CIDR
-          value: {{ .Values.env.clusterCIDR }}
-        - name: KUBE_DNS_IP
-          value: {{ .Values.env.kubeDnsIP }}
-        {{ if .Values.loganalytics.enabled }}
-        - name: LOG_ANALYTICS_AUTH_LOCATION
-          value: /etc/virtual-kubelet/loganalytics.json
-        {{ end }}
-        volumeMounts:
-        - name: credentials
-          mountPath: "/etc/virtual-kubelet"
-        - name: acs-credential
-          mountPath: "/etc/acs/azure.json"
-        command: ["virtual-kubelet"]
-        args: ["--provider", "azure", "--namespace", {{ default "" .Values.env.monitoredNamespace | quote }}, "--nodename", {{ default "virtual-kubelet" .Values.env.nodeName | quote }} , "--os", {{ default "Linux" .Values.env.nodeOsType | quote }} ]
-      volumes:
-      - name: credentials
-        secret:
-          secretName: {{ template "fullname" . }}
-      - name: acs-credential
-        hostPath:
-          path: /etc/kubernetes/azure.json
-          type: File
-      {{ if .Values.rbac.install }}
-      serviceAccountName: {{ template "fullname" . }}
-      {{ end }}
-      nodeSelector:
-        beta.kubernetes.io/os: linux

charts/virtual-kubelet-for-aks/templates/secrets.yaml

@@ -1,22 +0,0 @@
-apiVersion: v1
-kind: Secret
-metadata:
-  name: {{ template "fullname" . }}
-type: Opaque
-data:
-  {{- if (not .Values.env.apiserverCert) and (not .Values.env.apiserverKey) }}
-  {{- $ca := genCA "virtual-kubelet-ca" 3650 }}
-  {{- $cn := printf "%s-virtual-kubelet-apiserver" .Release.Name }}
-  {{- $altName1 := printf "%s-virtual-kubelet-apiserver.%s" .Release.Name .Release.Namespace }}
-  {{- $altName2 := printf "%s-virtual-kubelet-apiserver.%s.svc" .Release.Name .Release.Namespace }}
-  {{- $cert := genSignedCert $cn nil (list $altName1 $altName2) 3650 $ca }}
-  cert.pem: {{ b64enc $cert.Cert }}
-  key.pem: {{ b64enc $cert.Key }}
-  {{ else }}
-  cert.pem: {{ quote .Values.env.apiserverCert }}
-  key.pem: {{ quote .Values.env.apiserverKey }}
-  {{ end}}
-  clientSecret: {{ default "" .Values.env.azureClientKey | b64enc | quote }}
-  {{ if .Values.loganalytics.enabled }}
-  loganalytics.json: {{ printf "{\"workspaceID\": \"%s\",\"workspaceKey\": \"%s\"}" (required "workspaceID is required for loganalytics" .Values.loganalytics.workspaceID ) (required "workspaceKey is required for loganalytics" .Values.loganalytics.workspaceKey ) | b64enc | quote }}
-  {{ end }}

charts/virtual-kubelet-for-aks/templates/serviceaccount.yaml

@@ -1,6 +0,0 @@
-{{ if .Values.rbac.install }}
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: {{ template "fullname" . }}
-{{ end }}

charts/virtual-kubelet-for-aks/values.yaml

@@ -1,34 +0,0 @@
-image:
-  repository: microsoft/virtual-kubelet
-  tag: latest
-  pullPolicy: Always
-env:
-  azureClientId: 
-  azureClientKey: 
-  azureTenantId: 
-  azureSubscriptionId: 
-  aciResourceGroup: 
-  aciRegion: 
-  nodeName: 
-  nodeTaint: 
-  nodeOsType: 
-  apiserverCert: 
-  apiserverKey: 
-  monitoredNamespace:
-  aciVnetSubnetName:
-  aciVnetSubnetCidr:
-  masterUri:
-  clusterCidr:
-  kubeDnsIp:
-loganalytics:
-  enabled: false
-  workspaceID:
-  workspaceKey:
-
-# Install Default RBAC roles and bindings
-rbac:
-  install: true
-  ## RBAC api version
-  apiVersion: v1beta1
-  # Cluster role reference
-  roleRef: cluster-admin

charts/virtual-kubelet/Chart.yaml

@@ -1,10 +0,0 @@
-name: virtual-kubelet
-version: 0.5.0
-appVersion: 0.5
-description: A Helm chart to install virtual kubelet inside a Kubernetes cluster.
-icon: https://avatars2.githubusercontent.com/u/34250142
-sources:
-  - https://github.com/virtual-kubelet/virtual-kubelet
-maintainers:
-  - name: Robbie Zhang
-    email: junjiez@microsoft.com

charts/virtual-kubelet/templates/NOTES.txt

@@ -1,12 +0,0 @@
-The virtual kubelet is getting deployed on your cluster.
-
-To verify that virtual kubelet has started, run:
-
-  kubectl --namespace={{ .Release.Namespace }} get pods -l "app={{ template "vk.name" . }}"
-
-{{- if (not .Values.apiserverCert) and (not .Values.apiserverKey) }}
-
-Note: 
-TLS key pair not provided for VK HTTP listener. A key pair was generated for you. This generated key pair is not suitable for production use.
-
-{{- end }}

charts/virtual-kubelet/templates/_helpers.tpl

@@ -1,29 +0,0 @@
-{{/* vim: set filetype=mustache: */}}
-{{/*
-Expand the name of the chart.
-*/}}
-{{- define "vk.name" -}}
-{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
-{{- end -}}
-
-{{/*
-Create a default fully qualified app name.
-We truncate at 24 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
-*/}}
-{{- define "vk.fullname" -}}
-{{- $name := default .Chart.Name .Values.nameOverride -}}
-{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
-{{- end -}}
-
-{{/*
-Standard labels for helm resources
-*/}}
-{{- define "vk.labels" -}}
-labels:
-  heritage: "{{ .Release.Service }}"
-  release: "{{ .Release.Name }}"
-  revision: "{{ .Release.Revision }}"
-  chart: "{{ .Chart.Name }}"
-  chartVersion: "{{ .Chart.Version }}"
-  app: {{ template "vk.name" . }}
-{{- end -}}

charts/virtual-kubelet/templates/clusterrolebinding.yaml

@@ -1,15 +0,0 @@
-{{ if .Values.rbac.install }}
-apiVersion: "rbac.authorization.k8s.io/{{ .Values.rbac.apiVersion }}"
-kind: ClusterRoleBinding
-metadata:
-  name: {{ template "vk.fullname" . }}
-{{ include "vk.labels" . | indent 2 }}
-subjects:
-- kind: ServiceAccount
-  name: {{ template "vk.fullname" . }}
-  namespace: {{ .Release.Namespace }}
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: {{ .Values.rbac.roleRef }}
-{{ end }}

charts/virtual-kubelet/templates/deployment.yaml

@@ -1,159 +0,0 @@
-apiVersion: extensions/v1beta1
-kind: Deployment
-metadata:
-  name: {{ template "vk.fullname" . }}
-{{ include "vk.labels" . | indent 2 }}
-    component: kubelet
-spec:
-  replicas: 1
-  template:
-    metadata:
-{{ include "vk.labels" . | indent 6 }}
-        component: kubelet
-      annotations:
-        checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }}
-    spec:
-      containers:
-{{- if eq .Values.trace.exporter "jaeger" }}
-{{- with .Values.traceExporters.jaeger }}
-{{- if eq .endpoint "" }}
-      - name: {{ tpl .name $ }}
-        image: {{ .image.repository}}:{{.image.tag}}
-        imagePullPolicy: {{ .image.pullPolicy }}
-        ports:
-          - containerPort: 16686
-{{- end }}
-{{- end }}
-{{- end }}
-      - name: {{ template "vk.fullname" . }}
-        image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
-        imagePullPolicy: {{ .Values.image.pullPolicy }}
-        env:
-        - name: KUBELET_PORT
-          value: "10250"
-        - name: APISERVER_CERT_LOCATION
-          value: /etc/virtual-kubelet/cert.pem
-        - name: APISERVER_KEY_LOCATION
-          value: /etc/virtual-kubelet/key.pem
-        - name: VKUBELET_POD_IP
-          valueFrom:
-            fieldRef:
-              fieldPath: status.podIP
-        - name: VKUBELET_TAINT_KEY
-          value: {{ .Values.taint.key }}
-        - name: VKUBELET_TAINT_VALUE
-          value: {{ tpl .Values.taint.value $ }}
-        - name: VKUBELET_TAINT_EFFECT
-          value: {{ .Values.taint.effect }}
-{{- if eq (required "You must specify a Virtual Kubelet provider" .Values.provider) "azure" }}
-{{- with .Values.providers.azure }}
-{{- if .loganalytics.enabled }}
-        - name: LOG_ANALYTICS_AUTH_LOCATION
-          value: /etc/virtual-kubelet/loganalytics.json
-        - name: CLUSTER_RESOURCE_ID
-          value:  {{ .loganalytics.clusterResourceId }}
-{{- end }}
-{{- if .targetAKS }}
-        - name: ACS_CREDENTIAL_LOCATION
-          value: /etc/acs/azure.json
-        - name: AZURE_TENANT_ID
-          value: {{ .tenantId }}
-        - name: AZURE_SUBSCRIPTION_ID
-          value: {{ .subscriptionId }}
-        - name: AZURE_CLIENT_ID
-          value: {{ .clientId }}
-        - name: AZURE_CLIENT_SECRET
-          valueFrom:
-            secretKeyRef:
-              name: {{ template "vk.fullname" $ }}
-              key: clientSecret
-        - name: ACI_RESOURCE_GROUP
-          value: {{ .aciResourceGroup }}
-        - name: ACI_REGION
-          value: {{ .aciRegion }}
-        - name: ACI_EXTRA_USER_AGENT
-          value: {{ printf "helm-chart/aks/%s/%s" $.Chart.Name $.Chart.Version }}
-{{- else }}
-        - name: AZURE_AUTH_LOCATION
-          value: /etc/virtual-kubelet/credentials.json
-        - name: ACI_RESOURCE_GROUP
-          value: {{ required "aciResourceGroup is required" .aciResourceGroup }}
-        - name: ACI_REGION
-          value: {{ required "aciRegion is required" .aciRegion }}
-        - name: ACI_EXTRA_USER_AGENT
-          value: {{ printf "helm-chart/other/%s/%s" $.Chart.Name $.Chart.Version }}
-{{- end }}
-{{- if .vnet.enabled }}
-        - name: ACI_SUBNET_NAME
-          value: {{ required "subnetName is required" .vnet.subnetName }}
-        - name: ACI_SUBNET_CIDR
-          value: {{ .vnet.subnetCidr }}
-        - name: MASTER_URI
-          value: {{ required "masterUri is required" .masterUri }}
-        - name: CLUSTER_CIDR
-          value: {{ required "clusterCidr is required" .vnet.clusterCidr }}
-        - name: KUBE_DNS_IP
-          value: {{ required "kubeDnsIp is required" .vnet.kubeDnsIp }}
-{{- else }}
-        - name: MASTER_URI
-          value: {{ .masterUri }}
-{{- end }}          
-{{- end }}
-{{- end }}
-{{- if eq .Values.trace.exporter "jaeger" }}
-        - name: JAEGER_ENDPOINT
-{{- with .Values.traceExporters.jaeger }}
-{{- if eq .endpoint "" }}
-          value: "http://127.0.0.1:14268"
-{{- else }}
-          value: {{.endpoint}}
-{{- end }}
-{{- end }}
-{{- end }}
-        volumeMounts:
-        - name: credentials
-          mountPath: "/etc/virtual-kubelet"
-{{- if eq (required "You must specify a Virtual Kubelet provider" .Values.provider) "azure" }}
-{{- if .Values.providers.azure.targetAKS }}
-        - name: acs-credential
-          mountPath: "/etc/acs/azure.json"
-{{- end }}
-{{- end }}
-        command: ["virtual-kubelet"]
-        args: [
-{{- if not .Values.taint.enabled }}
-          "--disable-taint", "true",
-{{- end }}
-          "--provider", "{{ required "You must specify a Virtual Kubelet provider" .Values.provider }}",
-          "--namespace", "{{ .Values.monitoredNamespace }}",
-          "--nodename", "{{ required "nodeName is required" .Values.nodeName }}",
-          {{- if .Values.logLevel }}
-          "--log-level", "{{.Values.logLevel}}",
-          {{- end }}
-{{- if ne .Values.trace.exporter "" }}
-          "--trace-exporter", "{{ .Values.trace.exporter }}",
-{{- if gt .Values.trace.sampleRate 0.0 }}
-          "--trace-sample-rate", "{{ .Values.trace.sampleRate }}",
-{{- end }}
-{{- $serviceName := tpl .Values.trace.serviceName $ }}
-{{- if ne $serviceName "" }}
-          "--trace-service-name", "{{ $serviceName }}",
-{{- end}}
-{{- end}}
-          "--os", "{{ .Values.nodeOsType }}"
-        ]
-      volumes:
-      - name: credentials
-        secret:
-          secretName: {{ template "vk.fullname" . }}
-{{- if eq (required "You must specify a Virtual Kubelet provider" .Values.provider) "azure" }}
-{{- if .Values.providers.azure.targetAKS }}
-      - name: acs-credential
-        hostPath:
-          path: /etc/kubernetes/azure.json
-          type: File
-{{- end }}
-{{- end }}
-      serviceAccountName: {{ if .Values.rbac.install }} "{{ template "vk.fullname" . }}" {{ end }}
-      nodeSelector:
-        beta.kubernetes.io/os: linux

charts/virtual-kubelet/templates/secret.yaml

@@ -1,31 +0,0 @@
-apiVersion: v1
-kind: Secret
-metadata:
-  name: {{ template "vk.fullname" . }}
-{{ include "vk.labels" . | indent 2 }}
-type: Opaque
-data:
-{{- if (not .Values.apiserverCert) and (not .Values.apiserverKey) }}
-{{- $ca := genCA "virtual-kubelet-ca" 3650 }}
-{{- $cn := printf "%s-virtual-kubelet-apiserver" .Release.Name }}
-{{- $altName1 := printf "%s-virtual-kubelet-apiserver.%s" .Release.Name .Release.Namespace }}
-{{- $altName2 := printf "%s-virtual-kubelet-apiserver.%s.svc" .Release.Name .Release.Namespace }}
-{{- $cert := genSignedCert $cn nil (list $altName1 $altName2) 3650 $ca }}
-  cert.pem: {{ b64enc $cert.Cert }}
-  key.pem: {{ b64enc $cert.Key }}
-{{- else }}
-  cert.pem: {{ quote .Values.apiserverCert }}
-  key.pem: {{ quote .Values.apiserverKey }}
-{{- end }}
-{{- if eq (required "You must specify a Virtual Kubelet provider" .Values.provider) "azure" }}
-{{- with .Values.providers.azure }}
-{{- if .loganalytics.enabled }}
-  loganalytics.json: {{ printf "{\"workspaceID\": \"%s\",\"workspaceKey\": \"%s\"}" (required "workspaceId is required for loganalytics" .loganalytics.workspaceId ) (required "workspaceKey is required for loganalytics" .loganalytics.workspaceKey ) | b64enc | quote }}
-{{- end }}
-{{- if .targetAKS }}
-  clientSecret: {{ default "" .clientKey | b64enc | quote }}
-{{- else }}
-  credentials.json: {{ printf "{ \"clientId\": \"%s\", \"clientSecret\": \"%s\", \"subscriptionId\": \"%s\", \"tenantId\": \"%s\", \"activeDirectoryEndpointUrl\": \"https://login.microsoftonline.com/\", \"resourceManagerEndpointUrl\": \"https://management.azure.com/\", \"activeDirectoryGraphResourceId\": \"https://graph.windows.net/\", \"sqlManagementEndpointUrl\": \"database.windows.net\", \"galleryEndpointUrl\": \"https://gallery.azure.com/\", \"managementEndpointUrl\": \"https://management.core.windows.net/\" }" (default "MISSING" .clientId) (default "MISSING" .clientKey) (default "MISSING" .subscriptionId) (default "MISSING" .tenantId) | b64enc | quote }}
-{{- end }}
-{{- end }}
-{{- end }}

charts/virtual-kubelet/templates/serviceaccount.yaml

@@ -1,7 +0,0 @@
-{{ if .Values.rbac.install }}
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: {{ template "vk.fullname" . }}
-{{ include "vk.labels" . | indent 2 }}
-{{ end }}

charts/virtual-kubelet/templates/tests/helloworld.yaml

@@ -1,30 +0,0 @@
-apiVersion: v1
-kind: Pod
-metadata:
-  name: "{{ .Release.Name }}-{{ .Release.Revision }}-test"
-{{ include "vk.labels" . | indent 2 }}
-    component: test
-  annotations:
-    "helm.sh/hook": test-success
-spec:
-  containers:
-  - image: hello-world:linux
-    imagePullPolicy: Always
-    name: helloworld
-    resources:
-      requests:
-        memory: "0.1G"
-        cpu: 10m
-      limits:
-        memory: "0.1G"
-        cpu: 10m
-  dnsPolicy: ClusterFirst
-  nodeSelector:
-    kubernetes.io/hostname: "{{ .Values.nodeName }}"
-  restartPolicy: Never
-  tolerations:
-{{- if .Values.taint.enabled }}
-  - key: "{{ .Values.taint.key }}"
-    value: "{{ tpl .Values.taint.value $ }}"
-    effect: "{{ .Values.taint.effect }}"
-{{- end }}

charts/virtual-kubelet/values.yaml

@@ -1,65 +0,0 @@
-image:
-  repository: microsoft/virtual-kubelet
-  tag: latest
-  pullPolicy: Always
-
-nodeName: "virtual-kubelet"
-nodeOsType: "Linux"
-monitoredNamespace: ""
-apiserverCert: 
-apiserverKey: 
-logLevel: 
-
-taint:
-  enabled: true
-  key: virtual-kubelet.io/provider
-  value: "{{ .Values.provider }}"
-  ## `effect` must be `NoSchedule`, `PreferNoSchedule` or `NoExecute`.
-  effect: NoSchedule
-
-trace:
-  exporter: ""
-  serviceName: "{{ .Values.nodeName }}"
-  sampleRate: 0
-
-traceExporters:
-  jaeger:
-    name: "{{ .Values.trace.exporter }}"
-    endpoint: ""
-    image:
-      repository: jaegertracing/all-in-one
-      tag: 1.8
-      pullPolicy: Always
-
-providers:
-  azure:
-    ## Set to true if deploying to Azure Kubernetes Service (AKS), otherwise false
-    targetAKS: true
-    clientId: 
-    clientKey: 
-    tenantId: 
-    subscriptionId: 
-    ## `aciResourceGroup` and `aciRegion` are required only for non-AKS deployments
-    aciResourceGroup: 
-    aciRegion: 
-    masterUri: 
-    loganalytics:
-      enabled: false
-      workspaceId: 
-      workspaceKey: 
-      clusterResourceId: 
-    vnet:
-      enabled: false
-      subnetName: 
-      subnetCidr: 
-      clusterCidr: 
-      kubeDnsIp: 
-
-## Install Default RBAC roles and bindings
-rbac:
-  install: true
-  serviceAccountName: virtual-kubelet
-  ## RBAC api version
-  apiVersion: v1beta1
-  ## Cluster role reference
-  roleRef: cluster-admin

cmd/virtual-kubelet/internal/commands/root/flag.go

@@ -22,7 +22,7 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/spf13/pflag"
-	"k8s.io/klog"
+	klog "k8s.io/klog/v2"
 )
 
 type mapVar map[string]string
@@ -59,7 +59,13 @@ func (mv mapVar) Type() string {
 
 func installFlags(flags *pflag.FlagSet, c *Opts) {
 	flags.StringVar(&c.KubeConfigPath, "kubeconfig", c.KubeConfigPath, "kube config file to use for connecting to the Kubernetes API server")
+
 	flags.StringVar(&c.KubeNamespace, "namespace", c.KubeNamespace, "kubernetes namespace (default is 'all')")
+	/* #nosec */
+	flags.MarkDeprecated("namespace", "Nodes must watch for pods in all namespaces. This option is now ignored.") //nolint:errcheck
+	/* #nosec */
+	flags.MarkHidden("namespace") //nolint:errcheck
+
 	flags.StringVar(&c.KubeClusterDomain, "cluster-domain", c.KubeClusterDomain, "kubernetes cluster-domain (default is 'cluster.local')")
 	flags.StringVar(&c.NodeName, "nodename", c.NodeName, "kubernetes node name")
 	flags.StringVar(&c.OperatingSystem, "os", c.OperatingSystem, "Operating System (Linux/Windows)")
@@ -68,11 +74,18 @@ func installFlags(flags *pflag.FlagSet, c *Opts) {
 	flags.StringVar(&c.MetricsAddr, "metrics-addr", c.MetricsAddr, "address to listen for metrics/stats requests")
 
 	flags.StringVar(&c.TaintKey, "taint", c.TaintKey, "Set node taint key")
+
 	flags.BoolVar(&c.DisableTaint, "disable-taint", c.DisableTaint, "disable the virtual-kubelet node taint")
+	/* #nosec */
 	flags.MarkDeprecated("taint", "Taint key should now be configured using the VK_TAINT_KEY environment variable") //nolint:errcheck
 
 	flags.IntVar(&c.PodSyncWorkers, "pod-sync-workers", c.PodSyncWorkers, `set the number of pod synchronization workers`)
+
 	flags.BoolVar(&c.EnableNodeLease, "enable-node-lease", c.EnableNodeLease, `use node leases (1.13) for node heartbeats`)
+	/* #nosec */
+	flags.MarkDeprecated("enable-node-lease", "leases are always enabled") //nolint:errcheck
+	/* #nosec */
+	flags.MarkHidden("enable-node-lease") //nolint:errcheck
 
 	flags.StringSliceVar(&c.TraceExporters, "trace-exporter", c.TraceExporters, fmt.Sprintf("sets the tracing exporter to use, available exporters: %s", AvailableTraceExporters()))
 	flags.StringVar(&c.TraceConfig.ServiceName, "trace-service-name", c.TraceConfig.ServiceName, "sets the name of the service used to register with the trace exporter")

cmd/virtual-kubelet/internal/commands/root/http.go

@@ -15,156 +15,22 @@
 package root
 
 import (
-	"context"
-	"crypto/tls"
 	"fmt"
-	"io"
-	"net"
-	"net/http"
-	"os"
 	"time"
-
-	"github.com/pkg/errors"
-	"github.com/virtual-kubelet/virtual-kubelet/cmd/virtual-kubelet/internal/provider"
-	"github.com/virtual-kubelet/virtual-kubelet/log"
-	"github.com/virtual-kubelet/virtual-kubelet/node/api"
 )
 
-// AcceptedCiphers is the list of accepted TLS ciphers, with known weak ciphers elided
-// Note this list should be a moving target.
-var AcceptedCiphers = []uint16{
-	tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
-	tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
-	tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
-	tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
-
-	tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
-	tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
-	tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
-	tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
-}
-
-func loadTLSConfig(certPath, keyPath string) (*tls.Config, error) {
-	cert, err := tls.LoadX509KeyPair(certPath, keyPath)
-	if err != nil {
-		return nil, errors.Wrap(err, "error loading tls certs")
-	}
-
-	return &tls.Config{
-		Certificates:             []tls.Certificate{cert},
-		MinVersion:               tls.VersionTLS12,
-		PreferServerCipherSuites: true,
-		CipherSuites:             AcceptedCiphers,
-	}, nil
-}
-
-func setupHTTPServer(ctx context.Context, p provider.Provider, cfg *apiServerConfig, getPodsFromKubernetes api.PodListerFunc) (_ func(), retErr error) {
-	var closers []io.Closer
-	cancel := func() {
-		for _, c := range closers {
-			c.Close()
-		}
-	}
-	defer func() {
-		if retErr != nil {
-			cancel()
-		}
-	}()
-
-	if cfg.CertPath == "" || cfg.KeyPath == "" {
-		log.G(ctx).
-			WithField("certPath", cfg.CertPath).
-			WithField("keyPath", cfg.KeyPath).
-			Error("TLS certificates not provided, not setting up pod http server")
-	} else {
-		tlsCfg, err := loadTLSConfig(cfg.CertPath, cfg.KeyPath)
-		if err != nil {
-			return nil, err
-		}
-		l, err := tls.Listen("tcp", cfg.Addr, tlsCfg)
-		if err != nil {
-			return nil, errors.Wrap(err, "error setting up listener for pod http server")
-		}
-
-		mux := http.NewServeMux()
-
-		podRoutes := api.PodHandlerConfig{
-			RunInContainer:        p.RunInContainer,
-			GetContainerLogs:      p.GetContainerLogs,
-			GetPodsFromKubernetes: getPodsFromKubernetes,
-			GetPods:               p.GetPods,
-			StreamIdleTimeout:     cfg.StreamIdleTimeout,
-			StreamCreationTimeout: cfg.StreamCreationTimeout,
-		}
-
-		api.AttachPodRoutes(podRoutes, mux, true)
-
-		s := &http.Server{
-			Handler:   mux,
-			TLSConfig: tlsCfg,
-		}
-		go serveHTTP(ctx, s, l, "pods")
-		closers = append(closers, s)
-	}
-
-	if cfg.MetricsAddr == "" {
-		log.G(ctx).Info("Pod metrics server not setup due to empty metrics address")
-	} else {
-		l, err := net.Listen("tcp", cfg.MetricsAddr)
-		if err != nil {
-			return nil, errors.Wrap(err, "could not setup listener for pod metrics http server")
-		}
-
-		mux := http.NewServeMux()
-
-		var summaryHandlerFunc api.PodStatsSummaryHandlerFunc
-		if mp, ok := p.(provider.PodMetricsProvider); ok {
-			summaryHandlerFunc = mp.GetStatsSummary
-		}
-		podMetricsRoutes := api.PodMetricsConfig{
-			GetStatsSummary: summaryHandlerFunc,
-		}
-		api.AttachPodMetricsRoutes(podMetricsRoutes, mux)
-		s := &http.Server{
-			Handler: mux,
-		}
-		go serveHTTP(ctx, s, l, "pod metrics")
-		closers = append(closers, s)
-	}
-
-	return cancel, nil
-}
-
-func serveHTTP(ctx context.Context, s *http.Server, l net.Listener, name string) {
-	if err := s.Serve(l); err != nil {
-		select {
-		case <-ctx.Done():
-		default:
-			log.G(ctx).WithError(err).Errorf("Error setting up %s http server", name)
-		}
-	}
-	l.Close()
-}
-
 type apiServerConfig struct {
-	CertPath              string
-	KeyPath               string
 	Addr                  string
 	MetricsAddr           string
 	StreamIdleTimeout     time.Duration
 	StreamCreationTimeout time.Duration
 }
 
-func getAPIConfig(c Opts) (*apiServerConfig, error) {
-	config := apiServerConfig{
-		CertPath: os.Getenv("APISERVER_CERT_LOCATION"),
-		KeyPath:  os.Getenv("APISERVER_KEY_LOCATION"),
+func getAPIConfig(c Opts) apiServerConfig {
+	return apiServerConfig{
+		Addr:                  fmt.Sprintf(":%d", c.ListenPort),
+		MetricsAddr:           c.MetricsAddr,
+		StreamIdleTimeout:     c.StreamIdleTimeout,
+		StreamCreationTimeout: c.StreamCreationTimeout,
 	}
-
-	config.Addr = fmt.Sprintf(":%d", c.ListenPort)
-	config.MetricsAddr = c.MetricsAddr
-	config.StreamIdleTimeout = c.StreamIdleTimeout
-	config.StreamCreationTimeout = c.StreamCreationTimeout
-
-	return &config, nil
 }

cmd/virtual-kubelet/internal/commands/root/node.go

@@ -15,54 +15,10 @@
 package root
 
 import (
-	"context"
-	"strings"
-
-	"github.com/virtual-kubelet/virtual-kubelet/cmd/virtual-kubelet/internal/provider"
 	"github.com/virtual-kubelet/virtual-kubelet/errdefs"
 	corev1 "k8s.io/api/core/v1"
-	v1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-const osLabel = "beta.kubernetes.io/os"
-
-// NodeFromProvider builds a kubernetes node object from a provider
-// This is a temporary solution until node stuff actually split off from the provider interface itself.
-func NodeFromProvider(ctx context.Context, name string, taint *v1.Taint, p provider.Provider, version string) *v1.Node {
-	taints := make([]v1.Taint, 0)
-
-	if taint != nil {
-		taints = append(taints, *taint)
-	}
-
-	node := &v1.Node{
-		ObjectMeta: metav1.ObjectMeta{
-			Name: name,
-			Labels: map[string]string{
-				"type":                   "virtual-kubelet",
-				"kubernetes.io/role":     "agent",
-				"kubernetes.io/hostname": name,
-			},
-		},
-		Spec: v1.NodeSpec{
-			Taints: taints,
-		},
-		Status: v1.NodeStatus{
-			NodeInfo: v1.NodeSystemInfo{
-				Architecture:   "amd64",
-				KubeletVersion: version,
-			},
-		},
-	}
-
-	p.ConfigureNode(ctx, node)
-	if _, ok := node.ObjectMeta.Labels[osLabel]; !ok {
-		node.ObjectMeta.Labels[osLabel] = strings.ToLower(node.Status.NodeInfo.OperatingSystem)
-	}
-	return node
-}
-
 // getTaint creates a taint using the provided key/value.
 // Taint effect is read from the environment
 // The taint key/value may be overwritten by the environment.
@@ -80,7 +36,7 @@ func getTaint(c Opts) (*corev1.Taint, error) {
 
 	key = getEnv("VKUBELET_TAINT_KEY", key)
 	value = getEnv("VKUBELET_TAINT_VALUE", value)
-	effectEnv := getEnv("VKUBELET_TAINT_EFFECT", string(c.TaintEffect))
+	effectEnv := getEnv("VKUBELET_TAINT_EFFECT", c.TaintEffect)
 
 	var effect corev1.TaintEffect
 	switch effectEnv {

cmd/virtual-kubelet/internal/commands/root/opts.go

@@ -15,6 +15,7 @@
 package root
 
 import (
+	"fmt"
 	"os"
 	"path/filepath"
 	"strconv"
@@ -28,7 +29,7 @@ import (
 // Defaults for root command options
 const (
 	DefaultNodeName             = "virtual-kubelet"
-	DefaultOperatingSystem      = "Linux"
+	DefaultOperatingSystem      = "linux"
 	DefaultInformerResyncPeriod = 1 * time.Minute
 	DefaultMetricsAddr          = ":10255"
 	DefaultListenPort           = 10250 // TODO(cpuguy83)(VK1.0): Change this to an addr instead of just a port.. we should not be listening on all interfaces.
@@ -95,6 +96,8 @@ type Opts struct {
 	Version string
 }
 
+const maxInt32 = 1<<31 - 1
+
 // SetDefaultOpts sets default options for unset values on the passed in option struct.
 // Fields tht are already set will not be modified.
 func SetDefaultOpts(c *Opts) error {
@@ -128,6 +131,10 @@ func SetDefaultOpts(c *Opts) error {
 			if err != nil {
 				return errors.Wrap(err, "error parsing KUBELET_PORT environment variable")
 			}
+			if p > maxInt32 {
+				return fmt.Errorf("KUBELET_PORT environment variable is too large")
+			}
+			/* #nosec */
 			c.ListenPort = int32(p)
 		} else {
 			c.ListenPort = DefaultListenPort

cmd/virtual-kubelet/internal/commands/root/root.go

@@ -17,7 +17,7 @@ package root
 import (
 	"context"
 	"os"
-	"path"
+	"runtime"
 
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
@@ -28,12 +28,6 @@ import (
 	"github.com/virtual-kubelet/virtual-kubelet/node"
 	"github.com/virtual-kubelet/virtual-kubelet/node/nodeutil"
 	corev1 "k8s.io/api/core/v1"
-	k8serrors "k8s.io/apimachinery/pkg/api/errors"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	kubeinformers "k8s.io/client-go/informers"
-	"k8s.io/client-go/kubernetes/scheme"
-	corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
-	"k8s.io/client-go/tools/record"
 )
 
 // NewCommand creates a new top-level command.
@@ -75,33 +69,56 @@ func runRootCommand(ctx context.Context, s *provider.Store, c Opts) error {
 		}
 	}
 
-	client, err := nodeutil.ClientsetFromEnv(c.KubeConfigPath)
-	if err != nil {
-		return err
+	newProvider := func(cfg nodeutil.ProviderConfig) (nodeutil.Provider, node.NodeProvider, error) {
+		rm, err := manager.NewResourceManager(cfg.Pods, cfg.Secrets, cfg.ConfigMaps, cfg.Services)
+		if err != nil {
+			return nil, nil, errors.Wrap(err, "could not create resource manager")
+		}
+		initConfig := provider.InitConfig{
+			ConfigPath:        c.ProviderConfigPath,
+			NodeName:          c.NodeName,
+			OperatingSystem:   c.OperatingSystem,
+			ResourceManager:   rm,
+			DaemonPort:        c.ListenPort,
+			InternalIP:        os.Getenv("VKUBELET_POD_IP"),
+			KubeClusterDomain: c.KubeClusterDomain,
+		}
+		pInit := s.Get(c.Provider)
+		if pInit == nil {
+			return nil, nil, errors.Errorf("provider %q not found", c.Provider)
+		}
+
+		p, err := pInit(initConfig)
+		if err != nil {
+			return nil, nil, errors.Wrapf(err, "error initializing provider %s", c.Provider)
+		}
+		p.ConfigureNode(ctx, cfg.Node)
+		cfg.Node.Status.NodeInfo.KubeletVersion = c.Version
+		return p, nil, nil
 	}
 
-	// Create a shared informer factory for Kubernetes pods in the current namespace (if specified) and scheduled to the current node.
-	podInformerFactory := kubeinformers.NewSharedInformerFactoryWithOptions(
-		client,
-		c.InformerResyncPeriod,
-		kubeinformers.WithNamespace(c.KubeNamespace),
-		nodeutil.PodInformerFilter(c.NodeName),
+	apiConfig := getAPIConfig(c)
+	cm, err := nodeutil.NewNode(c.NodeName, newProvider, func(cfg *nodeutil.NodeConfig) error {
+		cfg.KubeconfigPath = c.KubeConfigPath
+		cfg.InformerResyncPeriod = c.InformerResyncPeriod
+
+		if taint != nil {
+			cfg.NodeSpec.Spec.Taints = append(cfg.NodeSpec.Spec.Taints, *taint)
+		}
+		cfg.NodeSpec.Status.NodeInfo.Architecture = runtime.GOARCH
+		cfg.NodeSpec.Status.NodeInfo.OperatingSystem = c.OperatingSystem
+
+		cfg.HTTPListenAddr = apiConfig.Addr
+		cfg.StreamCreationTimeout = apiConfig.StreamCreationTimeout
+		cfg.StreamIdleTimeout = apiConfig.StreamIdleTimeout
+		cfg.DebugHTTP = true
+
+		cfg.NumWorkers = c.PodSyncWorkers
+
+		return nil
+	},
+		nodeutil.WithBootstrapFromRestConfig(),
 	)
-	podInformer := podInformerFactory.Core().V1().Pods()
-
-	// Create another shared informer factory for Kubernetes secrets and configmaps (not subject to any selectors).
-	scmInformerFactory := kubeinformers.NewSharedInformerFactoryWithOptions(client, c.InformerResyncPeriod)
-	// Create a secret informer and a config map informer so we can pass their listers to the resource manager.
-	secretInformer := scmInformerFactory.Core().V1().Secrets()
-	configMapInformer := scmInformerFactory.Core().V1().ConfigMaps()
-	serviceInformer := scmInformerFactory.Core().V1().Services()
-
-	rm, err := manager.NewResourceManager(podInformer.Lister(), secretInformer.Lister(), configMapInformer.Lister(), serviceInformer.Lister())
-	if err != nil {
-		return errors.Wrap(err, "could not create resource manager")
-	}
-
-	apiConfig, err := getAPIConfig(c)
 	if err != nil {
 		return err
 	}
@@ -110,26 +127,6 @@ func runRootCommand(ctx context.Context, s *provider.Store, c Opts) error {
 		return err
 	}
 
-	initConfig := provider.InitConfig{
-		ConfigPath:        c.ProviderConfigPath,
-		NodeName:          c.NodeName,
-		OperatingSystem:   c.OperatingSystem,
-		ResourceManager:   rm,
-		DaemonPort:        c.ListenPort,
-		InternalIP:        os.Getenv("VKUBELET_POD_IP"),
-		KubeClusterDomain: c.KubeClusterDomain,
-	}
-
-	pInit := s.Get(c.Provider)
-	if pInit == nil {
-		return errors.Errorf("provider %q not found", c.Provider)
-	}
-
-	p, err := pInit(initConfig)
-	if err != nil {
-		return errors.Wrapf(err, "error initializing provider %s", c.Provider)
-	}
-
 	ctx = log.WithLogger(ctx, log.G(ctx).WithFields(log.Fields{
 		"provider":         c.Provider,
 		"operatingSystem":  c.OperatingSystem,
@@ -137,117 +134,25 @@ func runRootCommand(ctx context.Context, s *provider.Store, c Opts) error {
 		"watchedNamespace": c.KubeNamespace,
 	}))
 
-	pNode := NodeFromProvider(ctx, c.NodeName, taint, p, c.Version)
-	np := node.NewNaiveNodeProvider()
-	additionalOptions := []node.NodeControllerOpt{
-		node.WithNodeStatusUpdateErrorHandler(func(ctx context.Context, err error) error {
-			if !k8serrors.IsNotFound(err) {
-				return err
-			}
+	go cm.Run(ctx) //nolint:errcheck
 
-			log.G(ctx).Debug("node not found")
-			newNode := pNode.DeepCopy()
-			newNode.ResourceVersion = ""
-			_, err = client.CoreV1().Nodes().Create(ctx, newNode, metav1.CreateOptions{})
-			if err != nil {
-				return err
-			}
-			log.G(ctx).Debug("created new node")
-			return nil
-		}),
-	}
-	if c.EnableNodeLease {
-		leaseClient := nodeutil.NodeLeaseV1Client(client)
-		// 40 seconds is the default lease time in upstream kubelet
-		additionalOptions = append(additionalOptions, node.WithNodeEnableLeaseV1(leaseClient, 40))
-	}
-	nodeRunner, err := node.NewNodeController(
-		np,
-		pNode,
-		client.CoreV1().Nodes(),
-		additionalOptions...,
-	)
-	if err != nil {
-		log.G(ctx).Fatal(err)
-	}
+	defer func() {
+		log.G(ctx).Debug("Waiting for controllers to be done")
+		cancel()
+		<-cm.Done()
+	}()
 
-	eb := record.NewBroadcaster()
-	eb.StartLogging(log.G(ctx).Infof)
-	eb.StartRecordingToSink(&corev1client.EventSinkImpl{Interface: client.CoreV1().Events(c.KubeNamespace)})
-
-	pc, err := node.NewPodController(node.PodControllerConfig{
-		PodClient:         client.CoreV1(),
-		PodInformer:       podInformer,
-		EventRecorder:     eb.NewRecorder(scheme.Scheme, corev1.EventSource{Component: path.Join(pNode.Name, "pod-controller")}),
-		Provider:          p,
-		SecretInformer:    secretInformer,
-		ConfigMapInformer: configMapInformer,
-		ServiceInformer:   serviceInformer,
-	})
-	if err != nil {
-		return errors.Wrap(err, "error setting up pod controller")
-	}
-
-	go podInformerFactory.Start(ctx.Done())
-	go scmInformerFactory.Start(ctx.Done())
-
-	cancelHTTP, err := setupHTTPServer(ctx, p, apiConfig, func(context.Context) ([]*corev1.Pod, error) {
-		return rm.GetPods(), nil
-	})
-	if err != nil {
+	log.G(ctx).Info("Waiting for controller to be ready")
+	if err := cm.WaitReady(ctx, c.StartupTimeout); err != nil {
 		return err
 	}
-	defer cancelHTTP()
 
-	go func() {
-		if err := pc.Run(ctx, c.PodSyncWorkers); err != nil && errors.Cause(err) != context.Canceled {
-			log.G(ctx).Fatal(err)
-		}
-	}()
+	log.G(ctx).Info("Ready")
 
-	if c.StartupTimeout > 0 {
-		ctx, cancel := context.WithTimeout(ctx, c.StartupTimeout)
-		log.G(ctx).Info("Waiting for pod controller / VK to be ready")
-		select {
-		case <-ctx.Done():
-			cancel()
-			return ctx.Err()
-		case <-pc.Ready():
-		}
-		cancel()
-		if err := pc.Err(); err != nil {
-			return err
-		}
+	select {
+	case <-ctx.Done():
+	case <-cm.Done():
+		return cm.Err()
 	}
-
-	go func() {
-		if err := nodeRunner.Run(ctx); err != nil {
-			log.G(ctx).Fatal(err)
-		}
-	}()
-
-	setNodeReady(pNode)
-	if err := np.UpdateStatus(ctx, pNode); err != nil {
-		return errors.Wrap(err, "error marking the node as ready")
-	}
-	log.G(ctx).Info("Initialized")
-
-	<-ctx.Done()
 	return nil
 }
-
-func setNodeReady(n *corev1.Node) {
-	for i, c := range n.Status.Conditions {
-		if c.Type != "Ready" {
-			continue
-		}
-
-		c.Message = "Kubelet is ready"
-		c.Reason = "KubeletReady"
-		c.Status = corev1.ConditionTrue
-		c.LastHeartbeatTime = metav1.Now()
-		c.LastTransitionTime = metav1.Now()
-		n.Status.Conditions[i] = c
-		return
-	}
-}

cmd/virtual-kubelet/internal/commands/root/tracing.go

@@ -21,6 +21,7 @@ import (
 	"os"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/pkg/errors"
 	"github.com/virtual-kubelet/virtual-kubelet/errdefs"
@@ -105,7 +106,8 @@ func setupZpages(ctx context.Context) {
 	zpages.Handle(mux, "/debug")
 	go func() {
 		// This should never terminate, if it does, it will always terminate with an error
-		e := http.Serve(listener, mux)
+		srv := &http.Server{Handler: mux, ReadHeaderTimeout: 30 * time.Second}
+		e := srv.Serve(listener)
 		if e == http.ErrServerClosed {
 			return
 		}

cmd/virtual-kubelet/internal/commands/root/tracing_register.go

@@ -19,7 +19,8 @@ import (
 	"go.opencensus.io/trace"
 )
 
-type TracingExporterOptions struct { // nolint: golint
+// TracingExporterOptions is the options passed to the tracing exporter init function.
+type TracingExporterOptions struct { //nolint: golint
 	Tags        map[string]string
 	ServiceName string
 }

cmd/virtual-kubelet/internal/commands/root/tracing_register_jaeger.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build !no_jaeger_exporter
 // +build !no_jaeger_exporter
 
 package root
@@ -31,17 +32,17 @@ func init() {
 // NewJaegerExporter creates a new opencensus tracing exporter.
 func NewJaegerExporter(opts TracingExporterOptions) (trace.Exporter, error) {
 	jOpts := jaeger.Options{
-		Endpoint:      os.Getenv("JAEGER_ENDPOINT"),
-		AgentEndpoint: os.Getenv("JAEGER_AGENT_ENDPOINT"),
-		Username:      os.Getenv("JAEGER_USER"),
-		Password:      os.Getenv("JAEGER_PASSWORD"),
+		CollectorEndpoint: os.Getenv("JAEGER_COLLECTOR_ENDPOINT"),
+		AgentEndpoint:     os.Getenv("JAEGER_AGENT_ENDPOINT"),
+		Username:          os.Getenv("JAEGER_USER"),
+		Password:          os.Getenv("JAEGER_PASSWORD"),
 		Process: jaeger.Process{
 			ServiceName: opts.ServiceName,
 		},
 	}
 
-	if jOpts.Endpoint == "" && jOpts.AgentEndpoint == "" { // nolint:staticcheck
-		return nil, errors.New("Must specify either JAEGER_ENDPOINT or JAEGER_AGENT_ENDPOINT")
+	if jOpts.CollectorEndpoint == "" && jOpts.AgentEndpoint == "" { // nolintlint:staticcheck
+		return nil, errors.New("must specify either JAEGER_COLLECTOR_ENDPOINT or JAEGER_AGENT_ENDPOINT")
 	}
 
 	for k, v := range opts.Tags {

cmd/virtual-kubelet/internal/commands/root/tracing_register_ocagent.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build !no_ocagent_exporter
 // +build !no_ocagent_exporter
 
 package root

cmd/virtual-kubelet/internal/provider/mock/mock.go

@@ -5,11 +5,12 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"math/rand"
+	"os"
 	"strings"
 	"time"
 
+	dto "github.com/prometheus/client_model/go"
 	"github.com/virtual-kubelet/virtual-kubelet/errdefs"
 	"github.com/virtual-kubelet/virtual-kubelet/log"
 	"github.com/virtual-kubelet/virtual-kubelet/node/api"
@@ -17,7 +18,7 @@ import (
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	stats "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
+	stats "k8s.io/kubelet/pkg/apis/stats/v1alpha1"
 )
 
 const (
@@ -42,7 +43,7 @@ var (
 */
 
 // MockProvider implements the virtual-kubelet provider interface and stores pods in memory.
-type MockProvider struct { // nolint:golint
+type MockProvider struct {
 	nodeName           string
 	operatingSystem    string
 	internalIP         string
@@ -54,10 +55,12 @@ type MockProvider struct { // nolint:golint
 }
 
 // MockConfig contains a mock virtual-kubelet's configurable parameters.
-type MockConfig struct { // nolint:golint
-	CPU    string `json:"cpu,omitempty"`
-	Memory string `json:"memory,omitempty"`
-	Pods   string `json:"pods,omitempty"`
+type MockConfig struct {
+	CPU        string            `json:"cpu,omitempty"`
+	Memory     string            `json:"memory,omitempty"`
+	Pods       string            `json:"pods,omitempty"`
+	Others     map[string]string `json:"others,omitempty"`
+	ProviderID string            `json:"providerID,omitempty"`
 }
 
 // NewMockProviderMockConfig creates a new MockV0Provider. Mock legacy provider does not implement the new asynchronous podnotifier interface
@@ -97,9 +100,9 @@ func NewMockProvider(providerConfig, nodeName, operatingSystem string, internalI
 
 // loadConfig loads the given json configuration files.
 func loadConfig(providerConfig, nodeName string) (config MockConfig, err error) {
-	data, err := ioutil.ReadFile(providerConfig)
+	data, err := os.ReadFile(providerConfig)
 	if err != nil {
-		return config, err
+		return config, fmt.Errorf("error reaeding provider config: %w", err)
 	}
 	configMap := map[string]MockConfig{}
 	err = json.Unmarshal(data, &configMap)
@@ -120,13 +123,18 @@ func loadConfig(providerConfig, nodeName string) (config MockConfig, err error)
 	}
 
 	if _, err = resource.ParseQuantity(config.CPU); err != nil {
-		return config, fmt.Errorf("Invalid CPU value %v", config.CPU)
+		return config, fmt.Errorf("invalid CPU value %v", config.CPU)
 	}
 	if _, err = resource.ParseQuantity(config.Memory); err != nil {
-		return config, fmt.Errorf("Invalid memory value %v", config.Memory)
+		return config, fmt.Errorf("invalid memory value %v", config.Memory)
 	}
 	if _, err = resource.ParseQuantity(config.Pods); err != nil {
-		return config, fmt.Errorf("Invalid pods value %v", config.Pods)
+		return config, fmt.Errorf("invalid pods value %v", config.Pods)
+	}
+	for _, v := range config.Others {
+		if _, err = resource.ParseQuantity(v); err != nil {
+			return config, fmt.Errorf("invalid other value %v", v)
+		}
 	}
 	return config, nil
 }
@@ -283,7 +291,7 @@ func (p *MockProvider) GetContainerLogs(ctx context.Context, namespace, podName,
 	ctx = addAttributes(ctx, span, namespaceKey, namespace, nameKey, podName, containerNameKey, containerName)
 
 	log.G(ctx).Infof("receive GetContainerLogs %q", podName)
-	return ioutil.NopCloser(strings.NewReader("")), nil
+	return io.NopCloser(strings.NewReader("")), nil
 }
 
 // RunInContainer executes a command in a container in the pod, copying data
@@ -293,6 +301,19 @@ func (p *MockProvider) RunInContainer(ctx context.Context, namespace, name, cont
 	return nil
 }
 
+// AttachToContainer attaches to the executing process of a container in the pod, copying data
+// between in/out/err and the container's stdin/stdout/stderr.
+func (p *MockProvider) AttachToContainer(ctx context.Context, namespace, name, container string, attach api.AttachIO) error {
+	log.G(ctx).Infof("receive AttachToContainer %q", container)
+	return nil
+}
+
+// PortForward forwards a local port to a port on the pod
+func (p *MockProvider) PortForward(ctx context.Context, namespace, pod string, port int32, stream io.ReadWriteCloser) error {
+	log.G(ctx).Infof("receive PortForward %q", pod)
+	return nil
+}
+
 // GetPodStatus returns the status of a pod by name that is "running".
 // returns nil if a pod by that name is not found.
 func (p *MockProvider) GetPodStatus(ctx context.Context, namespace, name string) (*v1.PodStatus, error) {
@@ -328,10 +349,13 @@ func (p *MockProvider) GetPods(ctx context.Context) ([]*v1.Pod, error) {
 	return pods, nil
 }
 
-func (p *MockProvider) ConfigureNode(ctx context.Context, n *v1.Node) { // nolint:golint
-	ctx, span := trace.StartSpan(ctx, "mock.ConfigureNode") // nolint:staticcheck,ineffassign
+func (p *MockProvider) ConfigureNode(ctx context.Context, n *v1.Node) {
+	ctx, span := trace.StartSpan(ctx, "mock.ConfigureNode") //nolint:staticcheck,ineffassign
 	defer span.End()
 
+	if p.config.ProviderID != "" {
+		n.Spec.ProviderID = p.config.ProviderID
+	}
 	n.Status.Capacity = p.capacity()
 	n.Status.Allocatable = p.capacity()
 	n.Status.Conditions = p.nodeConditions()
@@ -339,21 +363,25 @@ func (p *MockProvider) ConfigureNode(ctx context.Context, n *v1.Node) { // nolin
 	n.Status.DaemonEndpoints = p.nodeDaemonEndpoints()
 	os := p.operatingSystem
 	if os == "" {
-		os = "Linux"
+		os = "linux"
 	}
 	n.Status.NodeInfo.OperatingSystem = os
 	n.Status.NodeInfo.Architecture = "amd64"
-	n.ObjectMeta.Labels["alpha.service-controller.kubernetes.io/exclude-balancer"] = "true"
-	n.ObjectMeta.Labels["node.kubernetes.io/exclude-from-external-load-balancers"] = "true"
+	n.Labels["alpha.service-controller.kubernetes.io/exclude-balancer"] = "true"
+	n.Labels["node.kubernetes.io/exclude-from-external-load-balancers"] = "true"
 }
 
 // Capacity returns a resource list containing the capacity limits.
 func (p *MockProvider) capacity() v1.ResourceList {
-	return v1.ResourceList{
+	rl := v1.ResourceList{
 		"cpu":    resource.MustParse(p.config.CPU),
 		"memory": resource.MustParse(p.config.Memory),
 		"pods":   resource.MustParse(p.config.Pods),
 	}
+	for k, v := range p.config.Others {
+		rl[v1.ResourceName(k)] = resource.MustParse(v)
+	}
+	return rl
 }
 
 // NodeConditions returns a list of conditions (Ready, OutOfDisk, etc), for updates to the node status
@@ -467,10 +495,14 @@ func (p *MockProvider) GetStatsSummary(ctx context.Context) (*stats.Summary, err
 		for _, container := range pod.Spec.Containers {
 			// Grab a dummy value to be used as the total CPU usage.
 			// The value should fit a uint32 in order to avoid overflows later on when computing pod stats.
+
+			/* #nosec */
 			dummyUsageNanoCores := uint64(rand.Uint32())
 			totalUsageNanoCores += dummyUsageNanoCores
 			// Create a dummy value to be used as the total RAM usage.
 			// The value should fit a uint32 in order to avoid overflows later on when computing pod stats.
+
+			/* #nosec */
 			dummyUsageBytes := uint64(rand.Uint32())
 			totalUsageBytes += dummyUsageBytes
 			// Append a ContainerStats object containing the dummy stats to the PodStats object.
@@ -504,6 +536,129 @@ func (p *MockProvider) GetStatsSummary(ctx context.Context) (*stats.Summary, err
 	return res, nil
 }
 
+func (p *MockProvider) generateMockMetrics(metricsMap map[string][]*dto.Metric, resourceType string, label []*dto.LabelPair) map[string][]*dto.Metric {
+	var (
+		cpuMetricSuffix    = "_cpu_usage_seconds_total"
+		memoryMetricSuffix = "_memory_working_set_bytes"
+		dummyValue         = float64(100)
+	)
+
+	if metricsMap == nil {
+		metricsMap = map[string][]*dto.Metric{}
+	}
+
+	finalCpuMetricName := resourceType + cpuMetricSuffix
+	finalMemoryMetricName := resourceType + memoryMetricSuffix
+
+	newCPUMetric := dto.Metric{
+		Label: label,
+		Counter: &dto.Counter{
+			Value: &dummyValue,
+		},
+	}
+	newMemoryMetric := dto.Metric{
+		Label: label,
+		Gauge: &dto.Gauge{
+			Value: &dummyValue,
+		},
+	}
+	// if metric family exists add to metric array
+	if cpuMetrics, ok := metricsMap[finalCpuMetricName]; ok {
+		metricsMap[finalCpuMetricName] = append(cpuMetrics, &newCPUMetric)
+	} else {
+		metricsMap[finalCpuMetricName] = []*dto.Metric{&newCPUMetric}
+	}
+	if memoryMetrics, ok := metricsMap[finalMemoryMetricName]; ok {
+		metricsMap[finalMemoryMetricName] = append(memoryMetrics, &newMemoryMetric)
+	} else {
+		metricsMap[finalMemoryMetricName] = []*dto.Metric{&newMemoryMetric}
+	}
+
+	return metricsMap
+}
+
+func (p *MockProvider) getMetricType(metricName string) *dto.MetricType {
+	var (
+		dtoCounterMetricType = dto.MetricType_COUNTER
+		dtoGaugeMetricType   = dto.MetricType_GAUGE
+		cpuMetricSuffix      = "_cpu_usage_seconds_total"
+		memoryMetricSuffix   = "_memory_working_set_bytes"
+	)
+	if strings.HasSuffix(metricName, cpuMetricSuffix) {
+		return &dtoCounterMetricType
+	}
+	if strings.HasSuffix(metricName, memoryMetricSuffix) {
+		return &dtoGaugeMetricType
+	}
+
+	return nil
+}
+
+func (p *MockProvider) GetMetricsResource(ctx context.Context) ([]*dto.MetricFamily, error) {
+	var span trace.Span
+	ctx, span = trace.StartSpan(ctx, "GetMetricsResource") //nolint: ineffassign,staticcheck
+	defer span.End()
+
+	var (
+		nodeNameStr      = "NodeName"
+		podNameStr       = "PodName"
+		containerNameStr = "containerName"
+	)
+	nodeLabels := []*dto.LabelPair{
+		{
+			Name:  &nodeNameStr,
+			Value: &p.nodeName,
+		},
+	}
+
+	metricsMap := p.generateMockMetrics(nil, "node", nodeLabels)
+	for _, pod := range p.pods {
+		podLabels := []*dto.LabelPair{
+			{
+				Name:  &nodeNameStr,
+				Value: &p.nodeName,
+			},
+			{
+				Name:  &podNameStr,
+				Value: &pod.Name,
+			},
+		}
+		metricsMap = p.generateMockMetrics(metricsMap, "pod", podLabels)
+		for _, container := range pod.Spec.Containers {
+			containerLabels := []*dto.LabelPair{
+				{
+					Name:  &nodeNameStr,
+					Value: &p.nodeName,
+				},
+				{
+					Name:  &podNameStr,
+					Value: &pod.Name,
+				},
+				{
+					Name:  &containerNameStr,
+					Value: &container.Name,
+				},
+			}
+			metricsMap = p.generateMockMetrics(metricsMap, "container", containerLabels)
+		}
+	}
+
+	res := []*dto.MetricFamily{}
+	for metricName := range metricsMap {
+		tempName := metricName
+		tempMetrics := metricsMap[tempName]
+
+		metricFamily := dto.MetricFamily{
+			Name:   &tempName,
+			Type:   p.getMetricType(tempName),
+			Metric: tempMetrics,
+		}
+		res = append(res, &metricFamily)
+	}
+
+	return res, nil
+}
+
 // NotifyPods is called to set a pod notifier callback function. This should be called before any operations are done
 // within the provider.
 func (p *MockProvider) NotifyPods(ctx context.Context, notifier func(*v1.Pod)) {
@@ -516,15 +671,15 @@ func buildKeyFromNames(namespace string, name string) (string, error) {
 
 // buildKey is a helper for building the "key" for the providers pod store.
 func buildKey(pod *v1.Pod) (string, error) {
-	if pod.ObjectMeta.Namespace == "" {
+	if pod.Namespace == "" {
 		return "", fmt.Errorf("pod namespace not found")
 	}
 
-	if pod.ObjectMeta.Name == "" {
+	if pod.Name == "" {
 		return "", fmt.Errorf("pod name not found")
 	}
 
-	return buildKeyFromNames(pod.ObjectMeta.Namespace, pod.ObjectMeta.Name)
+	return buildKeyFromNames(pod.Namespace, pod.Name)
 }
 
 // addAttributes adds the specified attributes to the provided span.

cmd/virtual-kubelet/internal/provider/provider.go

@@ -2,35 +2,15 @@ package provider
 
 import (
 	"context"
-	"io"
 
-	"github.com/virtual-kubelet/virtual-kubelet/node"
-	"github.com/virtual-kubelet/virtual-kubelet/node/api"
+	"github.com/virtual-kubelet/virtual-kubelet/node/nodeutil"
 	v1 "k8s.io/api/core/v1"
-	stats "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
 )
 
-// Provider contains the methods required to implement a virtual-kubelet provider.
-//
-// Errors produced by these methods should implement an interface from
-// github.com/virtual-kubelet/virtual-kubelet/errdefs package in order for the
-// core logic to be able to understand the type of failure.
+// Provider wraps the core provider type with an extra function needed to bootstrap the node
 type Provider interface {
-	node.PodLifecycleHandler
-
-	// GetContainerLogs retrieves the logs of a container by name from the provider.
-	GetContainerLogs(ctx context.Context, namespace, podName, containerName string, opts api.ContainerLogOpts) (io.ReadCloser, error)
-
-	// RunInContainer executes a command in a container in the pod, copying data
-	// between in/out/err and the container's stdin/stdout/stderr.
-	RunInContainer(ctx context.Context, namespace, podName, containerName string, cmd []string, attach api.AttachIO) error
-
+	nodeutil.Provider
 	// ConfigureNode enables a provider to configure the node object that
 	// will be used for Kubernetes.
 	ConfigureNode(context.Context, *v1.Node)
 }
-
-// PodMetricsProvider is an optional interface that providers can implement to expose pod stats
-type PodMetricsProvider interface {
-	GetStatsSummary(context.Context) (*stats.Summary, error)
-}

cmd/virtual-kubelet/internal/provider/store.go

@@ -13,7 +13,7 @@ type Store struct {
 	ls map[string]InitFunc
 }
 
-func NewStore() *Store { // nolint:golint
+func NewStore() *Store {
 	return &Store{
 		ls: make(map[string]InitFunc),
 	}
@@ -71,4 +71,4 @@ type InitConfig struct {
 	ResourceManager   *manager.ResourceManager
 }
 
-type InitFunc func(InitConfig) (Provider, error) // nolint:golint
+type InitFunc func(InitConfig) (Provider, error)

cmd/virtual-kubelet/internal/provider/types.go

@@ -2,12 +2,12 @@ package provider
 
 const (
 	// OperatingSystemLinux is the configuration value for defining Linux.
-	OperatingSystemLinux = "Linux"
+	OperatingSystemLinux = "linux"
 	// OperatingSystemWindows is the configuration value for defining Windows.
-	OperatingSystemWindows = "Windows"
+	OperatingSystemWindows = "windows"
 )
 
-type OperatingSystems map[string]bool // nolint:golint
+type OperatingSystems map[string]bool
 
 var (
 	// ValidOperatingSystems defines the group of operating systems
@@ -18,7 +18,7 @@ var (
 	}
 )
 
-func (o OperatingSystems) Names() []string { // nolint:golint
+func (o OperatingSystems) Names() []string {
 	keys := make([]string, 0, len(o))
 	for k := range o {
 		keys = append(keys, k)

cmd/virtual-kubelet/main.go

@@ -38,7 +38,7 @@ import (
 var (
 	buildVersion = "N/A"
 	buildTime    = "N/A"
-	k8sVersion   = "v1.15.2" // This should follow the version of k8s.io/kubernetes we are importing
+	k8sVersion   = "v1.31.4" // This should follow the version of k8s.io/client-go we are importing
 )
 
 func main() {

cmd/virtual-kubelet/register.go

@@ -6,6 +6,7 @@ import (
 )
 
 func registerMock(s *provider.Store) {
+	/* #nosec */
 	s.Register("mock", func(cfg provider.InitConfig) (provider.Provider, error) { //nolint:errcheck
 		return mock.NewMockProvider(
 			cfg.ConfigPath,

docs/proposals/MetricsUpdateProposal.md

@@ -0,0 +1,296 @@
+#  Virtual Kubelet Metrics Update
+
+<!-- toc -->
+- [Summary](#summary)
+- [Motivation](#motivation)
+  - [Goals](#goals)
+  - [Non-Goals](#non-goals)
+- [Proposal](#proposal)
+- [Design Details](#design-details)
+  - [API](#api)
+  - [Data](#data)
+  - [Changes to the Provider](#changes-to-the-provider)
+  - [Test Plan](#test-plan)
+<!-- /toc -->
+
+## Summary
+
+Add the new /metrics/resource endpoint in the virtual-kubelet to support the metrics server update for new Kubernetes versions `>=1.24`
+
+
+## Motivation
+
+The Kubernetes metrics server now tries to get metrics from the kubelet using the new metrics endpoint [/metrics/resource](https://github.com/kubernetes-sigs/metrics-server/commit/a2d732e5cdbfd93a6ebce221e8df0e8b463eecc6#diff-6e5b914d1403a14af1cc43582a2c9af727113037a3c6a77d8729aaefba084fb5R88),
+while Virtual Kubelet is still exposing the earlier metrics endpoint [/stats/summary](https://github.com/virtual-kubelet/virtual-kubelet/blob/master/node/api/server.go#L90). 
+This causes metrics to break when using virtual kubelet with newer Kubernetes versions (>=1.24). 
+To support the new metrics server, this document proposes adding a new handler to handle the updated metrics endpoint. 
+This will be an additive update, and the old 
+[/stats/summary](https://github.com/virtual-kubelet/virtual-kubelet/blob/master/node/api/server.go#L90) endpoint will still be available to maintain backward compatibility with 
+the older metrics server version.
+
+
+### Goals
+
+- Support metrics for kubernetes version `>=1.24` through adding /metrics/resource endpoint handler.
+
+### Non-Goals
+
+- Ensure pod autoscaling works as expected with the newer kubernetes versions `>=1.24` as expected
+
+## Proposal
+
+Add a new handler for `/metrics/resource` endpoint that calls a new `GetMetricsResource` method in the provider,
+which in-turn returns metrics using the prometheus `model.Samples` data structure as expected by the new metrics server.
+The provider will need to implement the `GetMetricsResource` method in order to add support for the new `/metrics/resource` endpoint with Kubernetes version >=1.24
+
+
+## Design Details
+Currently the virtual kubelet code uses the `PodStatsSummaryHandler` method to set up a http handler for serving pod metrics via the `/stats/summary` endpoint. 
+To support the updated metrics server, we need to add another handler `PodMetricsResourceHandler`  which can serve metrics via the `/metrics/resource` endpoint. 
+The `PodMetricsResourceHandler` calls the new `GetMetricsResource` method of the provider to get the metrics from the specific provider. 
+
+### API
+Add `GetMetricsResource` to `PodHandlerConfig`
+```go
+type PodHandlerConfig struct { //nolint:golint
+	RunInContainer   ContainerExecHandlerFunc
+	GetContainerLogs ContainerLogsHandlerFunc
+	// GetPods is meant to enumerate the pods that the provider knows about
+	GetPods PodListerFunc
+	// GetPodsFromKubernetes is meant to enumerate the pods that the node is meant to be running
+	GetPodsFromKubernetes PodListerFunc
+	GetStatsSummary       PodStatsSummaryHandlerFunc
+	GetMetricsResource    PodMetricsResourceHandlerFunc
+	StreamIdleTimeout     time.Duration
+	StreamCreationTimeout time.Duration
+}
+```
+Add endpoint to `PodHandler` method 
+```go
+const MetricsResourceRouteSuffix = "/metrics/resource"
+
+func PodHandler(p PodHandlerConfig, debug bool) http.Handler {
+	r := mux.NewRouter()
+
+	// This matches the behaviour in the reference kubelet
+	r.StrictSlash(true)
+	if debug {
+		r.HandleFunc("/runningpods/", HandleRunningPods(p.GetPods)).Methods("GET")
+	}
+
+	r.HandleFunc("/pods", HandleRunningPods(p.GetPodsFromKubernetes)).Methods("GET")
+	r.HandleFunc("/containerLogs/{namespace}/{pod}/{container}", HandleContainerLogs(p.GetContainerLogs)).Methods("GET")
+	r.HandleFunc(
+		"/exec/{namespace}/{pod}/{container}",
+		HandleContainerExec(
+			p.RunInContainer,
+			WithExecStreamCreationTimeout(p.StreamCreationTimeout),
+			WithExecStreamIdleTimeout(p.StreamIdleTimeout),
+		),
+	).Methods("POST", "GET")
+
+	if p.GetStatsSummary != nil {
+		f := HandlePodStatsSummary(p.GetStatsSummary)
+		r.HandleFunc("/stats/summary", f).Methods("GET")
+		r.HandleFunc("/stats/summary/", f).Methods("GET")
+	}
+
+	if p.GetMetricsResource != nil {
+		f := HandlePodMetricsResource(p.GetMetricsResource)
+		r.HandleFunc(MetricsResourceRouteSuffix, f).Methods("GET")
+		r.HandleFunc(MetricsResourceRouteSuffix+"/", f).Methods("GET")
+	}
+	r.NotFoundHandler = http.HandlerFunc(NotFound)
+	return r
+}
+```
+  
+New `PodMetricsResourceHandler` method, that uses the new `PodMetricsResourceHandlerFunc` definition.
+```go
+// PodMetricsResourceHandler creates an http handler for serving pod metrics.
+//
+// If the passed in handler func is nil this will create handlers which only
+// serves http.StatusNotImplemented
+func PodMetricsResourceHandler(f PodMetricsResourceHandlerFunc) http.Handler {
+	if f == nil {
+		return http.HandlerFunc(NotImplemented)
+	}
+
+	r := mux.NewRouter()
+
+	h := HandlePodMetricsResource(f)
+
+	r.Handle(MetricsResourceRouteSuffix, ochttp.WithRouteTag(h, "PodMetricsResourceHandler")).Methods("GET")
+	r.Handle(MetricsResourceRouteSuffix+"/", ochttp.WithRouteTag(h, "PodMetricsResourceHandler")).Methods("GET")
+
+	r.NotFoundHandler = http.HandlerFunc(NotFound)
+	return r
+}
+
+```
+ 
+ 
+`HandlePodMetricsResource` method returns a HandlerFunc which serves the metrics encoded in prometheus' text format encoding as expected by the  metrics-server
+```go
+// HandlePodMetricsResource makes an HTTP handler for implementing the kubelet /metrics/resource endpoint
+func HandlePodMetricsResource(h PodMetricsResourceHandlerFunc) http.HandlerFunc {
+	if h == nil {
+		return NotImplemented
+	}
+	return handleError(func(w http.ResponseWriter, req *http.Request) error {
+		metrics, err := h(req.Context())
+		if err != nil {
+			if isCancelled(err) {
+				return err
+			}
+			return errors.Wrap(err, "error getting status from provider")
+		}
+
+		b, err := json.Marshal(metrics)
+		if err != nil {
+			return errors.Wrap(err, "error marshalling metrics")
+		}
+
+		if _, err := w.Write(b); err != nil {
+			return errors.Wrap(err, "could not write to client")
+		}
+		return nil
+	})
+}
+```
+ 
+The `PodMetricsResourceHandlerFunc` returns the metrics data using Prometheus' `MetricFamily` data structure. More details are provided in the Data subsection
+```go
+// PodMetricsResourceHandlerFunc defines the handler for getting pod metrics
+type PodMetricsResourceHandlerFunc func(context.Context) ([]*dto.MetricFamily, error)
+```
+ 
+### Data
+
+The updated metrics server does not add any new fields to the metrics data but uses the Prometheus textparse series parser to parse and reconstruct the [MetricsBatch](https://github.com/kubernetes-sigs/metrics-server/blob/83b2e01f9825849ae5f562e47aa1a4178b5d06e5/pkg/storage/types.go#L31) data structure.  
+Currently virtual-kubelet is sending data to the server using the [summary](https://github.com/virtual-kubelet/virtual-kubelet/blob/be0a062aec9a5eeea3ad6fbe5aec557a235558f6/node/api/statsv1alpha1/types.go#L24) data structure. The Prometheus text parser expects a series of bytes as in the Prometheus [model.Samples](https://github.com/kubernetes/kubernetes/blob/a93eda9db305611cacd8b6ee930ab3149a08f9b0/vendor/github.com/prometheus/common/model/value.go#L184) data structure, similar to the test [here](https://github.com/prometheus/prometheus/blob/c70d85baed260f6013afd18d6cd0ffcac4339861/model/textparse/promparse_test.go#L31).
+ 
+Examples of how the new metrics are defined may be seen in the Kubernetes e2e test that calls the /metrics/resource endpoint [here](https://github.com/kubernetes/kubernetes/blob/a93eda9db305611cacd8b6ee930ab3149a08f9b0/test/e2e_node/resource_metrics_test.go#L76), and the kubelet metrics defined in the Kubernetes/kubelet code [here](https://github.com/kubernetes/kubernetes/blob/master/pkg/kubelet/metrics/collectors/resource_metrics.go) .
+ 
+```go
+var (
+	nodeCPUUsageDesc = metrics.NewDesc("node_cpu_usage_seconds_total",
+		"Cumulative cpu time consumed by the node in core-seconds",
+		nil,
+		nil,
+		metrics.ALPHA,
+		"")
+
+	nodeMemoryUsageDesc = metrics.NewDesc("node_memory_working_set_bytes",
+		"Current working set of the node in bytes",
+		nil,
+		nil,
+		metrics.ALPHA,
+		"")
+
+	containerCPUUsageDesc = metrics.NewDesc("container_cpu_usage_seconds_total",
+		"Cumulative cpu time consumed by the container in core-seconds",
+		[]string{"container", "pod", "namespace"},
+		nil,
+		metrics.ALPHA,
+		"")
+
+	containerMemoryUsageDesc = metrics.NewDesc("container_memory_working_set_bytes",
+		"Current working set of the container in bytes",
+		[]string{"container", "pod", "namespace"},
+		nil,
+		metrics.ALPHA,
+		"")
+
+	podCPUUsageDesc = metrics.NewDesc("pod_cpu_usage_seconds_total",
+		"Cumulative cpu time consumed by the pod in core-seconds",
+		[]string{"pod", "namespace"},
+		nil,
+		metrics.ALPHA,
+		"")
+
+	podMemoryUsageDesc = metrics.NewDesc("pod_memory_working_set_bytes",
+		"Current working set of the pod in bytes",
+		[]string{"pod", "namespace"},
+		nil,
+		metrics.ALPHA,
+		"")
+
+	resourceScrapeResultDesc = metrics.NewDesc("scrape_error",
+		"1 if there was an error while getting container metrics, 0 otherwise",
+		nil,
+		nil,
+		metrics.ALPHA,
+		"")
+
+	containerStartTimeDesc = metrics.NewDesc("container_start_time_seconds",
+		"Start time of the container since unix epoch in seconds",
+		[]string{"container", "pod", "namespace"},
+		nil,
+		metrics.ALPHA,
+		"")
+)
+```
+ 
+The kubernetes/kubelet code implements Prometheus' [collector](https://github.com/kubernetes/kubernetes/blob/master/pkg/kubelet/metrics/collectors/resource_metrics.go#L88) interface which is used along with the k8s.io/component-base implementation of the [registry](https://github.com/kubernetes/component-base/blob/40d14bdbd62f9e2ea697f97d81d4abc72839901e/metrics/registry.go#L114) interface in order to collect and return the metrics data using the Prometheus' [MetricFamily](https://github.com/prometheus/client_model/blob/master/go/metrics.pb.go#L773) data structure.
+ 
+The Gather method in the registry calls the kubelet collector's Collect method, and returns the data using the MetricFamily data structure. The metrics server expects metrics to be encoded in prometheus'
+text format, and the kubelet uses the http handler from prometheus' promhttp module which returns the metrics data encoded in prometheus' text format encoding.
+```go
+type KubeRegistry interface {
+	// Deprecated
+	RawMustRegister(...prometheus.Collector)
+	// CustomRegister is our internal variant of Prometheus registry.Register
+	CustomRegister(c StableCollector) error
+	// CustomMustRegister is our internal variant of Prometheus registry.MustRegister
+	CustomMustRegister(cs ...StableCollector)
+	// Register conforms to Prometheus registry.Register
+	Register(Registerable) error
+	// MustRegister conforms to Prometheus registry.MustRegister
+	MustRegister(...Registerable)
+	// Unregister conforms to Prometheus registry.Unregister
+	Unregister(collector Collector) bool
+	// Gather conforms to Prometheus gatherer.Gather
+	Gather() ([]*dto.MetricFamily, error)
+	// Reset invokes the Reset() function on all items in the registry
+	// which are added as resettables.
+	Reset()
+}
+```
+
+Prometheus’ MetricsFamily data structure:  
+```go
+type MetricFamily struct {
+	Name                 *string     `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"`
+	Help                 *string     `protobuf:"bytes,2,opt,name=help" json:"help,omitempty"`
+	Type                 *MetricType `protobuf:"varint,3,opt,name=type,enum=io.prometheus.client.MetricType" json:"type,omitempty"`
+	Metric               []*Metric   `protobuf:"bytes,4,rep,name=metric" json:"metric,omitempty"`
+	XXX_NoUnkeyedLiteral struct{}    `json:"-"`
+	XXX_unrecognized     []byte      `json:"-"`
+	XXX_sizecache        int32       `json:"-"`
+}
+```
+
+Therefore the provider's GetMetricsResource method should use the same return type as the Gather method in the registry interface.
+ 
+### Changes to the Provider.
+
+In order to support the new metrics endpoint the Provider must implement the GetMetricsResource method with definition
+
+```golang
+
+import (
+  dto "github.com/prometheus/client_model/go"
+  "context"
+)
+
+func GetMetricsResource(context.Context) ([]*dto.MetricsFamily, error) {
+...
+}
+```
+
+### Test Plan
+
+- Write a provider implementation for GetMetricsResource method in ACI Provider and deploy pods get metrics using kubectl 
+- Run end-to-end tests with the provider implementation
+

errdefs/auth.go

@@ -0,0 +1,33 @@
+package errdefs
+
+import (
+	"errors"
+	"fmt"
+)
+
+var (
+	// ErrForbidden is returned when the user is not authorized to perform the operation.
+	ErrForbidden = errors.New("forbidden")
+	// ErrUnauthorized is returned when the user is not authenticated.
+	ErrUnauthorized = errors.New("unauthorized")
+)
+
+// Unauthorized wraps ErrUnauthorized with a message.
+func Unauthorized(msg string) error {
+	return fmt.Errorf("%w: %s", ErrUnauthorized, msg)
+}
+
+// Forbidden wraps ErrForbidden with a message.
+func Forbidden(msg string) error {
+	return fmt.Errorf("%w: %s", ErrForbidden, msg)
+}
+
+// IsForbidden returns true if the error has ErrForbidden in the error chain.
+func IsForbidden(err error) bool {
+	return errors.Is(err, ErrForbidden)
+}
+
+// IsUnauthorized returns true if the error has ErrUnauthorized in the error chain.
+func IsUnauthorized(err error) bool {
+	return errors.Is(err, ErrUnauthorized)
+}

go.mod

@@ -1,77 +1,125 @@
 module github.com/virtual-kubelet/virtual-kubelet
 
-go 1.15
+go 1.23.0
+
+toolchain go1.23.4
 
 require (
-	contrib.go.opencensus.io/exporter/jaeger v0.1.0
-	contrib.go.opencensus.io/exporter/ocagent v0.4.12
-	github.com/bombsimon/logrusr v1.0.0
-	github.com/docker/spdystream v0.0.0-20170912183627-bc6354cbbc29 // indirect
-	github.com/elazarl/goproxy v0.0.0-20190421051319-9d40249d3c2f // indirect
-	github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2 // indirect
-	github.com/google/go-cmp v0.4.0
-	github.com/gorilla/mux v1.7.0
+	contrib.go.opencensus.io/exporter/jaeger v0.2.1
+	contrib.go.opencensus.io/exporter/ocagent v0.7.0
+	github.com/bombsimon/logrusr/v3 v3.1.0
+	github.com/google/go-cmp v0.7.0
+	github.com/gorilla/mux v1.8.1
 	github.com/mitchellh/go-homedir v1.1.0
-	github.com/pkg/errors v0.8.1
-	github.com/prometheus/client_golang v1.0.0
-	github.com/sirupsen/logrus v1.4.2
-	github.com/spf13/cobra v0.0.5
+	github.com/pkg/errors v0.9.1
+	github.com/prometheus/client_model v0.6.2
+	github.com/prometheus/common v0.64.0
+	github.com/sirupsen/logrus v1.9.3
+	github.com/spf13/cobra v1.8.1
 	github.com/spf13/pflag v1.0.5
-	go.opencensus.io v0.21.0
-	go.uber.org/goleak v1.1.10
-	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e
-	golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd
-	golang.org/x/time v0.0.0-20190308202827-9d24e82272b4
+	github.com/stretchr/testify v1.10.0
+	go.opencensus.io v0.24.0
+	go.opentelemetry.io/otel v1.36.0
+	go.opentelemetry.io/otel/sdk v1.36.0
+	go.opentelemetry.io/otel/trace v1.36.0
+	golang.org/x/sync v0.14.0
+	golang.org/x/time v0.9.0
+	google.golang.org/protobuf v1.36.6
 	gotest.tools v2.2.0+incompatible
-	k8s.io/api v0.18.6
-	k8s.io/apimachinery v0.18.6
-	k8s.io/apiserver v0.18.4
-	k8s.io/client-go v0.18.6
-	k8s.io/klog v1.0.0
-	k8s.io/klog/v2 v2.0.0
-	k8s.io/kubernetes v1.18.4
-	k8s.io/utils v0.0.0-20200603063816-c1c6865ac451
-	sigs.k8s.io/controller-runtime v0.6.3
+	k8s.io/api v0.31.4
+	k8s.io/apimachinery v0.31.4
+	k8s.io/apiserver v0.31.4
+	k8s.io/client-go v0.31.4
+	k8s.io/klog/v2 v2.130.1
+	k8s.io/kubelet v0.31.4
+	k8s.io/utils v0.0.0-20240711033017-18e509b52bc8
+	sigs.k8s.io/controller-runtime v0.19.4
 )
 
-replace k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.18.4
-
-replace k8s.io/cloud-provider => k8s.io/cloud-provider v0.18.4
-
-replace k8s.io/cli-runtime => k8s.io/cli-runtime v0.18.4
-
-replace k8s.io/apiserver => k8s.io/apiserver v0.18.4
-
-replace k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.18.4
-
-replace k8s.io/cri-api => k8s.io/cri-api v0.18.4
-
-replace k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.18.4
-
-replace k8s.io/kubelet => k8s.io/kubelet v0.18.4
-
-replace k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.18.4
-
-replace k8s.io/apimachinery => k8s.io/apimachinery v0.18.4
-
-replace k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.18.4
-
-replace k8s.io/kube-proxy => k8s.io/kube-proxy v0.18.4
-
-replace k8s.io/component-base => k8s.io/component-base v0.18.4
-
-replace k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.18.4
-
-replace k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.18.4
-
-replace k8s.io/metrics => k8s.io/metrics v0.18.4
-
-replace k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.18.4
-
-replace k8s.io/code-generator => k8s.io/code-generator v0.18.4
-
-replace k8s.io/client-go => k8s.io/client-go v0.18.4
-
-replace k8s.io/kubectl => k8s.io/kubectl v0.18.4
-
-replace k8s.io/api => k8s.io/api v0.18.4
+require (
+	github.com/NYTimes/gziphandler v1.1.1 // indirect
+	github.com/antlr4-go/antlr/v4 v4.13.0 // indirect
+	github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/blang/semver/v4 v4.0.0 // indirect
+	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
+	github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/coreos/go-semver v0.3.1 // indirect
+	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
+	github.com/evanphx/json-patch/v5 v5.9.0 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/fsnotify/fsnotify v1.7.0 // indirect
+	github.com/fxamacker/cbor/v2 v2.7.0 // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-openapi/jsonpointer v0.19.6 // indirect
+	github.com/go-openapi/jsonreference v0.20.2 // indirect
+	github.com/go-openapi/swag v0.22.4 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
+	github.com/google/cel-go v0.20.1 // indirect
+	github.com/google/gnostic-models v0.6.8 // indirect
+	github.com/google/gofuzz v1.2.0 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/gorilla/websocket v1.5.0 // indirect
+	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect
+	github.com/imdario/mergo v0.3.12 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/klauspost/compress v1.17.9 // indirect
+	github.com/kylelemons/godebug v1.1.0 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/moby/spdystream v0.4.0 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+	github.com/prometheus/client_golang v1.20.5 // indirect
+	github.com/prometheus/procfs v0.15.1 // indirect
+	github.com/stoewer/go-strcase v1.2.0 // indirect
+	github.com/uber/jaeger-client-go v2.25.0+incompatible // indirect
+	github.com/x448/float16 v0.8.4 // indirect
+	go.etcd.io/etcd/api/v3 v3.5.14 // indirect
+	go.etcd.io/etcd/client/pkg/v3 v3.5.14 // indirect
+	go.etcd.io/etcd/client/v3 v3.5.14 // indirect
+	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0 // indirect
+	go.opentelemetry.io/otel/metric v1.36.0 // indirect
+	go.opentelemetry.io/proto/otlp v1.3.1 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	go.uber.org/zap v1.26.0 // indirect
+	golang.org/x/crypto v0.38.0 // indirect
+	golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc // indirect
+	golang.org/x/net v0.40.0 // indirect
+	golang.org/x/oauth2 v0.30.0 // indirect
+	golang.org/x/sys v0.33.0 // indirect
+	golang.org/x/term v0.32.0 // indirect
+	golang.org/x/text v0.25.0 // indirect
+	google.golang.org/api v0.30.0 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect
+	google.golang.org/grpc v1.65.0 // indirect
+	gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	k8s.io/apiextensions-apiserver v0.31.0 // indirect
+	k8s.io/component-base v0.31.4 // indirect
+	k8s.io/kms v0.31.4 // indirect
+	k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
+	sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3 // indirect
+	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
+	sigs.k8s.io/yaml v1.4.0 // indirect
+)

hack/skaffold/virtual-kubelet/Dockerfile

@@ -1,8 +1,8 @@
 FROM gcr.io/distroless/base
 
-ENV APISERVER_CERT_LOCATION /vkubelet-mock-0-crt.pem
-ENV APISERVER_KEY_LOCATION /vkubelet-mock-0-key.pem
-ENV KUBELET_PORT 10250
+ENV APISERVER_CERT_LOCATION=/vkubelet-mock-0-crt.pem
+ENV APISERVER_KEY_LOCATION=/vkubelet-mock-0-key.pem
+ENV KUBELET_PORT=10250
 
 # Use the pre-built binary in "bin/virtual-kubelet".
 COPY bin/e2e/virtual-kubelet /virtual-kubelet

hack/skaffold/virtual-kubelet/base.yml

@@ -56,6 +56,14 @@ rules:
   verbs:
   - create
   - patch
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - get
+  - create
+  - update
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding

hack/skaffold/virtual-kubelet/pod.yml

@@ -4,6 +4,8 @@ metadata:
   name: vkubelet-mock-0
 spec:
   containers:
+  - name: jaeger-tracing
+    image: jaegertracing/all-in-one:1.22
   - name: vkubelet-mock-0
     image: virtual-kubelet
     # "IfNotPresent" is used to prevent Minikube from trying to pull from the registry (and failing) in the first place.
@@ -23,18 +25,16 @@ spec:
     - --klog.logtostderr
     - --log-level
     - debug
+    - --trace-exporter
+    - jaeger
+    - --trace-sample-rate=always
     env:
+    - name: JAEGER_AGENT_ENDPOINT
+      value: localhost:6831
     - name: KUBELET_PORT
       value: "10250"
     - name: VKUBELET_POD_IP
       valueFrom:
         fieldRef:
           fieldPath: status.podIP
-    ports:
-    - name: metrics
-      containerPort: 10255
-    readinessProbe:
-      httpGet:
-        path: /stats/summary
-        port: metrics
   serviceAccountName: virtual-kubelet

hack/skaffold/virtual-kubelet/skaffold.yml

@@ -1,18 +1,17 @@
-apiVersion: skaffold/v2beta10
+apiVersion: skaffold/v4beta11
 kind: Config
 build:
   artifacts:
-  - image: virtual-kubelet
-    docker:
-      # Use a Dockerfile specific for development only.
-      dockerfile: hack/skaffold/virtual-kubelet/Dockerfile
-deploy:
-  kubectl:
-    manifests:
+    - image: virtual-kubelet
+      docker:
+        dockerfile: hack/skaffold/virtual-kubelet/Dockerfile
+manifests:
+  rawYaml:
     - hack/skaffold/virtual-kubelet/base.yml
     - hack/skaffold/virtual-kubelet/pod.yml
+deploy:
+  kubectl: {}
 profiles:
-- name: local
-  build:
-    # For the "local" profile, we must perform the build locally.
-    local: {}
+  - name: local
+    build:
+      local: {}

internal/expansion/expand_test.go

@@ -2,24 +2,20 @@ package expansion
 
 import (
 	"testing"
-
-	api "k8s.io/kubernetes/pkg/apis/core"
 )
 
 func TestMapReference(t *testing.T) {
-	envs := []api.EnvVar{
-		{
-			Name:  "FOO",
-			Value: "bar",
-		},
-		{
-			Name:  "ZOO",
-			Value: "$(FOO)-1",
-		},
-		{
-			Name:  "BLU",
-			Value: "$(ZOO)-2",
-		},
+	// We use a struct here instead of a map because we need mappings to happen in order.
+	// Go maps are randomized.
+	type envVar struct {
+		Name  string
+		Value string
+	}
+
+	envs := []envVar{
+		{"FOO", "bar"},
+		{"ZOO", "$(FOO)-1"},
+		{"BLU", "$(ZOO)-2"},
 	}
 
 	declaredEnv := map[string]string{

internal/kubernetes/portforward/constants.go

@@ -0,0 +1,24 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package portforward contains server-side logic for handling port forwarding requests.
+package portforward
+
+// ProtocolV1Name is the name of the subprotocol used for port forwarding.
+const ProtocolV1Name = "portforward.k8s.io"
+
+// SupportedProtocols are the supported port forwarding protocols.
+var SupportedProtocols = []string{ProtocolV1Name}

internal/kubernetes/portforward/httpstream.go

@@ -0,0 +1,317 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package portforward
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/http"
+	"strconv"
+	"sync"
+	"time"
+
+	api "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/httpstream"
+	"k8s.io/apimachinery/pkg/util/httpstream/spdy"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+
+	"k8s.io/klog/v2"
+)
+
+func handleHTTPStreams(req *http.Request, w http.ResponseWriter, portForwarder PortForwarder, podName string, uid types.UID, supportedPortForwardProtocols []string, idleTimeout, streamCreationTimeout time.Duration) error {
+	_, err := httpstream.Handshake(req, w, supportedPortForwardProtocols)
+	// negotiated protocol isn't currently used server side, but could be in the future
+	if err != nil {
+		// Handshake writes the error to the client
+		return err
+	}
+	streamChan := make(chan httpstream.Stream, 1)
+
+	klog.V(5).InfoS("Upgrading port forward response")
+	
+	// TODO aka-somix: SPDY is deprecated and it should be replaced in order to support HTTP/2
+	upgrader := spdy.NewResponseUpgrader()
+	conn := upgrader.UpgradeResponse(w, req, httpStreamReceived(streamChan))
+	if conn == nil {
+		return errors.New("unable to upgrade httpstream connection")
+	}
+	defer conn.Close()
+
+	klog.V(5).InfoS("Connection setting port forwarding streaming connection idle timeout", "connection", conn, "idleTimeout", idleTimeout)
+	conn.SetIdleTimeout(idleTimeout)
+
+	h := &httpStreamHandler{
+		conn:                  conn,
+		streamChan:            streamChan,
+		streamPairs:           make(map[string]*httpStreamPair),
+		streamCreationTimeout: streamCreationTimeout,
+		pod:                   podName,
+		uid:                   uid,
+		forwarder:             portForwarder,
+	}
+	h.run()
+
+	return nil
+}
+
+// httpStreamReceived is the httpstream.NewStreamHandler for port
+// forward streams. It checks each stream's port and stream type headers,
+// rejecting any streams that with missing or invalid values. Each valid
+// stream is sent to the streams channel.
+func httpStreamReceived(streams chan httpstream.Stream) func(httpstream.Stream, <-chan struct{}) error {
+	return func(stream httpstream.Stream, replySent <-chan struct{}) error {
+		// make sure it has a valid port header
+		portString := stream.Headers().Get(api.PortHeader)
+		if len(portString) == 0 {
+			return fmt.Errorf("%q header is required", api.PortHeader)
+		}
+		port, err := strconv.ParseUint(portString, 10, 16)
+		if err != nil {
+			return fmt.Errorf("unable to parse %q as a port: %v", portString, err)
+		}
+		if port < 1 {
+			return fmt.Errorf("port %q must be > 0", portString)
+		}
+
+		// make sure it has a valid stream type header
+		streamType := stream.Headers().Get(api.StreamType)
+		if len(streamType) == 0 {
+			return fmt.Errorf("%q header is required", api.StreamType)
+		}
+		if streamType != api.StreamTypeError && streamType != api.StreamTypeData {
+			return fmt.Errorf("invalid stream type %q", streamType)
+		}
+
+		streams <- stream
+		return nil
+	}
+}
+
+// httpStreamHandler is capable of processing multiple port forward
+// requests over a single httpstream.Connection.
+type httpStreamHandler struct {
+	conn                  httpstream.Connection
+	streamChan            chan httpstream.Stream
+	streamPairsLock       sync.RWMutex
+	streamPairs           map[string]*httpStreamPair
+	streamCreationTimeout time.Duration
+	pod                   string
+	uid                   types.UID
+	forwarder             PortForwarder
+}
+
+// getStreamPair returns a httpStreamPair for requestID. This creates a
+// new pair if one does not yet exist for the requestID. The returned bool is
+// true if the pair was created.
+func (h *httpStreamHandler) getStreamPair(requestID string) (*httpStreamPair, bool) {
+	h.streamPairsLock.Lock()
+	defer h.streamPairsLock.Unlock()
+
+	if p, ok := h.streamPairs[requestID]; ok {
+		klog.V(5).InfoS("Connection request found existing stream pair", "connection", h.conn, "request", requestID)
+		return p, false
+	}
+
+	klog.V(5).InfoS("Connection request creating new stream pair", "connection", h.conn, "request", requestID)
+
+	p := newPortForwardPair(requestID)
+	h.streamPairs[requestID] = p
+
+	return p, true
+}
+
+// monitorStreamPair waits for the pair to receive both its error and data
+// streams, or for the timeout to expire (whichever happens first), and then
+// removes the pair.
+func (h *httpStreamHandler) monitorStreamPair(p *httpStreamPair, timeout <-chan time.Time) {
+	select {
+	case <-timeout:
+		err := fmt.Errorf("(conn=%v, request=%s) timed out waiting for streams", h.conn, p.requestID)
+		utilruntime.HandleError(err)
+		p.printError(err.Error())
+	case <-p.complete:
+		klog.V(5).InfoS("Connection request successfully received error and data streams", "connection", h.conn, "request", p.requestID)
+	}
+	h.removeStreamPair(p.requestID)
+}
+
+// hasStreamPair returns a bool indicating if a stream pair for requestID
+// exists.
+func (h *httpStreamHandler) hasStreamPair(requestID string) bool {
+	h.streamPairsLock.RLock()
+	defer h.streamPairsLock.RUnlock()
+
+	_, ok := h.streamPairs[requestID]
+	return ok
+}
+
+// removeStreamPair removes the stream pair identified by requestID from streamPairs.
+func (h *httpStreamHandler) removeStreamPair(requestID string) {
+	h.streamPairsLock.Lock()
+	defer h.streamPairsLock.Unlock()
+
+	if h.conn != nil {
+		pair := h.streamPairs[requestID]
+		h.conn.RemoveStreams(pair.dataStream, pair.errorStream)
+	}
+	delete(h.streamPairs, requestID)
+}
+
+// requestID returns the request id for stream.
+func (h *httpStreamHandler) requestID(stream httpstream.Stream) string {
+	requestID := stream.Headers().Get(api.PortForwardRequestIDHeader)
+	if len(requestID) == 0 {
+		klog.V(5).InfoS("Connection stream received without requestID header", "connection", h.conn)
+		// If we get here, it's because the connection came from an older client
+		// that isn't generating the request id header
+		// (https://github.com/kubernetes/kubernetes/blob/843134885e7e0b360eb5441e85b1410a8b1a7a0c/pkg/client/unversioned/portforward/portforward.go#L258-L287)
+		//
+		// This is a best-effort attempt at supporting older clients.
+		//
+		// When there aren't concurrent new forwarded connections, each connection
+		// will have a pair of streams (data, error), and the stream IDs will be
+		// consecutive odd numbers, e.g. 1 and 3 for the first connection. Convert
+		// the stream ID into a pseudo-request id by taking the stream type and
+		// using id = stream.Identifier() when the stream type is error,
+		// and id = stream.Identifier() - 2 when it's data.
+		//
+		// NOTE: this only works when there are not concurrent new streams from
+		// multiple forwarded connections; it's a best-effort attempt at supporting
+		// old clients that don't generate request ids.  If there are concurrent
+		// new connections, it's possible that 1 connection gets streams whose IDs
+		// are not consecutive (e.g. 5 and 9 instead of 5 and 7).
+		streamType := stream.Headers().Get(api.StreamType)
+		switch streamType {
+		case api.StreamTypeError:
+			requestID = strconv.Itoa(int(stream.Identifier()))
+		case api.StreamTypeData:
+			requestID = strconv.Itoa(int(stream.Identifier()) - 2)
+		}
+
+		klog.V(5).InfoS("Connection automatically assigning request ID from stream type and stream ID", "connection", h.conn, "request", requestID, "streamType", streamType, "stream", stream.Identifier())
+	}
+	return requestID
+}
+
+// run is the main loop for the httpStreamHandler. It processes new
+// streams, invoking portForward for each complete stream pair. The loop exits
+// when the httpstream.Connection is closed.
+func (h *httpStreamHandler) run() {
+	klog.V(5).InfoS("Connection waiting for port forward streams", "connection", h.conn)
+Loop:
+	for {
+		select {
+		case <-h.conn.CloseChan():
+			klog.V(5).InfoS("Connection upgraded connection closed", "connection", h.conn)
+			break Loop
+		case stream := <-h.streamChan:
+			requestID := h.requestID(stream)
+			streamType := stream.Headers().Get(api.StreamType)
+			klog.V(5).InfoS("Connection request received new type of stream", "connection", h.conn, "request", requestID, "streamType", streamType)
+
+			p, created := h.getStreamPair(requestID)
+			if created {
+				go h.monitorStreamPair(p, time.After(h.streamCreationTimeout))
+			}
+			if complete, err := p.add(stream); err != nil {
+				msg := fmt.Sprintf("error processing stream for request %s: %v", requestID, err)
+				utilruntime.HandleError(errors.New(msg))
+				p.printError(msg)
+			} else if complete {
+				go h.portForward(p)
+			}
+		}
+	}
+}
+
+// portForward invokes the httpStreamHandler's forwarder.PortForward
+// function for the given stream pair.
+func (h *httpStreamHandler) portForward(p *httpStreamPair) {
+	ctx := context.Background()
+	defer p.dataStream.Close()
+	defer p.errorStream.Close()
+
+	portString := p.dataStream.Headers().Get(api.PortHeader)
+	port, _ := strconv.ParseInt(portString, 10, 32)
+
+	klog.V(5).InfoS("Connection request invoking forwarder.PortForward for port", "connection", h.conn, "request", p.requestID, "port", portString)
+	err := h.forwarder.PortForward(ctx, h.pod, h.uid, int32(port), p.dataStream)
+	klog.V(5).InfoS("Connection request done invoking forwarder.PortForward for port", "connection", h.conn, "request", p.requestID, "port", portString)
+
+	if err != nil {
+		msg := fmt.Errorf("error forwarding port %d to pod %s, uid %v: %v", port, h.pod, h.uid, err)
+		utilruntime.HandleError(msg)
+		fmt.Fprint(p.errorStream, msg.Error())
+	}
+}
+
+// httpStreamPair represents the error and data streams for a port
+// forwarding request.
+type httpStreamPair struct {
+	lock        sync.RWMutex
+	requestID   string
+	dataStream  httpstream.Stream
+	errorStream httpstream.Stream
+	complete    chan struct{}
+}
+
+// newPortForwardPair creates a new httpStreamPair.
+func newPortForwardPair(requestID string) *httpStreamPair {
+	return &httpStreamPair{
+		requestID: requestID,
+		complete:  make(chan struct{}),
+	}
+}
+
+// add adds the stream to the httpStreamPair. If the pair already
+// contains a stream for the new stream's type, an error is returned. add
+// returns true if both the data and error streams for this pair have been
+// received.
+func (p *httpStreamPair) add(stream httpstream.Stream) (bool, error) {
+	p.lock.Lock()
+	defer p.lock.Unlock()
+
+	switch stream.Headers().Get(api.StreamType) {
+	case api.StreamTypeError:
+		if p.errorStream != nil {
+			return false, errors.New("error stream already assigned")
+		}
+		p.errorStream = stream
+	case api.StreamTypeData:
+		if p.dataStream != nil {
+			return false, errors.New("data stream already assigned")
+		}
+		p.dataStream = stream
+	}
+
+	complete := p.errorStream != nil && p.dataStream != nil
+	if complete {
+		close(p.complete)
+	}
+	return complete, nil
+}
+
+// printError writes s to p.errorStream if p.errorStream has been set.
+func (p *httpStreamPair) printError(s string) {
+	p.lock.RLock()
+	defer p.lock.RUnlock()
+	if p.errorStream != nil {
+		fmt.Fprint(p.errorStream, s)
+	}
+}

internal/kubernetes/portforward/httpstream_test.go

@@ -0,0 +1,267 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package portforward
+
+import (
+	"net/http"
+	"testing"
+	"time"
+
+	api "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/util/httpstream"
+)
+
+func TestHTTPStreamReceived(t *testing.T) {
+	tests := map[string]struct {
+		port          string
+		streamType    string
+		expectedError string
+	}{
+		"missing port": {
+			expectedError: `"port" header is required`,
+		},
+		"unable to parse port": {
+			port:          "abc",
+			expectedError: `unable to parse "abc" as a port: strconv.ParseUint: parsing "abc": invalid syntax`,
+		},
+		"negative port": {
+			port:          "-1",
+			expectedError: `unable to parse "-1" as a port: strconv.ParseUint: parsing "-1": invalid syntax`,
+		},
+		"missing stream type": {
+			port:          "80",
+			expectedError: `"streamType" header is required`,
+		},
+		"valid port with error stream": {
+			port:       "80",
+			streamType: "error",
+		},
+		"valid port with data stream": {
+			port:       "80",
+			streamType: "data",
+		},
+		"invalid stream type": {
+			port:          "80",
+			streamType:    "foo",
+			expectedError: `invalid stream type "foo"`,
+		},
+	}
+	for name, test := range tests {
+		streams := make(chan httpstream.Stream, 1)
+		f := httpStreamReceived(streams)
+		stream := newFakeHTTPStream()
+		if len(test.port) > 0 {
+			stream.headers.Set("port", test.port)
+		}
+		if len(test.streamType) > 0 {
+			stream.headers.Set("streamType", test.streamType)
+		}
+		replySent := make(chan struct{})
+		err := f(stream, replySent)
+		close(replySent)
+		if len(test.expectedError) > 0 {
+			if err == nil {
+				t.Errorf("%s: expected err=%q, but it was nil", name, test.expectedError)
+			}
+			if e, a := test.expectedError, err.Error(); e != a {
+				t.Errorf("%s: expected err=%q, got %q", name, e, a)
+			}
+			continue
+		}
+		if err != nil {
+			t.Errorf("%s: unexpected error %v", name, err)
+			continue
+		}
+		if s := <-streams; s != stream {
+			t.Errorf("%s: expected stream %#v, got %#v", name, stream, s)
+		}
+	}
+}
+
+type fakeConn struct {
+	removeStreamsCalled bool
+}
+
+func (*fakeConn) CreateStream(headers http.Header) (httpstream.Stream, error) { return nil, nil }
+func (*fakeConn) Close() error                                                { return nil }
+func (*fakeConn) CloseChan() <-chan bool                                      { return nil }
+func (*fakeConn) SetIdleTimeout(timeout time.Duration)                        {}
+func (f *fakeConn) RemoveStreams(streams ...httpstream.Stream)                { f.removeStreamsCalled = true }
+
+func TestGetStreamPair(t *testing.T) {
+	timeout := make(chan time.Time)
+
+	conn := &fakeConn{}
+	h := &httpStreamHandler{
+		streamPairs: make(map[string]*httpStreamPair),
+		conn:        conn,
+	}
+
+	// test adding a new entry
+	p, created := h.getStreamPair("1")
+	if p == nil {
+		t.Fatalf("unexpected nil pair")
+	}
+	if !created {
+		t.Fatal("expected created=true")
+	}
+	if p.dataStream != nil {
+		t.Errorf("unexpected non-nil data stream")
+	}
+	if p.errorStream != nil {
+		t.Errorf("unexpected non-nil error stream")
+	}
+
+	// start the monitor for this pair
+	monitorDone := make(chan struct{})
+	go func() {
+		h.monitorStreamPair(p, timeout)
+		close(monitorDone)
+	}()
+
+	if !h.hasStreamPair("1") {
+		t.Fatal("This should still be true")
+	}
+
+	// make sure we can retrieve an existing entry
+	p2, created := h.getStreamPair("1")
+	if created {
+		t.Fatal("expected created=false")
+	}
+	if p != p2 {
+		t.Fatalf("retrieving an existing pair: expected %#v, got %#v", p, p2)
+	}
+
+	// removed via complete
+	dataStream := newFakeHTTPStream()
+	dataStream.headers.Set(api.StreamType, api.StreamTypeData)
+	complete, err := p.add(dataStream)
+	if err != nil {
+		t.Fatalf("unexpected error adding data stream to pair: %v", err)
+	}
+	if complete {
+		t.Fatalf("unexpected complete")
+	}
+
+	errorStream := newFakeHTTPStream()
+	errorStream.headers.Set(api.StreamType, api.StreamTypeError)
+	complete, err = p.add(errorStream)
+	if err != nil {
+		t.Fatalf("unexpected error adding error stream to pair: %v", err)
+	}
+	if !complete {
+		t.Fatal("unexpected incomplete")
+	}
+
+	// make sure monitorStreamPair completed
+	<-monitorDone
+
+	if !conn.removeStreamsCalled {
+		t.Fatalf("connection remove stream not called")
+	}
+	conn.removeStreamsCalled = false
+
+	// make sure the pair was removed
+	if h.hasStreamPair("1") {
+		t.Fatal("expected removal of pair after both data and error streams received")
+	}
+
+	// removed via timeout
+	p, created = h.getStreamPair("2")
+	if !created {
+		t.Fatal("expected created=true")
+	}
+	if p == nil {
+		t.Fatal("expected p not to be nil")
+	}
+
+	monitorDone = make(chan struct{})
+	go func() {
+		h.monitorStreamPair(p, timeout)
+		close(monitorDone)
+	}()
+	// cause the timeout
+	close(timeout)
+	// make sure monitorStreamPair completed
+	<-monitorDone
+	if h.hasStreamPair("2") {
+		t.Fatal("expected stream pair to be removed")
+	}
+	if !conn.removeStreamsCalled {
+		t.Fatalf("connection remove stream not called")
+	}
+}
+
+func TestRequestID(t *testing.T) {
+	h := &httpStreamHandler{}
+
+	s := newFakeHTTPStream()
+	s.headers.Set(api.StreamType, api.StreamTypeError)
+	s.id = 1
+	if e, a := "1", h.requestID(s); e != a {
+		t.Errorf("expected %q, got %q", e, a)
+	}
+
+	s.headers.Set(api.StreamType, api.StreamTypeData)
+	s.id = 3
+	if e, a := "1", h.requestID(s); e != a {
+		t.Errorf("expected %q, got %q", e, a)
+	}
+
+	s.id = 7
+	s.headers.Set(api.PortForwardRequestIDHeader, "2")
+	if e, a := "2", h.requestID(s); e != a {
+		t.Errorf("expected %q, got %q", e, a)
+	}
+}
+
+type fakeHTTPStream struct {
+	headers http.Header
+	id      uint32
+}
+
+func newFakeHTTPStream() *fakeHTTPStream {
+	return &fakeHTTPStream{
+		headers: make(http.Header),
+	}
+}
+
+var _ httpstream.Stream = &fakeHTTPStream{}
+
+func (s *fakeHTTPStream) Read(data []byte) (int, error) {
+	return 0, nil
+}
+
+func (s *fakeHTTPStream) Write(data []byte) (int, error) {
+	return 0, nil
+}
+
+func (s *fakeHTTPStream) Close() error {
+	return nil
+}
+
+func (s *fakeHTTPStream) Reset() error {
+	return nil
+}
+
+func (s *fakeHTTPStream) Headers() http.Header {
+	return s.headers
+}
+
+func (s *fakeHTTPStream) Identifier() uint32 {
+	return s.id
+}

internal/kubernetes/portforward/portforward.go

@@ -0,0 +1,54 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package portforward
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"time"
+
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apiserver/pkg/util/wsstream"
+)
+
+// PortForwarder knows how to forward content from a data stream to/from a port
+// in a pod.
+type PortForwarder interface {
+	// PortForwarder copies data between a data stream and a port in a pod.
+	PortForward(ctx context.Context, name string, uid types.UID, port int32, stream io.ReadWriteCloser) error
+}
+
+// ServePortForward handles a port forwarding request.  A single request is
+// kept alive as long as the client is still alive and the connection has not
+// been timed out due to idleness. This function handles multiple forwarded
+// connections; i.e., multiple `curl http://localhost:8888/` requests will be
+// handled by a single invocation of ServePortForward.
+func ServePortForward(w http.ResponseWriter, req *http.Request, portForwarder PortForwarder, podName string, uid types.UID, portForwardOptions *V4Options, idleTimeout time.Duration, streamCreationTimeout time.Duration, supportedProtocols []string) {
+	var err error
+	if wsstream.IsWebSocketRequest(req) {
+		err = handleWebSocketStreams(req, w, portForwarder, podName, uid, portForwardOptions, supportedProtocols, idleTimeout, streamCreationTimeout)
+	} else {
+		err = handleHTTPStreams(req, w, portForwarder, podName, uid, supportedProtocols, idleTimeout, streamCreationTimeout)
+	}
+
+	if err != nil {
+		runtime.HandleError(err)
+		return
+	}
+}

internal/kubernetes/portforward/websocket.go

@@ -0,0 +1,199 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package portforward
+
+import (
+	"context"
+	"encoding/binary"
+	"fmt"
+	"io"
+	"net/http"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"k8s.io/klog/v2"
+
+	api "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apiserver/pkg/endpoints/responsewriter"
+	"k8s.io/apiserver/pkg/util/wsstream"
+)
+
+const (
+	dataChannel = iota
+	errorChannel
+
+	v4BinaryWebsocketProtocol = "v4." + wsstream.ChannelWebSocketProtocol
+	v4Base64WebsocketProtocol = "v4." + wsstream.Base64ChannelWebSocketProtocol
+)
+
+// V4Options contains details about which streams are required for port
+// forwarding.
+// All fields included in V4Options need to be expressed explicitly in the
+// CRI (k8s.io/cri-api/pkg/apis/{version}/api.proto) PortForwardRequest.
+type V4Options struct {
+	Ports []int32
+}
+
+// NewV4Options creates a new options from the Request.
+func NewV4Options(req *http.Request) (*V4Options, error) {
+	if !wsstream.IsWebSocketRequest(req) {
+		return &V4Options{}, nil
+	}
+
+	portStrings := req.URL.Query()[api.PortHeader]
+	if len(portStrings) == 0 {
+		return nil, fmt.Errorf("query parameter %q is required", api.PortHeader)
+	}
+
+	ports := make([]int32, 0, len(portStrings))
+	for _, portString := range portStrings {
+		if len(portString) == 0 {
+			return nil, fmt.Errorf("query parameter %q cannot be empty", api.PortHeader)
+		}
+		for _, p := range strings.Split(portString, ",") {
+			port, err := strconv.ParseUint(p, 10, 16)
+			if err != nil {
+				return nil, fmt.Errorf("unable to parse %q as a port: %v", portString, err)
+			}
+			if port < 1 {
+				return nil, fmt.Errorf("port %q must be > 0", portString)
+			}
+			ports = append(ports, int32(port))
+		}
+	}
+
+	return &V4Options{
+		Ports: ports,
+	}, nil
+}
+
+// BuildV4Options returns a V4Options based on the given information.
+func BuildV4Options(ports []int32) (*V4Options, error) {
+	return &V4Options{Ports: ports}, nil
+}
+
+// handleWebSocketStreams handles requests to forward ports to a pod via
+// a PortForwarder. A pair of streams are created per port (DATA n,
+// ERROR n+1). The associated port is written to each stream as a unsigned 16
+// bit integer in little endian format.
+func handleWebSocketStreams(req *http.Request, w http.ResponseWriter, portForwarder PortForwarder, podName string, uid types.UID, opts *V4Options, supportedPortForwardProtocols []string, idleTimeout, streamCreationTimeout time.Duration) error {
+	channels := make([]wsstream.ChannelType, 0, len(opts.Ports)*2)
+	for i := 0; i < len(opts.Ports); i++ {
+		channels = append(channels, wsstream.ReadWriteChannel, wsstream.WriteChannel)
+	}
+	conn := wsstream.NewConn(map[string]wsstream.ChannelProtocolConfig{
+		"": {
+			Binary:   true,
+			Channels: channels,
+		},
+		v4BinaryWebsocketProtocol: {
+			Binary:   true,
+			Channels: channels,
+		},
+		v4Base64WebsocketProtocol: {
+			Binary:   false,
+			Channels: channels,
+		},
+	})
+	conn.SetIdleTimeout(idleTimeout)
+	_, streams, err := conn.Open(responsewriter.GetOriginal(w), req)
+	if err != nil {
+		err = fmt.Errorf("unable to upgrade websocket connection: %v", err)
+		return err
+	}
+	defer conn.Close()
+	streamPairs := make([]*websocketStreamPair, len(opts.Ports))
+	for i := range streamPairs {
+		streamPair := websocketStreamPair{
+			port:        opts.Ports[i],
+			dataStream:  streams[i*2+dataChannel],
+			errorStream: streams[i*2+errorChannel],
+		}
+		streamPairs[i] = &streamPair
+
+		portBytes := make([]byte, 2)
+		// port is always positive so conversion is allowable
+		binary.LittleEndian.PutUint16(portBytes, uint16(streamPair.port))
+		streamPair.dataStream.Write(portBytes)
+		streamPair.errorStream.Write(portBytes)
+	}
+	h := &websocketStreamHandler{
+		conn:        conn,
+		streamPairs: streamPairs,
+		pod:         podName,
+		uid:         uid,
+		forwarder:   portForwarder,
+	}
+	h.run()
+
+	return nil
+}
+
+// websocketStreamPair represents the error and data streams for a port
+// forwarding request.
+type websocketStreamPair struct {
+	port        int32
+	dataStream  io.ReadWriteCloser
+	errorStream io.WriteCloser
+}
+
+// websocketStreamHandler is capable of processing a single port forward
+// request over a websocket connection
+type websocketStreamHandler struct {
+	conn        *wsstream.Conn
+	streamPairs []*websocketStreamPair
+	pod         string
+	uid         types.UID
+	forwarder   PortForwarder
+}
+
+// run invokes the websocketStreamHandler's forwarder.PortForward
+// function for the given stream pair.
+func (h *websocketStreamHandler) run() {
+	wg := sync.WaitGroup{}
+	wg.Add(len(h.streamPairs))
+
+	for _, pair := range h.streamPairs {
+		p := pair
+		go func() {
+			defer wg.Done()
+			h.portForward(p)
+		}()
+	}
+
+	wg.Wait()
+}
+
+func (h *websocketStreamHandler) portForward(p *websocketStreamPair) {
+	ctx := context.Background()
+	defer p.dataStream.Close()
+	defer p.errorStream.Close()
+
+	klog.V(5).InfoS("Connection invoking forwarder.PortForward for port", "connection", h.conn, "port", p.port)
+	err := h.forwarder.PortForward(ctx, h.pod, h.uid, p.port, p.dataStream)
+	klog.V(5).InfoS("Connection done invoking forwarder.PortForward for port", "connection", h.conn, "port", p.port)
+
+	if err != nil {
+		msg := fmt.Errorf("error forwarding port %d to pod %s, uid %v: %v", p.port, h.pod, h.uid, err)
+		runtime.HandleError(msg)
+		fmt.Fprint(p.errorStream, msg.Error())
+	}
+}

internal/kubernetes/portforward/websocket_test.go

@@ -0,0 +1,101 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package portforward
+
+import (
+	"net/http"
+	"reflect"
+	"testing"
+)
+
+func TestV4Options(t *testing.T) {
+	tests := map[string]struct {
+		url           string
+		websocket     bool
+		expectedOpts  *V4Options
+		expectedError string
+	}{
+		"non-ws request": {
+			url:          "http://example.com",
+			expectedOpts: &V4Options{},
+		},
+		"missing port": {
+			url:           "http://example.com",
+			websocket:     true,
+			expectedError: `query parameter "port" is required`,
+		},
+		"unable to parse port": {
+			url:           "http://example.com?port=abc",
+			websocket:     true,
+			expectedError: `unable to parse "abc" as a port: strconv.ParseUint: parsing "abc": invalid syntax`,
+		},
+		"negative port": {
+			url:           "http://example.com?port=-1",
+			websocket:     true,
+			expectedError: `unable to parse "-1" as a port: strconv.ParseUint: parsing "-1": invalid syntax`,
+		},
+		"one port": {
+			url:       "http://example.com?port=80",
+			websocket: true,
+			expectedOpts: &V4Options{
+				Ports: []int32{80},
+			},
+		},
+		"multiple ports": {
+			url:       "http://example.com?port=80,90,100",
+			websocket: true,
+			expectedOpts: &V4Options{
+				Ports: []int32{80, 90, 100},
+			},
+		},
+		"multiple port": {
+			url:       "http://example.com?port=80&port=90",
+			websocket: true,
+			expectedOpts: &V4Options{
+				Ports: []int32{80, 90},
+			},
+		},
+	}
+	for name, test := range tests {
+		req, err := http.NewRequest(http.MethodGet, test.url, nil)
+		if err != nil {
+			t.Errorf("%s: invalid url %q err=%q", name, test.url, err)
+			continue
+		}
+		if test.websocket {
+			req.Header.Set("Connection", "Upgrade")
+			req.Header.Set("Upgrade", "websocket")
+		}
+		opts, err := NewV4Options(req)
+		if len(test.expectedError) > 0 {
+			if err == nil {
+				t.Errorf("%s: expected err=%q, but it was nil", name, test.expectedError)
+			}
+			if e, a := test.expectedError, err.Error(); e != a {
+				t.Errorf("%s: expected err=%q, got %q", name, e, a)
+			}
+			continue
+		}
+		if err != nil {
+			t.Errorf("%s: unexpected error %v", name, err)
+			continue
+		}
+		if !reflect.DeepEqual(test.expectedOpts, opts) {
+			t.Errorf("%s: expected options %#v, got %#v", name, test.expectedOpts, err)
+		}
+	}
+}

internal/kubernetes/remotecommand/attach.go

@@ -0,0 +1,79 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package remotecommand
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"time"
+
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	remotecommandconsts "k8s.io/apimachinery/pkg/util/remotecommand"
+	"k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/client-go/tools/remotecommand"
+	utilexec "k8s.io/utils/exec"
+)
+
+// Attacher knows how to attach to a container in a pod.
+type Attacher interface {
+	// AttachToContainer attaches to a container in the pod, copying data
+	// between in/out/err and the container's stdin/stdout/stderr.
+	AttachToContainer(name string, uid types.UID, container string, in io.Reader, out, err io.WriteCloser, tty bool, resize <-chan remotecommand.TerminalSize, timeout time.Duration) error
+}
+
+// ServeAttach handles requests to attach to a container. After
+// creating/receiving the required streams, it delegates the actual attachment
+// to the attacher.
+func ServeAttach(w http.ResponseWriter, req *http.Request, attacher Attacher, podName string, uid types.UID, container string, streamOpts *Options, idleTimeout, streamCreationTimeout time.Duration, supportedProtocols []string) {
+	ctx, ok := createStreams(req, w, streamOpts, supportedProtocols, idleTimeout, streamCreationTimeout)
+	if !ok {
+		// error is handled by createStreams
+		return
+	}
+	defer ctx.conn.Close()
+
+	err := attacher.AttachToContainer(podName, uid, container, ctx.stdinStream, ctx.stdoutStream, ctx.stderrStream, ctx.tty, ctx.resizeChan, 0)
+	if err != nil {
+		if exitErr, ok := err.(utilexec.ExitError); ok && exitErr.Exited() {
+			rc := exitErr.ExitStatus()
+			ctx.writeStatus(&apierrors.StatusError{ErrStatus: metav1.Status{
+				Status: metav1.StatusFailure,
+				Reason: remotecommandconsts.NonZeroExitCodeReason,
+				Details: &metav1.StatusDetails{
+					Causes: []metav1.StatusCause{
+						{
+							Type:    remotecommandconsts.ExitCodeCauseType,
+							Message: fmt.Sprintf("%d", rc),
+						},
+					},
+				},
+				Message: fmt.Sprintf("command terminated with non-zero exit code: %v", exitErr),
+			}})
+			return
+		}
+		err = fmt.Errorf("error attaching to container: %v", err)
+		runtime.HandleError(err)
+		ctx.writeStatus(apierrors.NewInternalError(err))
+		return
+	}
+	ctx.writeStatus(&apierrors.StatusError{ErrStatus: metav1.Status{
+		Status: metav1.StatusSuccess,
+	}})
+}

internal/kubernetes/remotecommand/httpstream.go

@@ -148,6 +148,8 @@ func createHTTPStreamStreams(req *http.Request, w http.ResponseWriter, opts *Opt
 
 	var handler protocolHandler
 	switch protocol {
+	case remotecommandconsts.StreamProtocolV5Name:
+		handler = &v5ProtocolHandler{}
 	case remotecommandconsts.StreamProtocolV4Name:
 		handler = &v4ProtocolHandler{}
 	case remotecommandconsts.StreamProtocolV3Name:
@@ -159,6 +161,9 @@ func createHTTPStreamStreams(req *http.Request, w http.ResponseWriter, opts *Opt
 		fallthrough
 	case remotecommandconsts.StreamProtocolV1Name:
 		handler = &v1ProtocolHandler{}
+	default:
+		klog.Errorf("unable to create HTTP stream: unknown protocol %q", protocol)
+		return nil, false
 	}
 
 	// count the streams client asked for, starting with 1
@@ -199,6 +204,57 @@ type protocolHandler interface {
 	supportsTerminalResizing() bool
 }
 
+// v5ProtocolHandler implements the V5 protocol version for streaming command execution.
+type v5ProtocolHandler struct{}
+
+func (*v5ProtocolHandler) waitForStreams(streams <-chan streamAndReply, expectedStreams int, expired <-chan time.Time) (*context, error) {
+	ctx := &context{}
+	receivedStreams := 0
+	replyChan := make(chan struct{})
+	stop := make(chan struct{})
+	defer close(stop)
+WaitForStreams:
+	for {
+		select {
+		case stream := <-streams:
+			streamType := stream.Headers().Get(api.StreamType)
+			switch streamType {
+			case api.StreamTypeError:
+				ctx.writeStatus = v4WriteStatusFunc(stream) // write json errors
+				go waitStreamReply(stream.replySent, replyChan, stop)
+			case api.StreamTypeStdin:
+				ctx.stdinStream = stream
+				go waitStreamReply(stream.replySent, replyChan, stop)
+			case api.StreamTypeStdout:
+				ctx.stdoutStream = stream
+				go waitStreamReply(stream.replySent, replyChan, stop)
+			case api.StreamTypeStderr:
+				ctx.stderrStream = stream
+				go waitStreamReply(stream.replySent, replyChan, stop)
+			case api.StreamTypeResize:
+				ctx.resizeStream = stream
+				go waitStreamReply(stream.replySent, replyChan, stop)
+			default:
+				runtime.HandleError(fmt.Errorf("unexpected stream type: %q", streamType))
+			}
+		case <-replyChan:
+			receivedStreams++
+			if receivedStreams == expectedStreams {
+				break WaitForStreams
+			}
+		case <-expired:
+			// TODO find a way to return the error to the user. Maybe use a separate
+			// stream to report errors?
+			return nil, errors.New("timed out waiting for client to create streams")
+		}
+	}
+
+	return ctx, nil
+}
+
+// supportsTerminalResizing returns true because v5ProtocolHandler supports it
+func (*v5ProtocolHandler) supportsTerminalResizing() bool { return true }
+
 // v4ProtocolHandler implements the V4 protocol version for streaming command execution. It only differs
 // in from v3 in the error stream format using an json-marshaled metav1.Status which carries
 // the process' exit code.

internal/lockdeps/lockdeps.go

@@ -1,9 +0,0 @@
-package lockdeps
-
-import (
-	// TODO(Sargun): Remove in Go1.13
-	// This is a dep that `go mod tidy` keeps removing, because it's a transitive dep that's pulled in via a test
-	// See: https://github.com/golang/go/issues/29702
-	_ "github.com/prometheus/client_golang/prometheus"
-	_ "golang.org/x/sys/unix"
-)

internal/manager/resource.go

@@ -32,7 +32,12 @@ type ResourceManager struct {
 }
 
 // NewResourceManager returns a ResourceManager with the internal maps initialized.
-func NewResourceManager(podLister corev1listers.PodLister, secretLister corev1listers.SecretLister, configMapLister corev1listers.ConfigMapLister, serviceLister corev1listers.ServiceLister) (*ResourceManager, error) {
+func NewResourceManager(
+	podLister corev1listers.PodLister,
+	secretLister corev1listers.SecretLister,
+	configMapLister corev1listers.ConfigMapLister,
+	serviceLister corev1listers.ServiceLister,
+) (*ResourceManager, error) {
 	rm := ResourceManager{
 		podLister:       podLister,
 		secretLister:    secretLister,

internal/manager/resource_test.go

@@ -123,7 +123,7 @@ func TestGetConfigMap(t *testing.T) {
 	}
 	value := configMap.Data["key-0"]
 	if value != "val-0" {
-		t.Fatal("got unexpected value", string(value))
+		t.Fatal("got unexpected value", value)
 	}
 
 	// Try to get a configmap that does not exist, and make sure we've got a "not found" error as a response.

internal/podutils/README.md

@@ -0,0 +1,15 @@
+Much of this is copied from k8s.io/kubernetes, even if it isn't a 1-1 copy of a
+file.  This exists so we do not have to import from k8s.io/kubernetes which is
+currently problematic.  Ideally most or all of this will go away and an upstream
+solution is found so that we can share an implementation with Kubelet without
+importing from k8s.io/kubernetes
+
+
+| filename | upstream location |
+|----------|-------------------|
+| envvars.go | https://github.com/kubernetes/kubernetes/blob/98d5dc5d36d34a7ee13368a7893dcb400ec4e566/pkg/kubelet/envvars/envvars.go#L32 |
+| helper.go#ConvertDownwardAPIFieldLabel | https://github.com/kubernetes/kubernetes/blob/98d5dc5d36d34a7ee13368a7893dcb400ec4e566/pkg/apis/core/pods/helpers.go#L65 |
+| helper.go#ExtractFieldPathAsString | https://github.com/kubernetes/kubernetes/blob/98d5dc5d36d34a7ee13368a7893dcb400ec4e566/pkg/fieldpath/fieldpath.go#L46 |
+| helper.go#SplitMaybeSubscriptedPath | https://github.com/kubernetes/kubernetes/blob/98d5dc5d36d34a7ee13368a7893dcb400ec4e566/pkg/fieldpath/fieldpath.go#L96 |
+| helper.go#FormatMap | https://github.com/kubernetes/kubernetes/blob/ea0764452222146c47ec826977f49d7001b0ea8c/pkg/fieldpath/fieldpath.go#L29 |
+| helper.go#IsServiceIPSet | https://github.com/kubernetes/kubernetes/blob/ea0764452222146c47ec826977f49d7001b0ea8c/pkg/apis/core/v1/helper/helpers.go#L139 |

internal/podutils/env.go

@@ -29,11 +29,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/sets"
 	apivalidation "k8s.io/apimachinery/pkg/util/validation"
 	"k8s.io/client-go/tools/record"
-	podshelper "k8s.io/kubernetes/pkg/apis/core/pods"
-	v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper"
-	fieldpath "k8s.io/kubernetes/pkg/fieldpath"
-	"k8s.io/kubernetes/pkg/kubelet/envvars"
-	"k8s.io/utils/pointer"
+	"k8s.io/utils/ptr"
 )
 
 const (
@@ -139,7 +135,7 @@ func getServiceEnvVarMap(rm *manager.ResourceManager, ns string, enableServiceLi
 	for i := range services {
 		service := services[i]
 		// ignore services where ClusterIP is "None" or empty
-		if !v1helper.IsServiceIPSet(service) {
+		if !IsServiceIPSet(service) {
 			continue
 		}
 		serviceName := service.Name
@@ -162,7 +158,7 @@ func getServiceEnvVarMap(rm *manager.ResourceManager, ns string, enableServiceLi
 		mappedServices = append(mappedServices, serviceMap[key])
 	}
 
-	for _, e := range envvars.FromServices(mappedServices) {
+	for _, e := range FromServices(mappedServices) {
 		m[e.Name] = e.Value
 	}
 	return m, nil
@@ -338,7 +334,7 @@ func getEnvironmentVariableValue(ctx context.Context, env *corev1.EnvVar, mappin
 		return getEnvironmentVariableValueWithValueFrom(ctx, env, mappingFunc, pod, container, rm, recorder)
 	}
 	// Handle values that have been directly provided after expanding variable references.
-	return pointer.StringPtr(expansion.Expand(env.Value, mappingFunc)), nil
+	return ptr.To(expansion.Expand(env.Value, mappingFunc)), nil
 }
 
 func getEnvironmentVariableValueWithValueFrom(ctx context.Context, env *corev1.EnvVar, mappingFunc func(string) string, pod *corev1.Pod, container *corev1.Container, rm *manager.ResourceManager, recorder record.EventRecorder) (*string, error) {
@@ -415,7 +411,7 @@ func getEnvironmentVariableValueWithValueFromConfigMapKeyRef(ctx context.Context
 		return nil, fmt.Errorf("configmap %q doesn't contain the %q key required by pod %s", vf.Name, vf.Key, pod.Name)
 	}
 	// Populate the environment variable and continue on to the next reference.
-	return pointer.StringPtr(keyValue), nil
+	return ptr.To(keyValue), nil
 }
 
 func getEnvironmentVariableValueWithValueFromSecretKeyRef(ctx context.Context, env *corev1.EnvVar, mappingFunc func(string) string, pod *corev1.Pod, container *corev1.Container, rm *manager.ResourceManager, recorder record.EventRecorder) (*string, error) {
@@ -467,7 +463,7 @@ func getEnvironmentVariableValueWithValueFromSecretKeyRef(ctx context.Context, e
 		return nil, fmt.Errorf("secret %q doesn't contain the %q key required by pod %s", vf.Name, vf.Key, pod.Name)
 	}
 	// Populate the environment variable and continue on to the next reference.
-	return pointer.StringPtr(string(keyValue)), nil
+	return ptr.To(string(keyValue)), nil
 }
 
 // Handle population from a field (downward API).
@@ -480,13 +476,13 @@ func getEnvironmentVariableValueWithValueFromFieldRef(ctx context.Context, env *
 		return nil, err
 	}
 
-	return pointer.StringPtr(runtimeVal), nil
+	return ptr.To(runtimeVal), nil
 }
 
 // podFieldSelectorRuntimeValue returns the runtime value of the given
 // selector for a pod.
 func podFieldSelectorRuntimeValue(fs *corev1.ObjectFieldSelector, pod *corev1.Pod) (string, error) {
-	internalFieldPath, _, err := podshelper.ConvertDownwardAPIFieldLabel(fs.APIVersion, fs.FieldPath, "")
+	internalFieldPath, _, err := ConvertDownwardAPIFieldLabel(fs.APIVersion, fs.FieldPath, "")
 	if err != nil {
 		return "", err
 	}
@@ -497,5 +493,5 @@ func podFieldSelectorRuntimeValue(fs *corev1.ObjectFieldSelector, pod *corev1.Po
 		return pod.Spec.ServiceAccountName, nil
 
 	}
-	return fieldpath.ExtractFieldPathAsString(pod, internalFieldPath)
+	return ExtractFieldPathAsString(pod, internalFieldPath)
 }

internal/podutils/envvars.go

@@ -0,0 +1,112 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package podutils
+
+import (
+	"fmt"
+	"net"
+	"strconv"
+	"strings"
+
+	v1 "k8s.io/api/core/v1"
+)
+
+// FromServices builds environment variables that a container is started with,
+// which tell the container where to find the services it may need, which are
+// provided as an argument.
+func FromServices(services []*v1.Service) []v1.EnvVar {
+	var result []v1.EnvVar
+	for i := range services {
+		service := services[i]
+
+		// ignore services where ClusterIP is "None" or empty
+		// the services passed to this method should be pre-filtered
+		// only services that have the cluster IP set should be included here
+		if !IsServiceIPSet(service) {
+			continue
+		}
+
+		// Host
+		name := makeEnvVariableName(service.Name) + "_SERVICE_HOST"
+		result = append(result, v1.EnvVar{Name: name, Value: service.Spec.ClusterIP})
+		// First port - give it the backwards-compatible name
+		name = makeEnvVariableName(service.Name) + "_SERVICE_PORT"
+		result = append(result, v1.EnvVar{Name: name, Value: strconv.Itoa(int(service.Spec.Ports[0].Port))})
+		// All named ports (only the first may be unnamed, checked in validation)
+		for i := range service.Spec.Ports {
+			sp := &service.Spec.Ports[i]
+			if sp.Name != "" {
+				pn := name + "_" + makeEnvVariableName(sp.Name)
+				result = append(result, v1.EnvVar{Name: pn, Value: strconv.Itoa(int(sp.Port))})
+			}
+		}
+		// Docker-compatible vars.
+		result = append(result, makeLinkVariables(service)...)
+	}
+	return result
+}
+
+func makeEnvVariableName(str string) string {
+	// TODO: If we simplify to "all names are DNS1123Subdomains" this
+	// will need two tweaks:
+	//   1) Handle leading digits
+	//   2) Handle dots
+	return strings.ToUpper(strings.Replace(str, "-", "_", -1))
+}
+
+func makeLinkVariables(service *v1.Service) []v1.EnvVar {
+	prefix := makeEnvVariableName(service.Name)
+	all := []v1.EnvVar{}
+	for i := range service.Spec.Ports {
+		sp := &service.Spec.Ports[i]
+
+		protocol := string(v1.ProtocolTCP)
+		if sp.Protocol != "" {
+			protocol = string(sp.Protocol)
+		}
+
+		hostPort := net.JoinHostPort(service.Spec.ClusterIP, strconv.Itoa(int(sp.Port)))
+
+		if i == 0 {
+			// Docker special-cases the first port.
+			all = append(all, v1.EnvVar{
+				Name:  prefix + "_PORT",
+				Value: fmt.Sprintf("%s://%s", strings.ToLower(protocol), hostPort),
+			})
+		}
+		portPrefix := fmt.Sprintf("%s_PORT_%d_%s", prefix, sp.Port, strings.ToUpper(protocol))
+		all = append(all, []v1.EnvVar{
+			{
+				Name:  portPrefix,
+				Value: fmt.Sprintf("%s://%s", strings.ToLower(protocol), hostPort),
+			},
+			{
+				Name:  portPrefix + "_PROTO",
+				Value: strings.ToLower(protocol),
+			},
+			{
+				Name:  portPrefix + "_PORT",
+				Value: strconv.Itoa(int(sp.Port)),
+			},
+			{
+				Name:  portPrefix + "_ADDR",
+				Value: service.Spec.ClusterIP,
+			},
+		}...)
+	}
+	return all
+}

internal/podutils/helper.go

@@ -0,0 +1,156 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package podutils
+
+import (
+	"fmt"
+	"strings"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/meta"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apimachinery/pkg/util/validation"
+)
+
+// ConvertDownwardAPIFieldLabel converts the specified downward API field label
+// and its value in the pod of the specified version to the internal version,
+// and returns the converted label and value. This function returns an error if
+// the conversion fails.
+func ConvertDownwardAPIFieldLabel(version, label, value string) (string, string, error) {
+	if version != "v1" {
+		return "", "", fmt.Errorf("unsupported pod version: %s", version)
+	}
+
+	if path, _, ok := SplitMaybeSubscriptedPath(label); ok {
+		switch path {
+		case "metadata.annotations", "metadata.labels":
+			return label, value, nil
+		default:
+			return "", "", fmt.Errorf("field label does not support subscript: %s", label)
+		}
+	}
+
+	switch label {
+	case "metadata.annotations",
+		"metadata.labels",
+		"metadata.name",
+		"metadata.namespace",
+		"metadata.uid",
+		"spec.nodeName",
+		"spec.restartPolicy",
+		"spec.serviceAccountName",
+		"spec.schedulerName",
+		"status.phase",
+		"status.hostIP",
+		"status.podIP",
+		"status.podIPs":
+		return label, value, nil
+	// This is for backwards compatibility with old v1 clients which send spec.host
+	case "spec.host":
+		return "spec.nodeName", value, nil
+	default:
+		return "", "", fmt.Errorf("field label not supported: %s", label)
+	}
+}
+
+// ExtractFieldPathAsString extracts the field from the given object
+// and returns it as a string.  The object must be a pointer to an
+// API type.
+func ExtractFieldPathAsString(obj interface{}, fieldPath string) (string, error) {
+	accessor, err := meta.Accessor(obj)
+	if err != nil {
+		return "", err
+	}
+
+	if path, subscript, ok := SplitMaybeSubscriptedPath(fieldPath); ok {
+		switch path {
+		case "metadata.annotations":
+			if errs := validation.IsQualifiedName(strings.ToLower(subscript)); len(errs) != 0 {
+				return "", fmt.Errorf("invalid key subscript in %s: %s", fieldPath, strings.Join(errs, ";"))
+			}
+			return accessor.GetAnnotations()[subscript], nil
+		case "metadata.labels":
+			if errs := validation.IsQualifiedName(subscript); len(errs) != 0 {
+				return "", fmt.Errorf("invalid key subscript in %s: %s", fieldPath, strings.Join(errs, ";"))
+			}
+			return accessor.GetLabels()[subscript], nil
+		default:
+			return "", fmt.Errorf("fieldPath %q does not support subscript", fieldPath)
+		}
+	}
+
+	switch fieldPath {
+	case "metadata.annotations":
+		return FormatMap(accessor.GetAnnotations()), nil
+	case "metadata.labels":
+		return FormatMap(accessor.GetLabels()), nil
+	case "metadata.name":
+		return accessor.GetName(), nil
+	case "metadata.namespace":
+		return accessor.GetNamespace(), nil
+	case "metadata.uid":
+		return string(accessor.GetUID()), nil
+	}
+
+	return "", fmt.Errorf("unsupported fieldPath: %v", fieldPath)
+}
+
+// SplitMaybeSubscriptedPath checks whether the specified fieldPath is
+// subscripted, and
+//   - if yes, this function splits the fieldPath into path and subscript, and
+//     returns (path, subscript, true).
+//   - if no, this function returns (fieldPath, "", false).
+//
+// Example inputs and outputs:
+//   - "metadata.annotations['myKey']" --> ("metadata.annotations", "myKey", true)
+//   - "metadata.annotations['a[b]c']" --> ("metadata.annotations", "a[b]c", true)
+//   - "metadata.labels[”]"           --> ("metadata.labels", "", true)
+//   - "metadata.labels"               --> ("metadata.labels", "", false)
+func SplitMaybeSubscriptedPath(fieldPath string) (string, string, bool) {
+	if !strings.HasSuffix(fieldPath, "']") {
+		return fieldPath, "", false
+	}
+	s := strings.TrimSuffix(fieldPath, "']")
+	parts := strings.SplitN(s, "['", 2)
+	if len(parts) < 2 {
+		return fieldPath, "", false
+	}
+	if len(parts[0]) == 0 {
+		return fieldPath, "", false
+	}
+	return parts[0], parts[1], true
+}
+
+// FormatMap formats map[string]string to a string.
+func FormatMap(m map[string]string) (fmtStr string) {
+	// output with keys in sorted order to provide stable output
+	keys := sets.NewString()
+	for key := range m {
+		keys.Insert(key)
+	}
+	for _, key := range keys.List() {
+		fmtStr += fmt.Sprintf("%v=%q\n", key, m[key])
+	}
+	fmtStr = strings.TrimSuffix(fmtStr, "\n")
+
+	return
+}
+
+// IsServiceIPSet aims to check if the service's ClusterIP is set or not the objective is not to perform validation here
+func IsServiceIPSet(service *corev1.Service) bool {
+	return service.Spec.ClusterIP != corev1.ClusterIPNone && service.Spec.ClusterIP != ""
+}

internal/queue/queue.go

@@ -15,6 +15,7 @@
 package queue
 
 import (
+	"container/list"
 	"context"
 	"fmt"
 	"sync"
@@ -23,8 +24,10 @@ import (
 	pkgerrors "github.com/pkg/errors"
 	"github.com/virtual-kubelet/virtual-kubelet/log"
 	"github.com/virtual-kubelet/virtual-kubelet/trace"
+	"golang.org/x/sync/semaphore"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/util/workqueue"
+	"k8s.io/utils/clock"
 )
 
 const (
@@ -32,58 +35,275 @@ const (
 	MaxRetries = 20
 )
 
+// ShouldRetryFunc is a mechanism to have a custom retry policy
+type ShouldRetryFunc func(ctx context.Context, key string, timesTried int, originallyAdded time.Time, err error) (*time.Duration, error)
+
 // ItemHandler is a callback that handles a single key on the Queue
 type ItemHandler func(ctx context.Context, key string) error
 
 // Queue implements a wrapper around workqueue with native VK instrumentation
 type Queue struct {
-	lock      sync.Mutex
-	running   bool
-	name      string
-	workqueue workqueue.RateLimitingInterface
-	handler   ItemHandler
+	// clock is used for testing
+	clock clock.Clock
+	// lock protects running, and the items list / map
+	lock    sync.Mutex
+	running bool
+	name    string
+	handler ItemHandler
+
+	ratelimiter workqueue.TypedRateLimiter[any]
+	// items are items that are marked dirty waiting for processing.
+	items *list.List
+	// itemInQueue is a map of (string) key -> item while it is in the items list
+	itemsInQueue map[string]*list.Element
+	// itemsBeingProcessed is a map of (string) key -> item once it has been moved
+	itemsBeingProcessed map[string]*queueItem
+	// Wait for next semaphore is an exclusive (1 item) lock that is taken every time items is checked to see if there
+	// is an item in queue for work
+	waitForNextItemSemaphore *semaphore.Weighted
+
+	// wakeup
+	wakeupCh chan struct{}
+
+	retryFunc ShouldRetryFunc
+}
+
+type queueItem struct {
+	key                    string
+	plannedToStartWorkAt   time.Time
+	redirtiedAt            time.Time
+	redirtiedWithRatelimit bool
+	forget                 bool
+	requeues               int
+
+	// Debugging information only
+	originallyAdded     time.Time
+	addedViaRedirty     bool
+	delayedViaRateLimit *time.Duration
+}
+
+func (item *queueItem) String() string {
+	return fmt.Sprintf("<plannedToStartWorkAt:%s key: %s>", item.plannedToStartWorkAt.String(), item.key)
 }
 
 // New creates a queue
 //
-// It expects to get a item rate limiter, and a friendly name which is used in logs, and
-// in the internal kubernetes metrics.
-func New(ratelimiter workqueue.RateLimiter, name string, handler ItemHandler) *Queue {
+// It expects to get a item rate limiter, and a friendly name which is used in logs, and in the internal kubernetes
+// metrics. If retryFunc is nil, the default retry function.
+func New(ratelimiter workqueue.TypedRateLimiter[any], name string, handler ItemHandler, retryFunc ShouldRetryFunc) *Queue {
+	if retryFunc == nil {
+		retryFunc = DefaultRetryFunc
+	}
 	return &Queue{
-		name:      name,
-		workqueue: workqueue.NewNamedRateLimitingQueue(ratelimiter, name),
-		handler:   handler,
+		clock:                    clock.RealClock{},
+		name:                     name,
+		ratelimiter:              ratelimiter,
+		items:                    list.New(),
+		itemsBeingProcessed:      make(map[string]*queueItem),
+		itemsInQueue:             make(map[string]*list.Element),
+		handler:                  handler,
+		wakeupCh:                 make(chan struct{}, 1),
+		waitForNextItemSemaphore: semaphore.NewWeighted(1),
+		retryFunc:                retryFunc,
 	}
 }
 
 // Enqueue enqueues the key in a rate limited fashion
-func (q *Queue) Enqueue(key string) {
-	q.workqueue.AddRateLimited(key)
+func (q *Queue) Enqueue(ctx context.Context, key string) {
+	q.lock.Lock()
+	defer q.lock.Unlock()
+
+	q.insert(ctx, key, true, nil)
 }
 
 // EnqueueWithoutRateLimit enqueues the key without a rate limit
-func (q *Queue) EnqueueWithoutRateLimit(key string) {
-	q.workqueue.Add(key)
+func (q *Queue) EnqueueWithoutRateLimit(ctx context.Context, key string) {
+	q.lock.Lock()
+	defer q.lock.Unlock()
+
+	q.insert(ctx, key, false, nil)
 }
 
 // Forget forgets the key
-func (q *Queue) Forget(key string) {
-	q.workqueue.Forget(key)
+func (q *Queue) Forget(ctx context.Context, key string) {
+	q.lock.Lock()
+	defer q.lock.Unlock()
+	ctx, span := trace.StartSpan(ctx, "Forget")
+	defer span.End()
+
+	ctx = span.WithFields(ctx, map[string]interface{}{
+		"queue": q.name,
+		"key":   key,
+	})
+
+	if item, ok := q.itemsInQueue[key]; ok {
+		span.WithField(ctx, "status", "itemInQueue")
+		delete(q.itemsInQueue, key)
+		q.items.Remove(item)
+		return
+	}
+
+	if qi, ok := q.itemsBeingProcessed[key]; ok {
+		span.WithField(ctx, "status", "itemBeingProcessed")
+		qi.forget = true
+		return
+	}
+	span.WithField(ctx, "status", "notfound")
 }
 
-// EnqueueAfter enqueues the item after this period
-//
-// Since it wrap workqueue semantics, if an item has been enqueued  after, and it is immediately scheduled for work,
-// it will process the immediate item, and then upon the latter delayed processing it will be processed again
-func (q *Queue) EnqueueAfter(key string, after time.Duration) {
-	q.workqueue.AddAfter(key, after)
+func durationDeref(duration *time.Duration, def time.Duration) time.Duration {
+	if duration == nil {
+		return def
+	}
+
+	return *duration
+}
+
+// insert inserts a new item to be processed at time time. It will not further delay items if when is later than the
+// original time the item was scheduled to be processed. If when is earlier, it will "bring it forward"
+// If ratelimit is specified, and delay is nil, then the ratelimiter's delay (return from When function) will be used
+// If ratelimit is specified, and the delay is non-nil, then the delay value will be used
+// If ratelimit is false, then only delay is used to schedule the work. If delay is nil, it will be considered 0.
+func (q *Queue) insert(ctx context.Context, key string, ratelimit bool, delay *time.Duration) *queueItem {
+	ctx, span := trace.StartSpan(ctx, "insert")
+	defer span.End()
+
+	ctx = span.WithFields(ctx, map[string]interface{}{
+		"queue":     q.name,
+		"key":       key,
+		"ratelimit": ratelimit,
+	})
+	if delay == nil {
+		ctx = span.WithField(ctx, "delay", "nil")
+	} else {
+		ctx = span.WithField(ctx, "delay", delay.String())
+	}
+
+	defer func() {
+		select {
+		case q.wakeupCh <- struct{}{}:
+		default:
+		}
+	}()
+
+	// First see if the item is already being processed
+	if item, ok := q.itemsBeingProcessed[key]; ok {
+		span.WithField(ctx, "status", "itemsBeingProcessed")
+		when := q.clock.Now().Add(durationDeref(delay, 0))
+		// Is the item already been redirtied?
+		if item.redirtiedAt.IsZero() {
+			item.redirtiedAt = when
+			item.redirtiedWithRatelimit = ratelimit
+		} else if when.Before(item.redirtiedAt) {
+			item.redirtiedAt = when
+			item.redirtiedWithRatelimit = ratelimit
+		}
+		item.forget = false
+		return item
+	}
+
+	// Is the item already in the queue?
+	if item, ok := q.itemsInQueue[key]; ok {
+		span.WithField(ctx, "status", "itemsInQueue")
+		qi := item.Value.(*queueItem)
+		when := q.clock.Now().Add(durationDeref(delay, 0))
+		q.adjustPosition(qi, item, when)
+		return qi
+	}
+
+	span.WithField(ctx, "status", "added")
+	now := q.clock.Now()
+	val := &queueItem{
+		key:                  key,
+		plannedToStartWorkAt: now,
+		originallyAdded:      now,
+	}
+
+	if ratelimit {
+		actualDelay := q.ratelimiter.When(key)
+		// Check if delay is overridden
+		if delay != nil {
+			actualDelay = *delay
+		}
+		span.WithField(ctx, "delay", actualDelay.String())
+		val.plannedToStartWorkAt = val.plannedToStartWorkAt.Add(actualDelay)
+		val.delayedViaRateLimit = &actualDelay
+	} else {
+		val.plannedToStartWorkAt = val.plannedToStartWorkAt.Add(durationDeref(delay, 0))
+	}
+
+	for item := q.items.Back(); item != nil; item = item.Prev() {
+		qi := item.Value.(*queueItem)
+		if qi.plannedToStartWorkAt.Before(val.plannedToStartWorkAt) {
+			q.itemsInQueue[key] = q.items.InsertAfter(val, item)
+			return val
+		}
+	}
+
+	q.itemsInQueue[key] = q.items.PushFront(val)
+	return val
+}
+
+func (q *Queue) adjustPosition(qi *queueItem, element *list.Element, when time.Time) {
+	if when.After(qi.plannedToStartWorkAt) {
+		// The item has already been delayed appropriately
+		return
+	}
+
+	qi.plannedToStartWorkAt = when
+	for prev := element.Prev(); prev != nil; prev = prev.Prev() {
+		item := prev.Value.(*queueItem)
+		// does this item plan to start work *before* the new time? If so add it
+		if item.plannedToStartWorkAt.Before(when) {
+			q.items.MoveAfter(element, prev)
+			return
+		}
+	}
+
+	q.items.MoveToFront(element)
+}
+
+// EnqueueWithoutRateLimitWithDelay enqueues without rate limiting, but work will not start for this given delay period
+func (q *Queue) EnqueueWithoutRateLimitWithDelay(ctx context.Context, key string, after time.Duration) {
+	q.lock.Lock()
+	defer q.lock.Unlock()
+	q.insert(ctx, key, false, &after)
 }
 
 // Empty returns if the queue has no items in it
 //
-// It should only be used for debugging, as delayed items are not counted, leading to confusion
+// It should only be used for debugging.
 func (q *Queue) Empty() bool {
-	return q.workqueue.Len() == 0
+	return q.Len() == 0
+}
+
+// Len includes items that are in the queue, and are being processed
+func (q *Queue) Len() int {
+	q.lock.Lock()
+	defer q.lock.Unlock()
+	if q.items.Len() != len(q.itemsInQueue) {
+		panic("Internally inconsistent state")
+	}
+
+	return q.items.Len() + len(q.itemsBeingProcessed)
+}
+
+// UnprocessedLen returns the count of items yet to be processed in the queue
+func (q *Queue) UnprocessedLen() int {
+	q.lock.Lock()
+	defer q.lock.Unlock()
+	if q.items.Len() != len(q.itemsInQueue) {
+		panic("Internally inconsistent state")
+	}
+
+	return len(q.itemsInQueue)
+}
+
+// ProcessedLen returns the count items that are being processed
+func (q *Queue) ItemsBeingProcessedLen() int {
+	q.lock.Lock()
+	defer q.lock.Unlock()
+	return len(q.itemsBeingProcessed)
 }
 
 // Run starts the workers
@@ -112,13 +332,14 @@ func (q *Queue) Run(ctx context.Context, workers int) {
 
 	group := &wait.Group{}
 	for i := 0; i < workers; i++ {
+		// This is required because i is referencing a mutable variable and that's running in a separate goroutine
+		idx := i
 		group.StartWithContext(ctx, func(ctx context.Context) {
-			q.worker(ctx, i)
+			q.worker(ctx, idx)
 		})
 	}
 	defer group.Wait()
 	<-ctx.Done()
-	q.workqueue.ShutDown()
 }
 
 func (q *Queue) worker(ctx context.Context, i int) {
@@ -130,6 +351,54 @@ func (q *Queue) worker(ctx context.Context, i int) {
 	}
 }
 
+func (q *Queue) getNextItem(ctx context.Context) (*queueItem, error) {
+	if err := q.waitForNextItemSemaphore.Acquire(ctx, 1); err != nil {
+		return nil, err
+	}
+	defer q.waitForNextItemSemaphore.Release(1)
+
+	for {
+		q.lock.Lock()
+		element := q.items.Front()
+		if element == nil {
+			// Wait for the next item
+			q.lock.Unlock()
+			select {
+			case <-ctx.Done():
+				return nil, ctx.Err()
+			case <-q.wakeupCh:
+			}
+		} else {
+			qi := element.Value.(*queueItem)
+			timeUntilProcessing := time.Until(qi.plannedToStartWorkAt)
+
+			// Do we need to sleep? If not, let's party.
+			if timeUntilProcessing <= 0 {
+				q.itemsBeingProcessed[qi.key] = qi
+				q.items.Remove(element)
+				delete(q.itemsInQueue, qi.key)
+				q.lock.Unlock()
+				return qi, nil
+			}
+
+			q.lock.Unlock()
+			if err := func() error {
+				timer := q.clock.NewTimer(timeUntilProcessing)
+				defer timer.Stop()
+				select {
+				case <-timer.C():
+				case <-ctx.Done():
+					return ctx.Err()
+				case <-q.wakeupCh:
+				}
+				return nil
+			}(); err != nil {
+				return nil, err
+			}
+		}
+	}
+}
+
 // handleQueueItem handles a single item
 //
 // A return value of "false" indicates that further processing should be stopped.
@@ -137,8 +406,9 @@ func (q *Queue) handleQueueItem(ctx context.Context) bool {
 	ctx, span := trace.StartSpan(ctx, "handleQueueItem")
 	defer span.End()
 
-	obj, shutdown := q.workqueue.Get()
-	if shutdown {
+	qi, err := q.getNextItem(ctx)
+	if err != nil {
+		span.SetStatus(err)
 		return false
 	}
 
@@ -146,11 +416,10 @@ func (q *Queue) handleQueueItem(ctx context.Context) bool {
 	// These are of the form namespace/name.
 	// We do this as the delayed nature of the work Queue means the items in the informer cache may actually be more u
 	// to date that when the item was initially put onto the workqueue.
-	key := obj.(string)
-	ctx = span.WithField(ctx, "key", key)
+	ctx = span.WithField(ctx, "key", qi.key)
 	log.G(ctx).Debug("Got Queue object")
 
-	err := q.handleQueueItemObject(ctx, key)
+	err = q.handleQueueItemObject(ctx, qi)
 	if err != nil {
 		// We've actually hit an error, so we set the span's status based on the error.
 		span.SetStatus(err)
@@ -162,35 +431,90 @@ func (q *Queue) handleQueueItem(ctx context.Context) bool {
 	return true
 }
 
-func (q *Queue) handleQueueItemObject(ctx context.Context, key string) error {
+func (q *Queue) handleQueueItemObject(ctx context.Context, qi *queueItem) error {
 	// This is a separate function / span, because the handleQueueItem span is the time spent waiting for the object
 	// plus the time spend handling the object. Instead, this function / span is scoped to a single object.
 	ctx, span := trace.StartSpan(ctx, "handleQueueItemObject")
 	defer span.End()
-	ctx = span.WithField(ctx, "key", key)
 
-	// We call Done here so the work Queue knows we have finished processing this item.
-	// We also must remember to call Forget if we do not want this work item being re-queued.
-	// For example, we do not call Forget if a transient error occurs.
-	// Instead, the item is put back on the work Queue and attempted again after a back-off period.
-	defer q.workqueue.Done(key)
+	ctx = span.WithFields(ctx, map[string]interface{}{
+		"requeues":        qi.requeues,
+		"originallyAdded": qi.originallyAdded.String(),
+		"addedViaRedirty": qi.addedViaRedirty,
+		"plannedForWork":  qi.plannedToStartWorkAt.String(),
+	})
+
+	if qi.delayedViaRateLimit != nil {
+		ctx = span.WithField(ctx, "delayedViaRateLimit", qi.delayedViaRateLimit.String())
+	}
 
 	// Add the current key as an attribute to the current span.
-	ctx = span.WithField(ctx, "key", key)
+	ctx = span.WithField(ctx, "key", qi.key)
 	// Run the syncHandler, passing it the namespace/name string of the Pod resource to be synced.
-	if err := q.handler(ctx, key); err != nil {
-		if q.workqueue.NumRequeues(key) < MaxRetries {
+	err := q.handler(ctx, qi.key)
+
+	q.lock.Lock()
+	defer q.lock.Unlock()
+
+	delete(q.itemsBeingProcessed, qi.key)
+	if qi.forget {
+		q.ratelimiter.Forget(qi.key)
+		log.G(ctx).WithError(err).Warnf("forgetting %q as told to forget while in progress", qi.key)
+		return nil
+	}
+
+	if err != nil {
+		ctx = span.WithField(ctx, "error", err.Error())
+		var delay *time.Duration
+
+		// Stash the original error for logging below
+		originalError := err
+		delay, err = q.retryFunc(ctx, qi.key, qi.requeues+1, qi.originallyAdded, err)
+		if err == nil {
 			// Put the item back on the work Queue to handle any transient errors.
-			log.G(ctx).WithError(err).Warnf("requeuing %q due to failed sync", key)
-			q.workqueue.AddRateLimited(key)
+			log.G(ctx).WithError(originalError).Warnf("requeuing %q due to failed sync", qi.key)
+			newQI := q.insert(ctx, qi.key, true, delay)
+			newQI.requeues = qi.requeues + 1
+			newQI.originallyAdded = qi.originallyAdded
+
 			return nil
 		}
-		// We've exceeded the maximum retries, so we must Forget the key.
-		q.workqueue.Forget(key)
-		return pkgerrors.Wrapf(err, "forgetting %q due to maximum retries reached", key)
+		if !qi.redirtiedAt.IsZero() {
+			err = fmt.Errorf("temporarily (requeued) forgetting %q due to: %w", qi.key, err)
+		} else {
+			err = fmt.Errorf("forgetting %q due to: %w", qi.key, err)
+		}
 	}
-	// Finally, if no error occurs we Forget this item so it does not get queued again until another change happens.
-	q.workqueue.Forget(key)
 
-	return nil
+	// We've exceeded the maximum retries or we were successful.
+	q.ratelimiter.Forget(qi.key)
+	if !qi.redirtiedAt.IsZero() {
+		delay := time.Until(qi.redirtiedAt)
+		newQI := q.insert(ctx, qi.key, qi.redirtiedWithRatelimit, &delay)
+		newQI.addedViaRedirty = true
+	}
+
+	span.SetStatus(err)
+	return err
+}
+
+func (q *Queue) String() string {
+	q.lock.Lock()
+	defer q.lock.Unlock()
+
+	items := make([]string, 0, q.items.Len())
+
+	for next := q.items.Front(); next != nil; next = next.Next() {
+		items = append(items, next.Value.(*queueItem).String())
+	}
+	return fmt.Sprintf("<items:%s>", items)
+}
+
+// DefaultRetryFunc is the default function used for retries by the queue subsystem.
+func DefaultRetryFunc(ctx context.Context, key string, timesTried int, originallyAdded time.Time, err error) (*time.Duration, error) {
+	if timesTried < MaxRetries {
+		return nil, nil
+	}
+
+	return nil, pkgerrors.Wrapf(err, "maximum retries (%d) reached", MaxRetries)
 }

internal/queue/queue_test.go

@@ -5,20 +5,24 @@ import (
 	"errors"
 	"strconv"
 	"sync"
+	"sync/atomic"
 	"testing"
 	"time"
 
 	"github.com/sirupsen/logrus"
 	"github.com/virtual-kubelet/virtual-kubelet/log"
 	logruslogger "github.com/virtual-kubelet/virtual-kubelet/log/logrus"
-	"go.uber.org/goleak"
 	"golang.org/x/time/rate"
 	"gotest.tools/assert"
 	is "gotest.tools/assert/cmp"
-	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/util/workqueue"
+	"k8s.io/utils/clock"
 )
 
+func durationPtr(d time.Duration) *time.Duration {
+	return &d
+}
+
 func TestQueueMaxRetries(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
@@ -31,19 +35,70 @@ func TestQueueMaxRetries(t *testing.T) {
 		n++
 		return knownErr
 	}
-	wq := New(workqueue.NewMaxOfRateLimiter(
+	wq := New(workqueue.NewTypedMaxOfRateLimiter[any](
 		// The default upper bound is 1000 seconds. Let's not use that.
-		workqueue.NewItemExponentialFailureRateLimiter(5*time.Millisecond, 10*time.Millisecond),
-		&workqueue.BucketRateLimiter{Limiter: rate.NewLimiter(rate.Limit(10), 100)},
-	), t.Name(), handler)
-	wq.Enqueue("test")
+		workqueue.NewTypedItemExponentialFailureRateLimiter[any](5*time.Millisecond, 10*time.Millisecond),
+		&workqueue.TypedBucketRateLimiter[any]{Limiter: rate.NewLimiter(rate.Limit(10), 100)},
+	), t.Name(), handler, nil)
+	wq.Enqueue(context.TODO(), "test")
 
 	for n < MaxRetries {
 		assert.Assert(t, wq.handleQueueItem(ctx))
 	}
 
 	assert.Assert(t, is.Equal(n, MaxRetries))
-	assert.Assert(t, is.Equal(0, wq.workqueue.Len()))
+	assert.Assert(t, is.Equal(0, wq.Len()))
+}
+
+func TestQueueCustomRetries(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	logger := logrus.New()
+	logger.SetLevel(logrus.DebugLevel)
+	ctx = log.WithLogger(ctx, logruslogger.FromLogrus(logrus.NewEntry(logger)))
+	n := 0
+	errorSeen := 0
+	retryTestError := errors.New("Error should be retried every 10 milliseconds")
+	handler := func(ctx context.Context, key string) error {
+		if key == "retrytest" {
+			n++
+			return retryTestError
+		}
+		return errors.New("Unknown error")
+	}
+
+	shouldRetryFunc := func(ctx context.Context, key string, timesTried int, originallyAdded time.Time, err error) (*time.Duration, error) {
+		var sleepTime *time.Duration
+		if errors.Is(err, retryTestError) {
+			errorSeen++
+			sleepTime = durationPtr(10 * time.Millisecond)
+		}
+		_, retErr := DefaultRetryFunc(ctx, key, timesTried, originallyAdded, err)
+		return sleepTime, retErr
+	}
+
+	wq := New(&workqueue.TypedBucketRateLimiter[any]{Limiter: rate.NewLimiter(rate.Limit(1000), 1000)}, t.Name(), handler, shouldRetryFunc)
+
+	timeTaken := func(key string) time.Duration {
+		start := time.Now()
+		wq.Enqueue(context.TODO(), key)
+		for i := 0; i < MaxRetries; i++ {
+			assert.Assert(t, wq.handleQueueItem(ctx))
+		}
+		return time.Since(start)
+	}
+
+	unknownTime := timeTaken("unknown")
+	assert.Assert(t, n == 0)
+	assert.Assert(t, unknownTime < 10*time.Millisecond)
+
+	retrytestTime := timeTaken("retrytest")
+	assert.Assert(t, is.Equal(n, MaxRetries))
+	assert.Assert(t, is.Equal(errorSeen, MaxRetries))
+
+	assert.Assert(t, is.Equal(0, wq.Len()))
+	assert.Assert(t, retrytestTime > 10*time.Millisecond*time.Duration(n-1))
+	assert.Assert(t, retrytestTime < 2*10*time.Millisecond*time.Duration(n-1))
 }
 
 func TestForget(t *testing.T) {
@@ -51,65 +106,405 @@ func TestForget(t *testing.T) {
 	handler := func(ctx context.Context, key string) error {
 		panic("Should never be called")
 	}
-	wq := New(workqueue.DefaultItemBasedRateLimiter(), t.Name(), handler)
+	wq := New(workqueue.DefaultTypedItemBasedRateLimiter[any](), t.Name(), handler, nil)
 
-	wq.Forget("val")
-	assert.Assert(t, is.Equal(0, wq.workqueue.Len()))
+	wq.Forget(context.TODO(), "val")
+	assert.Assert(t, is.Equal(0, wq.Len()))
 
 	v := "test"
-	wq.EnqueueWithoutRateLimit(v)
-	assert.Assert(t, is.Equal(1, wq.workqueue.Len()))
-
-	t.Skip("This is broken")
-	// Workqueue docs:
-	// Forget indicates that an item is finished being retried.  Doesn't matter whether it's for perm failing
-	// or for success, we'll stop the rate limiter from tracking it.  This only clears the `rateLimiter`, you
-	// still have to call `Done` on the queue.
-	// Even if you do this, it doesn't work: https://play.golang.com/p/8vfL_RCsFGI
-	assert.Assert(t, is.Equal(0, wq.workqueue.Len()))
-
+	wq.EnqueueWithoutRateLimit(context.TODO(), v)
+	assert.Assert(t, is.Equal(1, wq.Len()))
 }
 
-func TestQueueTerminate(t *testing.T) {
+func TestQueueEmpty(t *testing.T) {
 	t.Parallel()
-	defer goleak.VerifyNone(t,
-		// Ignore existing goroutines
-		goleak.IgnoreCurrent(),
-		// Ignore klog background flushers
-		goleak.IgnoreTopFunction("k8s.io/klog.(*loggingT).flushDaemon"),
-		goleak.IgnoreTopFunction("k8s.io/klog/v2.(*loggingT).flushDaemon"),
-		// Workqueue runs a goroutine in the background to handle background functions. AFAICT, they're unkillable
-		// and are designed to stop after a certain idle window
-		goleak.IgnoreTopFunction("k8s.io/client-go/util/workqueue.(*Type).updateUnfinishedWorkLoop"),
-		goleak.IgnoreTopFunction("k8s.io/client-go/util/workqueue.(*delayingType).waitingLoop"),
-	)
-	ctx, cancel := context.WithCancel(context.Background())
+	ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
 	defer cancel()
 
-	testMap := &sync.Map{}
-	handler := func(ctx context.Context, key string) error {
-		testMap.Store(key, struct{}{})
+	q := New(workqueue.DefaultTypedItemBasedRateLimiter[any](), t.Name(), func(ctx context.Context, key string) error {
 		return nil
-	}
+	}, nil)
 
-	wq := New(workqueue.DefaultItemBasedRateLimiter(), t.Name(), handler)
-	group := &wait.Group{}
-	group.StartWithContext(ctx, func(ctx context.Context) {
-		wq.Run(ctx, 10)
+	item, err := q.getNextItem(ctx)
+	assert.Error(t, err, context.DeadlineExceeded.Error())
+	assert.Assert(t, is.Nil(item))
+}
+
+func TestQueueItemNoSleep(t *testing.T) {
+	t.Parallel()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 1000*time.Millisecond)
+	defer cancel()
+
+	q := New(workqueue.DefaultTypedItemBasedRateLimiter[any](), t.Name(), func(ctx context.Context, key string) error {
+		return nil
+	}, nil)
+
+	q.lock.Lock()
+	q.insert(ctx, "foo", false, durationPtr(-1*time.Hour))
+	q.insert(ctx, "bar", false, durationPtr(-1*time.Hour))
+	q.lock.Unlock()
+
+	item, err := q.getNextItem(ctx)
+	assert.NilError(t, err)
+	assert.Assert(t, is.Equal(item.key, "foo"))
+
+	item, err = q.getNextItem(ctx)
+	assert.NilError(t, err)
+	assert.Assert(t, is.Equal(item.key, "bar"))
+}
+
+func TestQueueItemSleep(t *testing.T) {
+	t.Parallel()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 1000*time.Millisecond)
+	defer cancel()
+
+	q := New(workqueue.DefaultTypedItemBasedRateLimiter[any](), t.Name(), func(ctx context.Context, key string) error {
+		return nil
+	}, nil)
+	q.lock.Lock()
+	q.insert(ctx, "foo", false, durationPtr(100*time.Millisecond))
+	q.insert(ctx, "bar", false, durationPtr(100*time.Millisecond))
+	q.lock.Unlock()
+
+	item, err := q.getNextItem(ctx)
+	assert.NilError(t, err)
+	assert.Assert(t, is.Equal(item.key, "foo"))
+}
+
+func TestQueueBackgroundAdd(t *testing.T) {
+	t.Parallel()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5000*time.Millisecond)
+	defer cancel()
+
+	q := New(workqueue.DefaultTypedItemBasedRateLimiter[any](), t.Name(), func(ctx context.Context, key string) error {
+		return nil
+	}, nil)
+	start := time.Now()
+	time.AfterFunc(100*time.Millisecond, func() {
+		q.lock.Lock()
+		defer q.lock.Unlock()
+		q.insert(ctx, "foo", false, nil)
 	})
-	for i := 0; i < 1000; i++ {
-		wq.EnqueueWithoutRateLimit(strconv.Itoa(i))
+
+	item, err := q.getNextItem(ctx)
+	assert.NilError(t, err)
+	assert.Assert(t, is.Equal(item.key, "foo"))
+	assert.Assert(t, time.Since(start) > 100*time.Millisecond)
+}
+
+func TestQueueBackgroundAdvance(t *testing.T) {
+	t.Parallel()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5000*time.Millisecond)
+	defer cancel()
+
+	q := New(workqueue.DefaultTypedItemBasedRateLimiter[any](), t.Name(), func(ctx context.Context, key string) error {
+		return nil
+	}, nil)
+	start := time.Now()
+	q.lock.Lock()
+	q.insert(ctx, "foo", false, durationPtr(10*time.Second))
+	q.lock.Unlock()
+
+	time.AfterFunc(200*time.Millisecond, func() {
+		q.lock.Lock()
+		defer q.lock.Unlock()
+		q.insert(ctx, "foo", false, nil)
+	})
+
+	item, err := q.getNextItem(ctx)
+	assert.NilError(t, err)
+	assert.Assert(t, is.Equal(item.key, "foo"))
+	assert.Assert(t, time.Since(start) > 200*time.Millisecond)
+	assert.Assert(t, time.Since(start) < 5*time.Second)
+}
+
+func TestQueueRedirty(t *testing.T) {
+	t.Parallel()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5000*time.Millisecond)
+	defer cancel()
+
+	var times int64
+	var q *Queue
+	q = New(workqueue.DefaultTypedItemBasedRateLimiter[any](), t.Name(), func(ctx context.Context, key string) error {
+		assert.Assert(t, is.Equal(key, "foo"))
+		if atomic.AddInt64(&times, 1) == 1 {
+			q.EnqueueWithoutRateLimit(context.TODO(), "foo")
+		} else {
+			cancel()
+		}
+		return nil
+	}, nil)
+
+	q.EnqueueWithoutRateLimit(context.TODO(), "foo")
+	q.Run(ctx, 1)
+	for !q.Empty() {
+		time.Sleep(100 * time.Millisecond)
+	}
+	assert.Assert(t, is.Equal(atomic.LoadInt64(&times), int64(2)))
+}
+
+func TestHeapConcurrency(t *testing.T) {
+	t.Parallel()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5000*time.Millisecond)
+	defer cancel()
+
+	start := time.Now()
+	seen := sync.Map{}
+	q := New(workqueue.DefaultTypedItemBasedRateLimiter[any](), t.Name(), func(ctx context.Context, key string) error {
+		seen.Store(key, struct{}{})
+		time.Sleep(time.Second)
+		return nil
+	}, nil)
+	for i := 0; i < 20; i++ {
+		q.EnqueueWithoutRateLimit(context.TODO(), strconv.Itoa(i))
 	}
 
-	for wq.workqueue.Len() > 0 {
+	assert.Assert(t, q.Len() == 20)
+
+	go q.Run(ctx, 20)
+	for q.Len() > 0 {
 		time.Sleep(100 * time.Millisecond)
 	}
 
-	for i := 0; i < 1000; i++ {
-		_, ok := testMap.Load(strconv.Itoa(i))
-		assert.Assert(t, ok, "Item %d missing", i)
+	for i := 0; i < 20; i++ {
+		_, ok := seen.Load(strconv.Itoa(i))
+		assert.Assert(t, ok, "Did not observe: %d", i)
+	}
+	assert.Assert(t, time.Since(start) < 5*time.Second)
+}
+
+func checkConsistency(t *testing.T, q *Queue) {
+	q.lock.Lock()
+	defer q.lock.Unlock()
+
+	for next := q.items.Front(); next != nil && next.Next() != nil; next = next.Next() {
+		qi := next.Value.(*queueItem)
+		qiNext := next.Next().Value.(*queueItem)
+		assert.Assert(t, qi.plannedToStartWorkAt.Before(qiNext.plannedToStartWorkAt) || qi.plannedToStartWorkAt.Equal(qiNext.plannedToStartWorkAt))
+	}
+}
+
+func TestHeapOrder(t *testing.T) {
+	q := New(workqueue.DefaultTypedItemBasedRateLimiter[any](), t.Name(), func(ctx context.Context, key string) error {
+		return nil
+	}, nil)
+	q.clock = nonmovingClock{}
+
+	q.EnqueueWithoutRateLimitWithDelay(context.TODO(), "a", 1000)
+	q.EnqueueWithoutRateLimitWithDelay(context.TODO(), "b", 2000)
+	q.EnqueueWithoutRateLimitWithDelay(context.TODO(), "c", 3000)
+	q.EnqueueWithoutRateLimitWithDelay(context.TODO(), "d", 4000)
+	q.EnqueueWithoutRateLimitWithDelay(context.TODO(), "e", 5000)
+	checkConsistency(t, q)
+	t.Logf("%v", q)
+	q.EnqueueWithoutRateLimitWithDelay(context.TODO(), "d", 1000)
+	checkConsistency(t, q)
+	t.Logf("%v", q)
+	q.EnqueueWithoutRateLimitWithDelay(context.TODO(), "c", 1001)
+	checkConsistency(t, q)
+	t.Logf("%v", q)
+	q.EnqueueWithoutRateLimitWithDelay(context.TODO(), "e", 999)
+	checkConsistency(t, q)
+	t.Logf("%v", q)
+}
+
+type rateLimitWrapper struct {
+	addedMap     sync.Map
+	forgottenMap sync.Map
+	rl           workqueue.TypedRateLimiter[any]
+}
+
+func (r *rateLimitWrapper) When(item interface{}) time.Duration {
+	if _, ok := r.forgottenMap.Load(item); ok {
+		r.forgottenMap.Delete(item)
+		// Reset the added map
+		r.addedMap.Store(item, 1)
+	} else {
+		actual, loaded := r.addedMap.LoadOrStore(item, 1)
+		if loaded {
+			r.addedMap.Store(item, actual.(int)+1)
+		}
 	}
 
-	cancel()
-	group.Wait()
+	return r.rl.When(item)
+}
+
+func (r *rateLimitWrapper) Forget(item interface{}) {
+	r.forgottenMap.Store(item, struct{}{})
+	r.rl.Forget(item)
+}
+
+func (r *rateLimitWrapper) NumRequeues(item interface{}) int {
+	return r.rl.NumRequeues(item)
+}
+
+func TestRateLimiter(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 5000*time.Millisecond)
+	defer cancel()
+
+	syncMap := sync.Map{}
+	syncMap.Store("foo", 0)
+	syncMap.Store("bar", 0)
+	syncMap.Store("baz", 0)
+	syncMap.Store("quux", 0)
+
+	start := time.Now()
+	ratelimiter := &rateLimitWrapper{
+		rl: workqueue.NewTypedItemFastSlowRateLimiter[any](1*time.Millisecond, 100*time.Millisecond, 1),
+	}
+
+	q := New(ratelimiter, t.Name(), func(ctx context.Context, key string) error {
+		oldValue, _ := syncMap.Load(key)
+		syncMap.Store(key, oldValue.(int)+1)
+		if oldValue.(int) < 9 {
+			return errors.New("test")
+		}
+		return nil
+	}, nil)
+
+	enqueued := 0
+	syncMap.Range(func(key, value interface{}) bool {
+		enqueued++
+		q.Enqueue(context.TODO(), key.(string))
+		return true
+	})
+
+	assert.Assert(t, enqueued == 4)
+	go q.Run(ctx, 10)
+
+	incomplete := true
+	for incomplete {
+		time.Sleep(10 * time.Millisecond)
+		incomplete = false
+		// Wait for all items to finish processing.
+		syncMap.Range(func(key, value interface{}) bool {
+			if value.(int) < 10 {
+				incomplete = true
+			}
+			return true
+		})
+	}
+
+	// Make sure there were ~9 "slow" rate limits per item, and 1 fast
+	assert.Assert(t, time.Since(start) > 9*100*time.Millisecond)
+	// Make sure we didn't go off the deep end.
+	assert.Assert(t, time.Since(start) < 2*9*100*time.Millisecond)
+
+	// Make sure each item was seen. And Forgotten.
+	syncMap.Range(func(key, value interface{}) bool {
+		_, ok := ratelimiter.forgottenMap.Load(key)
+		assert.Assert(t, ok, "%s in forgotten map", key)
+		val, ok := ratelimiter.addedMap.Load(key)
+		assert.Assert(t, ok, "%s in added map", key)
+		assert.Assert(t, val == 10)
+		return true
+	})
+
+	q.lock.Lock()
+	defer q.lock.Unlock()
+	assert.Assert(t, len(q.itemsInQueue) == 0)
+	assert.Assert(t, len(q.itemsBeingProcessed) == 0)
+	assert.Assert(t, q.items.Len() == 0)
+
+}
+
+func TestQueueForgetInProgress(t *testing.T) {
+	t.Parallel()
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	var times int64
+	var q *Queue
+	q = New(workqueue.DefaultTypedItemBasedRateLimiter[any](), t.Name(), func(ctx context.Context, key string) error {
+		assert.Assert(t, is.Equal(key, "foo"))
+		atomic.AddInt64(&times, 1)
+		q.Forget(context.TODO(), key)
+		return errors.New("test")
+	}, nil)
+
+	q.EnqueueWithoutRateLimit(context.TODO(), "foo")
+	go q.Run(ctx, 1)
+	for !q.Empty() {
+		time.Sleep(100 * time.Millisecond)
+	}
+	assert.Assert(t, is.Equal(atomic.LoadInt64(&times), int64(1)))
+}
+
+func TestQueueForgetBeforeStart(t *testing.T) {
+	t.Parallel()
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	q := New(workqueue.DefaultTypedItemBasedRateLimiter[any](), t.Name(), func(ctx context.Context, key string) error {
+		panic("shouldn't be called")
+	}, nil)
+
+	q.EnqueueWithoutRateLimit(context.TODO(), "foo")
+	q.Forget(context.TODO(), "foo")
+	go q.Run(ctx, 1)
+	for !q.Empty() {
+		time.Sleep(100 * time.Millisecond)
+	}
+}
+
+func TestQueueMoveItem(t *testing.T) {
+	t.Parallel()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	q := New(workqueue.DefaultTypedItemBasedRateLimiter[any](), t.Name(), func(ctx context.Context, key string) error {
+		panic("shouldn't be called")
+	}, nil)
+	q.clock = nonmovingClock{}
+
+	q.insert(ctx, "foo", false, durationPtr(3000))
+	q.insert(ctx, "bar", false, durationPtr(2000))
+	q.insert(ctx, "baz", false, durationPtr(1000))
+	checkConsistency(t, q)
+	t.Log(q)
+
+	q.insert(ctx, "foo", false, durationPtr(2000))
+	checkConsistency(t, q)
+	t.Log(q)
+
+	q.insert(ctx, "foo", false, durationPtr(1999))
+	checkConsistency(t, q)
+	t.Log(q)
+
+	q.insert(ctx, "foo", false, durationPtr(999))
+	checkConsistency(t, q)
+	t.Log(q)
+}
+
+type nonmovingClock struct {
+}
+
+func (n nonmovingClock) Now() time.Time {
+	return time.Time{}
+}
+
+func (n nonmovingClock) Since(t time.Time) time.Duration {
+	return n.Now().Sub(t)
+}
+
+func (n nonmovingClock) After(d time.Duration) <-chan time.Time {
+	panic("implement me")
+}
+
+func (n nonmovingClock) NewTimer(d time.Duration) clock.Timer {
+	panic("implement me")
+}
+
+func (n nonmovingClock) Sleep(d time.Duration) {
+	panic("implement me")
+}
+
+func (n nonmovingClock) Tick(d time.Duration) <-chan time.Time {
+	panic("implement me")
 }

internal/test/e2e/framework/pod.go

@@ -12,7 +12,6 @@ import (
 	watchapi "k8s.io/apimachinery/pkg/watch"
 	"k8s.io/client-go/tools/cache"
 	"k8s.io/client-go/tools/watch"
-	podutil "k8s.io/kubernetes/pkg/api/v1/pod"
 )
 
 // CreateDummyPodObjectWithPrefix creates a dujmmy pod object using the specified prefix as the value of .metadata.generateName.
@@ -101,15 +100,25 @@ func (f *Framework) WaitUntilPodCondition(namespace, name string, fn watch.Condi
 func (f *Framework) WaitUntilPodReady(namespace, name string) (*corev1.Pod, error) {
 	return f.WaitUntilPodCondition(namespace, name, func(event watchapi.Event) (bool, error) {
 		pod := event.Object.(*corev1.Pod)
-		return pod.Status.Phase == corev1.PodRunning && podutil.IsPodReady(pod) && pod.Status.PodIP != "", nil
+		return pod.Status.Phase == corev1.PodRunning && IsPodReady(pod) && pod.Status.PodIP != "", nil
 	})
 }
 
+// IsPodReady returns true if a pod is ready.
+func IsPodReady(pod *corev1.Pod) bool {
+	for _, cond := range pod.Status.Conditions {
+		if cond.Type == corev1.PodReady && cond.Status == corev1.ConditionTrue {
+			return true
+		}
+	}
+	return false
+}
+
 // WaitUntilPodDeleted blocks until the pod with the specified name and namespace is deleted from apiserver.
 func (f *Framework) WaitUntilPodDeleted(namespace, name string) (*corev1.Pod, error) {
 	return f.WaitUntilPodCondition(namespace, name, func(event watchapi.Event) (bool, error) {
 		pod := event.Object.(*corev1.Pod)
-		return event.Type == watchapi.Deleted || pod.ObjectMeta.DeletionTimestamp != nil, nil
+		return event.Type == watchapi.Deleted || pod.DeletionTimestamp != nil, nil
 	})
 }
 

internal/test/e2e/framework/stats.go

@@ -3,10 +3,10 @@ package framework
 import (
 	"context"
 	"encoding/json"
-	"strconv"
 
+	api "github.com/virtual-kubelet/virtual-kubelet/node/api"
 	"k8s.io/apimachinery/pkg/util/net"
-	stats "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
+	stats "k8s.io/kubelet/pkg/apis/stats/v1alpha1"
 )
 
 // GetStatsSummary queries the /stats/summary endpoint of the virtual-kubelet and returns the Summary object obtained as a response.
@@ -18,7 +18,7 @@ func (f *Framework) GetStatsSummary(ctx context.Context) (*stats.Summary, error)
 		Namespace(f.Namespace).
 		Resource("pods").
 		SubResource("proxy").
-		Name(net.JoinSchemeNamePort("http", f.NodeName, strconv.Itoa(10255))).
+		Name(net.JoinSchemeNamePort("https", f.NodeName, "10250")).
 		Suffix("/stats/summary").DoRaw(ctx)
 	if err != nil {
 		return nil, err
@@ -30,3 +30,21 @@ func (f *Framework) GetStatsSummary(ctx context.Context) (*stats.Summary, error)
 	}
 	return res, nil
 }
+
+// GetStatsSummary queries the /metrics/resource endpoint of the virtual-kubelet and returns the Summary object obtained as a response.
+func (f *Framework) GetMetricsResource(ctx context.Context) ([]byte, error) {
+	// Query the /stats/summary endpoint.
+	b, err := f.KubeClient.CoreV1().
+		RESTClient().
+		Get().
+		Namespace(f.Namespace).
+		Resource("pods").
+		SubResource("proxy").
+		Name(net.JoinSchemeNamePort("https", f.NodeName, "10250")).
+		Suffix(api.MetricsResourceRouteSuffix).DoRaw(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	return b, nil
+}

internal/test/e2e/main_test.go

@@ -1,3 +1,4 @@
+//go:build e2e
 // +build e2e
 
 package e2e
@@ -48,7 +49,7 @@ func teardown() error {
 	return nil
 }
 
-//  Provider-specific shouldSkipTest function
+// Provider-specific shouldSkipTest function
 func shouldSkipTest(testName string) bool {
 	return false
 }

log/klogv2/klogv2_test.go

@@ -4,7 +4,7 @@
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
-//     http://www.apache.org/licenses/LICENSE-2.0
+//	http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,

log/slog/slog.go

@@ -0,0 +1,109 @@
+// Copyright © 2021 The virtual-kubelet authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Package slog implements a virtual-kubelet/log.Logger using slog as a backend
+// You can use this by creating a slog logger and calling `FromSlog(logger)`
+// If you want this to be the default logger for virtual-kubelet, set `log.L` to the value returned by `FromSlog(logger)`
+package slog
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+
+	"github.com/virtual-kubelet/virtual-kubelet/log"
+)
+
+// Ensure log.Logger is fully implemented during compile time.
+var _ log.Logger = (*adapter)(nil)
+
+// Create a custom logging level for the Fatal level as slog does not
+// have this level by default
+const LevelFatal = slog.Level(12)
+
+// adapter implements the `log.Logger` interface for slog
+type adapter struct {
+	inner *slog.Logger
+}
+
+// FromSlog creates a new `log.Logger` from a slog logger
+func FromSlog(logger *slog.Logger) log.Logger {
+	return &adapter{inner: logger}
+}
+
+func (l *adapter) Debug(args ...interface{}) {
+	msg := args[0].(string)
+	l.inner.Debug(msg)
+}
+
+func (l *adapter) Debugf(format string, args ...interface{}) {
+	formattedArgs := fmt.Sprintf(format, args...)
+	l.inner.Debug(formattedArgs)
+}
+
+func (l *adapter) Info(args ...interface{}) {
+	msg := args[0].(string)
+	l.inner.Info(msg)
+}
+
+func (l *adapter) Infof(format string, args ...interface{}) {
+	formattedArgs := fmt.Sprintf(format, args...)
+	l.inner.Info(formattedArgs)
+}
+
+func (l *adapter) Warn(args ...interface{}) {
+	msg := args[0].(string)
+	l.inner.Warn(msg)
+}
+
+func (l *adapter) Warnf(format string, args ...interface{}) {
+	formattedArgs := fmt.Sprintf(format, args...)
+	l.inner.Warn(formattedArgs)
+}
+
+func (l *adapter) Error(args ...interface{}) {
+	msg := args[0].(string)
+	l.inner.Error(msg)
+}
+
+func (l *adapter) Errorf(format string, args ...interface{}) {
+	formattedArgs := fmt.Sprintf(format, args...)
+	l.inner.Error(formattedArgs)
+}
+
+func (l *adapter) Fatal(args ...interface{}) {
+	msg := args[0].(string)
+	l.inner.Log(context.Background(), LevelFatal, msg)
+}
+
+func (l *adapter) Fatalf(format string, args ...interface{}) {
+	formattedArgs := fmt.Sprintf(format, args...)
+	l.inner.Log(context.Background(), LevelFatal, formattedArgs)
+}
+
+func (l *adapter) WithField(key string, val interface{}) log.Logger {
+	return &adapter{inner: l.inner.With(key, val)}
+}
+
+func (l *adapter) WithFields(f log.Fields) log.Logger {
+	logger := l.inner
+	for k, v := range f {
+		logger = logger.With(k, v)
+	}
+	return &adapter{inner: logger}
+}
+
+func (l *adapter) WithError(err error) log.Logger {
+	return &adapter{inner: l.inner.With("error", err)}
+}

node/api/attach.go

@@ -0,0 +1,148 @@
+// Copyright © 2017 The virtual-kubelet authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package api
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/gorilla/mux"
+	"github.com/virtual-kubelet/virtual-kubelet/errdefs"
+	"github.com/virtual-kubelet/virtual-kubelet/internal/kubernetes/remotecommand"
+	"k8s.io/apimachinery/pkg/types"
+	remoteutils "k8s.io/client-go/tools/remotecommand"
+)
+
+// ContainerAttachHandlerFunc defines the handler function used for "execing" into a
+// container in a pod.
+type ContainerAttachHandlerFunc func(ctx context.Context, namespace, podName, containerName string, attach AttachIO) error
+
+// HandleContainerAttach makes an http handler func from a Provider which execs a command in a pod's container
+// Note that this handler currently depends on gorrilla/mux to get url parts as variables.
+// TODO(@cpuguy83): don't force gorilla/mux on consumers of this function
+func HandleContainerAttach(h ContainerAttachHandlerFunc, opts ...ContainerExecHandlerOption) http.HandlerFunc {
+	if h == nil {
+		return NotImplemented
+	}
+
+	var cfg ContainerExecHandlerConfig
+	for _, o := range opts {
+		o(&cfg)
+	}
+
+	if cfg.StreamIdleTimeout == 0 {
+		cfg.StreamIdleTimeout = 30 * time.Second
+	}
+	if cfg.StreamCreationTimeout == 0 {
+		cfg.StreamCreationTimeout = 30 * time.Second
+	}
+
+	return handleError(func(w http.ResponseWriter, req *http.Request) error {
+		vars := mux.Vars(req)
+
+		namespace := vars["namespace"]
+		pod := vars["pod"]
+		container := vars["container"]
+
+		supportedStreamProtocols := strings.Split(req.Header.Get("X-Stream-Protocol-Version"), ",")
+
+		streamOpts, err := getExecOptions(req)
+		if err != nil {
+			return errdefs.AsInvalidInput(err)
+		}
+
+		ctx, cancel := context.WithCancel(req.Context())
+		defer cancel()
+
+		attach := &containerAttachContext{ctx: ctx, h: h, pod: pod, namespace: namespace, container: container}
+		remotecommand.ServeAttach(
+			w,
+			req,
+			attach,
+			"",
+			"",
+			container,
+			streamOpts,
+			cfg.StreamIdleTimeout,
+			cfg.StreamCreationTimeout,
+			supportedStreamProtocols,
+		)
+
+		return nil
+	})
+}
+
+type containerAttachContext struct {
+	h                         ContainerAttachHandlerFunc
+	namespace, pod, container string
+	ctx                       context.Context
+}
+
+// AttachToContainer Implements remotecommand.Attacher
+// This is called by remotecommand.ServeAttach
+func (c *containerAttachContext) AttachToContainer(
+	name string,
+	uid types.UID,
+	container string,
+	in io.Reader,
+	out, err io.WriteCloser,
+	tty bool,
+	resize <-chan remoteutils.TerminalSize,
+	timeout time.Duration,
+) error {
+
+	eio := &execIO{
+		tty:    tty,
+		stdin:  in,
+		stdout: out,
+		stderr: err,
+	}
+
+	if tty {
+		eio.chResize = make(chan TermSize)
+	}
+
+	ctx, cancel := context.WithCancel(c.ctx)
+	defer cancel()
+
+	if tty {
+		go func() {
+			send := func(s remoteutils.TerminalSize) bool {
+				select {
+				case eio.chResize <- TermSize{Width: s.Width, Height: s.Height}:
+					return false
+				case <-ctx.Done():
+					return true
+				}
+			}
+
+			for {
+				select {
+				case s := <-resize:
+					if send(s) {
+						return
+					}
+				case <-ctx.Done():
+					return
+				}
+			}
+		}()
+	}
+
+	return c.h(c.ctx, c.namespace, c.pod, c.container, eio)
+}

node/api/exec.go

@@ -27,7 +27,6 @@ import (
 	"github.com/virtual-kubelet/virtual-kubelet/internal/kubernetes/remotecommand"
 	"k8s.io/apimachinery/pkg/types"
 	remoteutils "k8s.io/client-go/tools/remotecommand"
-	api "k8s.io/kubernetes/pkg/apis/core"
 )
 
 // ContainerExecHandlerFunc defines the handler function used for "execing" into a
@@ -136,11 +135,18 @@ func HandleContainerExec(h ContainerExecHandlerFunc, opts ...ContainerExecHandle
 	})
 }
 
+const (
+	execTTYParam    = "tty"
+	execStdinParam  = "input"
+	execStdoutParam = "output"
+	execStderrParam = "error"
+)
+
 func getExecOptions(req *http.Request) (*remotecommand.Options, error) {
-	tty := req.FormValue(api.ExecTTYParam) == "1"
-	stdin := req.FormValue(api.ExecStdinParam) == "1"
-	stdout := req.FormValue(api.ExecStdoutParam) == "1"
-	stderr := req.FormValue(api.ExecStderrParam) == "1"
+	tty := req.FormValue(execTTYParam) == "1"
+	stdin := req.FormValue(execStdinParam) == "1"
+	stdout := req.FormValue(execStdoutParam) == "1"
+	stderr := req.FormValue(execStderrParam) == "1"
 	if tty && stderr {
 		return nil, errors.New("cannot exec with tty and stderr")
 	}
@@ -165,7 +171,17 @@ type containerExecContext struct {
 
 // ExecInContainer Implements remotecommand.Executor
 // This is called by remotecommand.ServeExec
-func (c *containerExecContext) ExecInContainer(name string, uid types.UID, container string, cmd []string, in io.Reader, out, err io.WriteCloser, tty bool, resize <-chan remoteutils.TerminalSize, timeout time.Duration) error {
+func (c *containerExecContext) ExecInContainer(
+	name string,
+	uid types.UID,
+	container string,
+	cmd []string,
+	in io.Reader,
+	out, err io.WriteCloser,
+	tty bool,
+	resize <-chan remoteutils.TerminalSize,
+	timeout time.Duration,
+) error {
 
 	eio := &execIO{
 		tty:    tty,

node/api/helpers.go

@@ -33,7 +33,9 @@ func handleError(f handlerFunc) http.HandlerFunc {
 
 		code := httpStatusCode(err)
 		w.WriteHeader(code)
-		io.WriteString(w, err.Error()) //nolint:errcheck
+		if _, err := io.WriteString(w, err.Error()); err != nil {
+			log.G(req.Context()).WithError(err).Error("error writing error response")
+		}
 		logger := log.G(req.Context()).WithError(err).WithField("httpStatusCode", code)
 
 		if code >= 500 {

node/api/metrics.go

@@ -0,0 +1,67 @@
+// Copyright © 2017 The virtual-kubelet authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package api
+
+import (
+	"bytes"
+	"context"
+	"net/http"
+
+	"github.com/pkg/errors"
+	dto "github.com/prometheus/client_model/go"
+	"github.com/prometheus/common/expfmt"
+)
+
+const (
+	PrometheusTextFormatContentType = "text/plain; version=0.0.4"
+)
+
+// PodMetricsResourceHandlerFunc defines the handler for getting pod metrics
+type PodMetricsResourceHandlerFunc func(context.Context) ([]*dto.MetricFamily, error)
+
+// HandlePodMetricsResource makes an HTTP handler for implementing the kubelet /metrics/resource endpoint
+func HandlePodMetricsResource(h PodMetricsResourceHandlerFunc) http.HandlerFunc {
+	if h == nil {
+		return NotImplemented
+	}
+	return handleError(func(w http.ResponseWriter, req *http.Request) error {
+		metrics, err := h(req.Context())
+		if err != nil {
+			if isCancelled(err) {
+				return err
+			}
+			return errors.Wrap(err, "error getting status from provider")
+		}
+
+		// Convert metrics to Prometheus text format.
+		var buffer bytes.Buffer
+		enc := expfmt.NewEncoder(&buffer, expfmt.NewFormat(expfmt.TypeTextPlain))
+		for _, mf := range metrics {
+			if err := enc.Encode(mf); err != nil {
+				return errors.Wrap(err, "could not convert metrics to prometheus text format")
+			}
+		}
+
+		// Set the response content type to "text/plain; version=0.0.4".
+		w.Header().Set("Content-Type", PrometheusTextFormatContentType)
+
+		// Write the metrics in Prometheus text format to the response writer.
+		if _, err := w.Write(buffer.Bytes()); err != nil {
+			return errors.Wrap(err, "could not write to client")
+		}
+
+		return nil
+	})
+}

node/api/metrics_test.go

@@ -0,0 +1,149 @@
+// Copyright © 2017 The virtual-kubelet authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package api_test
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/pkg/errors"
+	dto "github.com/prometheus/client_model/go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/virtual-kubelet/virtual-kubelet/node/api"
+	"google.golang.org/protobuf/proto"
+)
+
+const (
+	prometheusContentType = "text/plain; version=0.0.4"
+)
+
+func TestHandlePodMetricsResource(t *testing.T) {
+	testCases := []struct {
+		name               string
+		handler            api.PodMetricsResourceHandlerFunc
+		expectedStatusCode int
+		expectedError      error
+	}{
+		{
+			name: "Valid PodMetricsResourceHandlerFunc",
+			handler: func(_ context.Context) ([]*dto.MetricFamily, error) {
+				// Create the expected metrics.
+				cpuUsageMetric := &dto.MetricFamily{
+					Name:   proto.String("container_cpu_usage_seconds_total"),
+					Help:   proto.String("[ALPHA] Cumulative cpu time consumed by the container in core-seconds"),
+					Type:   dto.MetricType_GAUGE.Enum(),
+					Metric: []*dto.Metric{},
+				}
+				memoryUsageMetric := &dto.MetricFamily{
+					Name:   proto.String("container_memory_working_set_bytes"),
+					Help:   proto.String("[ALPHA] Current working set of the container in bytes"),
+					Type:   dto.MetricType_GAUGE.Enum(),
+					Metric: []*dto.Metric{},
+				}
+
+				// Add the sample metrics to the metric families.
+				cpuUsageMetric.Metric = append(cpuUsageMetric.Metric, createSampleMetric(
+					map[string]string{
+						"container": "simple-hello-world-container",
+						"namespace": "k8se-apps",
+						"pod":       "test-pod--zruwatj-86454fdc54-2wwpw",
+					},
+					0.1104636, 1680536423102,
+				))
+				cpuUsageMetric.Metric = append(cpuUsageMetric.Metric, createSampleMetric(
+					map[string]string{
+						"container": "simple-hello-world-container",
+						"namespace": "k8se-apps",
+						"pod":       "test-pod--zruwatj-86454fdc54-4mzd4",
+					},
+					0.11322, 1680536423103,
+				))
+				memoryUsageMetric.Metric = append(memoryUsageMetric.Metric, createSampleMetric(
+					map[string]string{
+						"container": "simple-hello-world-container",
+						"namespace": "k8se-apps",
+						"pod":       "test-pod--zruwatj-86454fdc54-2wwpw",
+					},
+					2.3277568e+07, 1680536423102,
+				))
+				memoryUsageMetric.Metric = append(memoryUsageMetric.Metric, createSampleMetric(
+					map[string]string{
+						"container": "simple-hello-world-container",
+						"namespace": "k8se-apps",
+						"pod":       "test-pod--zruwatj-86454fdc54-4mzd4",
+					},
+					2.2450176e+07, 1680536423104,
+				))
+
+				return []*dto.MetricFamily{cpuUsageMetric, memoryUsageMetric}, nil
+			},
+			expectedStatusCode: http.StatusOK,
+		},
+		{
+			name:               "Nil PodMetricsResourceHandlerFunc",
+			handler:            nil,
+			expectedStatusCode: http.StatusNotImplemented,
+		},
+		{
+			name: "Error in PodMetricsResourceHandlerFunc",
+			handler: func(_ context.Context) ([]*dto.MetricFamily, error) {
+				return nil, errors.New("test error")
+			},
+			expectedStatusCode: http.StatusInternalServerError,
+			expectedError:      errors.New("error getting status from provider: test error"),
+		},
+		// Add more test cases as needed
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			h := api.HandlePodMetricsResource(tc.handler)
+			require.NotNil(t, h)
+
+			rr := httptest.NewRecorder()
+			req, err := http.NewRequest("GET", "/metrics/resource", nil)
+			require.NoError(t, err)
+
+			h.ServeHTTP(rr, req)
+
+			assert.Equal(t, tc.expectedStatusCode, rr.Code)
+
+			if tc.expectedError != nil {
+				bodyBytes, err := io.ReadAll(rr.Body)
+				require.NoError(t, err)
+				assert.Contains(t, string(bodyBytes), tc.expectedError.Error())
+			} else if tc.expectedStatusCode == http.StatusOK {
+				contentType := rr.Header().Get("Content-Type")
+				assert.Equal(t, prometheusContentType, contentType)
+			}
+		})
+	}
+}
+
+func createSampleMetric(labels map[string]string, value float64, timestamp int64) *dto.Metric {
+	labelPairs := []*dto.LabelPair{}
+	for k, v := range labels {
+		labelPairs = append(labelPairs, &dto.LabelPair{
+			Name:  proto.String(k),
+			Value: proto.String(v),
+		})
+	}
+
+	return &dto.Metric{Label: labelPairs, Gauge: &dto.Gauge{Value: &value}, TimestampMs: &timestamp}
+}