Add trailing newline to stats response

This was just an annoyance when working with the API from a terminal.

Signed-off-by: Brian Goff <cpuguy83@gmail.com>

.github/workflows/ci.yml

@@ -9,97 +9,102 @@ on:
     branches: [master]
   pull_request:
 
+permissions:
+  contents: read
+
 env:
-  GO_VERSION: "1.18"
+  GO_VERSION: "1.23"
 
 jobs:
   lint:
     name: Lint
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
+    runs-on: ubuntu-22.04
+    timeout-minutes: 20
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
-      - uses: actions/setup-go@v3
+        uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
         with:
           go-version: ${{ env.GO_VERSION }}
-      - uses: actions/checkout@v3
-      - uses: golangci/golangci-lint-action@v3
+          cache: false
+      - uses: actions/checkout@v4
+      - uses: golangci/golangci-lint-action@v8
         with:
-          version: v1.48.0
-          args: --timeout=5m
+          version: v2.1
+          args: --timeout=15m --config=.golangci.yml
+          skip-cache: true
 
   unit-tests:
     name: Unit Tests
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
+    runs-on: ubuntu-22.04
+    timeout-minutes: 20
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
-      - uses: actions/setup-go@v3
+        uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
         with:
           go-version: ${{ env.GO_VERSION }}
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Run Tests
         run: make test
 
   env-tests:
     name: Envtest Tests
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     timeout-minutes: 10
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
-      - uses: actions/setup-go@v3
+        uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
         with:
           go-version: ${{ env.GO_VERSION }}
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Run Tests
         run: make envtest
 
   e2e:
     name: E2E
     runs-on: ubuntu-22.04
-    timeout-minutes: 10
+    timeout-minutes: 20
     env:
       CHANGE_MINIKUBE_NONE_USER: true
-      KUBERNETES_VERSION: v1.20.1
-      MINIKUBE_HOME: /home/circleci
-      MINIKUBE_VERSION: v1.16.0
+      KUBERNETES_VERSION: v1.31
+      MINIKUBE_HOME: /home/runner
+      MINIKUBE_VERSION: v1.34.0
       MINIKUBE_WANTUPDATENOTIFICATION: false
       MINIKUBE_WANTREPORTERRORPROMPT: false
-      SKAFFOLD_VERSION: v1.17.2
+      SKAFFOLD_VERSION: v2.13.2
       GO111MODULE: "on"
 
     steps:
-      - uses: actions/setup-go@v3
+      - uses: actions/setup-go@v5
         with:
           go-version: ${{ env.GO_VERSION }}
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Install Skaffold
         run: |
-          curl -Lo skaffold https://storage.googleapis.com/skaffold/releases/${SKAFFOLD_VERSION}/skaffold-linux-amd64
+          curl -sLo skaffold https://storage.googleapis.com/skaffold/releases/${SKAFFOLD_VERSION}/skaffold-linux-amd64
           chmod +x skaffold
           sudo mv skaffold /usr/local/bin/
           echo /usr/local/bin >> $GITHUB_PATH
       - name: Install Minikube dependencies
         run: |
           sudo apt-get update && sudo apt-get install -y apt-transport-https curl
-          curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -
-          cat <<EOF | sudo tee /etc/apt/sources.list.d/kubernetes.list
-          deb https://apt.kubernetes.io/ kubernetes-xenial main
-          EOF
+          echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/${KUBERNETES_VERSION}/deb/ /" | sudo tee /etc/apt/sources.list.d/kubernetes.list
+          curl -fsSL https://pkgs.k8s.io/core:/stable:/${KUBERNETES_VERSION}/deb/Release.key | sudo gpg --no-tty --yes --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
           sudo apt-get update
-          sudo apt-get install -y kubelet # systemd unit is disabled
+          sudo apt-get remove -y containerd.io containerd
+          sudo apt-get install -y kubectl docker.io
       - name: Install Minikube
         run: |
-          curl -Lo minikube https://storage.googleapis.com/minikube/releases/${MINIKUBE_VERSION}/minikube-linux-amd64
+          curl -sLo minikube https://storage.googleapis.com/minikube/releases/${MINIKUBE_VERSION}/minikube-linux-amd64
           chmod +x minikube
           sudo mv minikube /usr/local/bin/
       - name: Start Minikube
         run: |
-          sudo -E PATH=$PATH minikube start --vm-driver=none --cpus 2 --memory 2048 --kubernetes-version=${KUBERNETES_VERSION}
+          sudo usermod -aG docker $USER && newgrp docker
+          minikube start --vm-driver=docker --cpus 2 --memory 2048 --kubernetes-version=${KUBERNETES_VERSION}
       - name: Wait for Minikube
         run: |
           JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}';

.github/workflows/codeql-analysis.yml

@@ -27,11 +27,11 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v2
+        uses: github/codeql-action/init@v3
         with:
           languages: ${{ matrix.language }}
           # If you wish to specify custom queries, you can do so here or in a config file.
@@ -42,7 +42,7 @@ jobs:
       # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
       # If this step fails, then you should remove it and run the build manually (see below)
       - name: Autobuild
-        uses: github/codeql-action/autobuild@v2
+        uses: github/codeql-action/autobuild@v3
 
       # ℹ️ Command-line programs to run using the OS shell.
       # 📚 https://git.io/JvXDl
@@ -56,4 +56,4 @@ jobs:
       #   make release
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v2
+        uses: github/codeql-action/analyze@v3

.gitignore

@@ -30,6 +30,9 @@ credentials.json
 # Test loganalytics file
 loganalytics.json
 
+# Envtest
+.envtest/
+
 # VS Code files
 .vscode/
 
@@ -41,3 +44,5 @@ loganalytics.json
 **/terraform-provider-kubernetes
 **/*.tfstate*
 debug
+
+vendor/

.golangci.bck.yml

@@ -0,0 +1,34 @@
+issues:
+  exclude-use-default: false
+  exclude:
+    # EXC0001 errcheck: Almost all programs ignore errors on these functions and in most cases it's ok
+    - Error return value of .((os\.)?std(out|err)\..*|.*Close|.*Flush|os\.Remove(All)?|.*print(f|ln)?|os\.(Un)?Setenv). (is not checked|Errors unhandled)
+
+  exclude-dirs:
+    # This directory contains copy code from upstream kubernetes/kubernetes, skip it.
+    - internal/kubernetes
+    # This is mostly copied from upstream, rather than fixing that code here just ignore the errors.
+    - internal/podutils
+
+linters:
+  enable:
+    - errcheck
+    - staticcheck
+    - unconvert
+    - gofmt
+    - goimports
+    - ineffassign
+    - govet
+    - unused
+    - misspell
+    - gosec
+    - copyloopvar # Checks for pointers to enclosing loop variables
+    - tenv # Detects using os.Setenv instead of t.Setenv since Go 1.17
+    - lll
+
+linters-settings:
+  gosec:
+    excludes:
+      - G304 # Potential file inclusion via variable
+  lll:
+    line-length: 200

.golangci.yml

@@ -1,39 +1,51 @@
-linter-settings:
-  lll:
-    line-length: 200
-
-timeout: 10m
-
-run:
-  skip-dirs:
-    # This directory contains copy code from upstream kubernetes/kubernetes, skip it.
-    - internal/kubernetes
-    # This is mostly copied from upstream, rather than fixing that code here just ignore the errors.
-    - internal/podutils
-
+version: "2"
 linters:
   enable:
-    - errcheck
-    - structcheck
-    - varcheck
-    - staticcheck
+    - copyloopvar
+    - gosec
+    - lll
+    - misspell
     - unconvert
+  settings:
+    gosec:
+      excludes:
+        - G304
+    lll:
+      line-length: 200
+  exclusions:
+    generated: lax
+    rules:
+      - path: (.+)\.go$
+        text: Error return value of .((os\.)?std(out|err)\..*|.*Close|.*Flush|os\.Remove(All)?|.*print(f|ln)?|os\.(Un)?Setenv). (is not checked|Errors unhandled)
+    paths:
+      - internal/kubernetes
+      - internal/podutils
+      - third_party$
+      - builtin$
+      - examples$
+formatters:
+  enable:
     - gofmt
     - goimports
-    - ineffassign
-    - vet
-    - unused
-    - misspell
-    - gosec
-    - exportloopref # Checks for pointers to enclosing loop variables
-    - tenv # Detects using os.Setenv instead of t.Setenv since Go 1.17
-
-linters-settings:
-  gosec:
-    excludes:
-      - G304
+  exclusions:
+    generated: lax
+    paths:
+      - internal/kubernetes
+      - internal/podutils
+      - third_party$
+      - builtin$
+      - examples$
 issues:
-  exclude-use-default: false
-  exclude:
-    # EXC0001 errcheck: Almost all programs ignore errors on these functions and in most cases it's ok
-    - Error return value of .((os\.)?std(out|err)\..*|.*Close|.*Flush|os\.Remove(All)?|.*print(f|ln)?|os\.(Un)?Setenv). (is not checked|Errors unhandled)
+  # Maximum issues count per one linter.
+  # Set to 0 to disable.
+  # Default: 50
+  max-issues-per-linter: 0
+
+  # Maximum count of issues with the same text.
+  # Set to 0 to disable.
+  # Default: 3
+  max-same-issues: 0
+
+  # Make issues output unique by line.
+  # Default: true
+  uniq-by-line: false

Dockerfile

@@ -1,6 +1,6 @@
 ARG GOLANG_CI_LINT_VERSION
 
-FROM golang:1.18 as builder
+FROM golang:1.23 as builder
 ENV PATH /go/bin:/usr/local/go/bin:$PATH
 ENV GOPATH /go
 COPY . /go/src/github.com/virtual-kubelet/virtual-kubelet

Makefile

@@ -165,26 +165,25 @@ authors:
 	rm -f NEWAUTHORS
 	rm -f GITAUTHORS
 
-checksums_2.3.1.txt:
-	curl -o checksums_2.3.1.txt -L https://github.com/kubernetes-sigs/kubebuilder/releases/download/v2.3.1/checksums.txt
-kubebuilder_2.3.1_${TEST_OS}_${TEST_ARCH}.tar.gz:
-	curl -C - -O -L https://github.com/kubernetes-sigs/kubebuilder/releases/download/v2.3.1/kubebuilder_2.3.1_${TEST_OS}_${TEST_ARCH}.tar.gz
-kubebuilder_2.3.1_${TEST_OS}_${TEST_ARCH}: kubebuilder_2.3.1_${TEST_OS}_${TEST_ARCH}.tar.gz checksums_2.3.1.txt
-	sha256sum -c --ignore-missing checksums_2.3.1.txt
-	tar -xvf kubebuilder_2.3.1_${TEST_OS}_${TEST_ARCH}.tar.gz
+SETUP_ENVTEST_VERSION ?= v0.0.0-20250604165838-d6126d850224
+ENVTEST_K8S_VERSION := 1.31.x
+
+ENVTEST ?= go run sigs.k8s.io/controller-runtime/tools/setup-envtest@$(SETUP_ENVTEST_VERSION)
+ENVTEST_DIR ?= $(shell pwd)/.envtest
+export KUBEBUILDER_ASSETS ?= $(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(ENVTEST_DIR) -p path)
 
 .PHONY: envtest
-envtest: kubebuilder_2.3.1_${TEST_OS}_${TEST_ARCH}
+envtest:
 	# You can add klog flags for debugging, like: -klog.v=10 -klog.logtostderr
 	# klogv2 flags just wraps our existing logrus.
-	KUBEBUILDER_ASSETS=$(PWD)/kubebuilder_2.3.1_${TEST_OS}_${TEST_ARCH}/bin $(GOTEST) -run=TestEnvtest ./node -envtest=true
+	$(GOTEST) -run=TestEnvtest ./node -envtest=true
 
 .PHONY: fmt
 fmt:
 	goimports -w $(shell go list -f '{{.Dir}}' ./...)
 
 
-export GOLANG_CI_LINT_VERSION ?= v1.48.0
+export GOLANG_CI_LINT_VERSION ?= v1.49.0
 DOCKER_BUILD ?= docker buildx build
 
 .PHONY: lint

Makefile.e2e

@@ -1,7 +1,17 @@
+
+# skaffold checks for kubectl context
+# For minikube, docker-for-desktop and docker-desktop the context matches above names
+# If one wants to use kind, kind gives context names based on the cluster-name
+# But as of now they match the following syntax: kind-*
+# The first check verifies that this is a kind kubernetes context
+# Second check verifies the other ones
 .PHONY: skaffold.validate
 skaffold.validate: kubectl_context := $(shell kubectl config current-context)
 skaffold.validate:
-	@if [[ ! "minikube,docker-for-desktop,docker-desktop" =~ .*"$(kubectl_context)".* ]]; then \
+
+	@if [[ "$(kubectl_context)" =~ .*"kind".* ]]; then \
+		true; \
+	elif [[ ! "minikube,docker-for-desktop,docker-desktop" =~ .*"$(kubectl_context)".* ]]; then \
 		echo current-context is [$(kubectl_context)]. Must be one of [minikube,docker-for-desktop,docker-desktop]; \
 		false; \
 	fi

README.md

@@ -1,5 +1,7 @@
 # Virtual Kubelet
 
+[![Go Reference](https://pkg.go.dev/badge/github.com/virtual-kubelet/virtual-kubelet.svg)](https://pkg.go.dev/github.com/virtual-kubelet/virtual-kubelet)
+
 Virtual Kubelet is an open source [Kubernetes kubelet](https://kubernetes.io/docs/reference/generated/kubelet/)
 implementation that masquerades as a kubelet for the purposes of connecting Kubernetes to other APIs.
 This allows the nodes to be backed by other services like ACI, AWS Fargate, [IoT Edge](https://github.com/Azure/iot-edge-virtual-kubelet-provider), [Tensile Kube](https://github.com/virtual-kubelet/tensile-kube) etc. The primary scenario for VK is enabling the extension of the Kubernetes API into serverless container platforms like ACI and Fargate, though we are open to others. However, it should be noted that VK is explicitly not intended to be an alternative to Kubernetes federation.
@@ -23,6 +25,7 @@ The best description is "Kubernetes API on top, programmable back."
     + [AWS Fargate Provider](#aws-fargate-provider)
     + [Elotl Kip](#elotl-kip)
 	+ [HashiCorp Nomad](#hashicorp-nomad-provider)
+    + [InterLink](#interlink-provider)
     + [Liqo](#liqo-provider)
     + [OpenStack Zun](#openstack-zun-provider)
     + [Tensile Kube Provider](#tensile-kube-provider)
@@ -135,6 +138,14 @@ would on a Kubernetes node.
 
 For detailed instructions, follow the guide [here](https://github.com/virtual-kubelet/nomad/blob/master/README.md).
 
+### Interlink Provider
+
+[interLink](https://intertwin-eu.github.io/interLink/) provides an abstraction for the execution of a Kubernetes pod on any remote resource that has the capability to manage a container's execution lifecycle. The use cases that drove the initial development of the tool are the Slurm-powered HPC centers, regardless the plugin based design is enabling several additional use cases to provide Kubernetes-API based access to infrastracture that cannot host a Kubelet processes. 
+InterLink is a Virtual Kubelet provider that can manage container lifecycle through a well defined API specification, allowing for any resource provider to be integrated with a simple http server and a handful of methods.
+In other words, this is an attempt to streamline the process of creating custom Virtual Kubelet providers, avoiding the need for any resource provider to implement its own version of a Kubelet workflow, which would require having some domain expertise in the Kubernetes internals.
+
+For detailed instruction, follow the guide [here](https://intertwin-eu.github.io/interLink/docs/category/guides).
+
 ### Liqo Provider
 
 [Liqo](https://liqo.io) implements a provider for Virtual Kubelet designed to transparently offload pods and services to "peered" Kubernetes remote cluster. Liqo is capable of discovering neighbor clusters (using DNS, mDNS) and "peer" with them, or in other words, establish a relationship to share part of the cluster resources. When a cluster has established a peering, a new instance of the Liqo Virtual Kubelet is spawned to seamlessly extend the capacity of the cluster, by providing an abstraction of the resources of the remote cluster. The provider combined with the Liqo network fabric extends the cluster networking by enabling Pod-to-Pod traffic and multi-cluster east-west services, supporting endpoints on both clusters.
@@ -169,12 +180,12 @@ performing the neccessary actions.
 
 There are 3 main interfaces:
 
-#### PodLifecylceHandler
+#### PodLifecycleHandler
 
 When pods are created, updated, or deleted from Kubernetes, these methods are
 called to handle those actions.
 
-[godoc#PodLifecylceHandler](https://godoc.org/github.com/virtual-kubelet/virtual-kubelet/node#PodLifecycleHandler)
+[godoc#PodLifecycleHandler](https://godoc.org/github.com/virtual-kubelet/virtual-kubelet/node#PodLifecycleHandler)
 
 ```go
 type PodLifecycleHandler interface {
@@ -305,7 +316,7 @@ Enable the ServiceNodeExclusion flag, by modifying the Controller Manager manife
 Virtual Kubelet follows the [CNCF Code of Conduct](https://github.com/cncf/foundation/blob/master/code-of-conduct.md).
 Sign the [CNCF CLA](https://github.com/kubernetes/community/blob/master/CLA.md) to be able to make Pull Requests to this repo.
 
-Monthly Virtual Kubelet Office Hours are held at 10am PST on the last Thursday of every month in this [zoom meeting room](https://zoom.us/j/94701509915). Check out the calendar [here](https://calendar.google.com/calendar?cid=bjRtbGMxYWNtNXR0NXQ1a2hqZmRkNTRncGNAZ3JvdXAuY2FsZW5kYXIuZ29vZ2xlLmNvbQ).
+Monthly Virtual Kubelet Office Hours are held at 10am PST on the second Thursday of every month in this [zoom meeting room](https://zoom.us/j/94701509915). Check out the calendar [here](https://calendar.google.com/calendar/embed?src=b119ced62134053de07d6c261b50d21ebde0da54f4163f5771b60ecf906e8b90%40group.calendar.google.com&ctz=America%2FLos_Angeles).
 
 Our google drive with design specifications and meeting notes are [here](https://drive.google.com/drive/folders/19Ndu11WBCCBDowo9CrrGUHoIfd2L8Ueg?usp=sharing).
 

cmd/virtual-kubelet/internal/commands/root/http.go

@@ -16,31 +16,21 @@ package root
 
 import (
 	"fmt"
-	"os"
 	"time"
 )
 
 type apiServerConfig struct {
-	CertPath              string
-	KeyPath               string
-	CACertPath            string
 	Addr                  string
 	MetricsAddr           string
 	StreamIdleTimeout     time.Duration
 	StreamCreationTimeout time.Duration
 }
 
-func getAPIConfig(c Opts) (*apiServerConfig, error) {
-	config := apiServerConfig{
-		CertPath:   os.Getenv("APISERVER_CERT_LOCATION"),
-		KeyPath:    os.Getenv("APISERVER_KEY_LOCATION"),
-		CACertPath: os.Getenv("APISERVER_CA_CERT_LOCATION"),
+func getAPIConfig(c Opts) apiServerConfig {
+	return apiServerConfig{
+		Addr:                  fmt.Sprintf(":%d", c.ListenPort),
+		MetricsAddr:           c.MetricsAddr,
+		StreamIdleTimeout:     c.StreamIdleTimeout,
+		StreamCreationTimeout: c.StreamCreationTimeout,
 	}
-
-	config.Addr = fmt.Sprintf(":%d", c.ListenPort)
-	config.MetricsAddr = c.MetricsAddr
-	config.StreamIdleTimeout = c.StreamIdleTimeout
-	config.StreamCreationTimeout = c.StreamCreationTimeout
-
-	return &config, nil
 }

cmd/virtual-kubelet/internal/commands/root/root.go

@@ -16,8 +16,6 @@ package root
 
 import (
 	"context"
-	"crypto/tls"
-	"net/http"
 	"os"
 	"runtime"
 
@@ -28,10 +26,8 @@ import (
 	"github.com/virtual-kubelet/virtual-kubelet/internal/manager"
 	"github.com/virtual-kubelet/virtual-kubelet/log"
 	"github.com/virtual-kubelet/virtual-kubelet/node"
-	"github.com/virtual-kubelet/virtual-kubelet/node/api"
 	"github.com/virtual-kubelet/virtual-kubelet/node/nodeutil"
 	corev1 "k8s.io/api/core/v1"
-	"k8s.io/apiserver/pkg/server/dynamiccertificates"
 )
 
 // NewCommand creates a new top-level command.
@@ -73,7 +69,6 @@ func runRootCommand(ctx context.Context, s *provider.Store, c Opts) error {
 		}
 	}
 
-	mux := http.NewServeMux()
 	newProvider := func(cfg nodeutil.ProviderConfig) (nodeutil.Provider, node.NodeProvider, error) {
 		rm, err := manager.NewResourceManager(cfg.Pods, cfg.Secrets, cfg.ConfigMaps, cfg.Services)
 		if err != nil {
@@ -102,14 +97,9 @@ func runRootCommand(ctx context.Context, s *provider.Store, c Opts) error {
 		return p, nil, nil
 	}
 
-	apiConfig, err := getAPIConfig(c)
-	if err != nil {
-		return err
-	}
-
+	apiConfig := getAPIConfig(c)
 	cm, err := nodeutil.NewNode(c.NodeName, newProvider, func(cfg *nodeutil.NodeConfig) error {
 		cfg.KubeconfigPath = c.KubeConfigPath
-		cfg.Handler = mux
 		cfg.InformerResyncPeriod = c.InformerResyncPeriod
 
 		if taint != nil {
@@ -127,12 +117,7 @@ func runRootCommand(ctx context.Context, s *provider.Store, c Opts) error {
 
 		return nil
 	},
-		setAuth(c.NodeName, apiConfig),
-		nodeutil.WithTLSConfig(
-			nodeutil.WithKeyPairFromPath(apiConfig.CertPath, apiConfig.KeyPath),
-			maybeCA(apiConfig.CACertPath),
-		),
-		nodeutil.AttachProviderRoutes(mux),
+		nodeutil.WithBootstrapFromRestConfig(),
 	)
 	if err != nil {
 		return err
@@ -171,32 +156,3 @@ func runRootCommand(ctx context.Context, s *provider.Store, c Opts) error {
 	}
 	return nil
 }
-
-func setAuth(node string, apiCfg *apiServerConfig) nodeutil.NodeOpt {
-	if apiCfg.CACertPath == "" {
-		return func(cfg *nodeutil.NodeConfig) error {
-			cfg.Handler = api.InstrumentHandler(nodeutil.WithAuth(nodeutil.NoAuth(), cfg.Handler))
-			return nil
-		}
-	}
-
-	return func(cfg *nodeutil.NodeConfig) error {
-		auth, err := nodeutil.WebhookAuth(cfg.Client, node, func(cfg *nodeutil.WebhookAuthConfig) error {
-			var err error
-			cfg.AuthnConfig.ClientCertificateCAContentProvider, err = dynamiccertificates.NewDynamicCAContentFromFile("ca-cert-bundle", apiCfg.CACertPath)
-			return err
-		})
-		if err != nil {
-			return err
-		}
-		cfg.Handler = api.InstrumentHandler(nodeutil.WithAuth(auth, cfg.Handler))
-		return nil
-	}
-}
-
-func maybeCA(p string) func(*tls.Config) error {
-	if p == "" {
-		return func(*tls.Config) error { return nil }
-	}
-	return nodeutil.WithCAFromPath(p)
-}

cmd/virtual-kubelet/internal/provider/mock/mock.go

@@ -10,14 +10,15 @@ import (
 	"strings"
 	"time"
 
+	dto "github.com/prometheus/client_model/go"
 	"github.com/virtual-kubelet/virtual-kubelet/errdefs"
 	"github.com/virtual-kubelet/virtual-kubelet/log"
 	"github.com/virtual-kubelet/virtual-kubelet/node/api"
-	stats "github.com/virtual-kubelet/virtual-kubelet/node/api/statsv1alpha1"
 	"github.com/virtual-kubelet/virtual-kubelet/trace"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	stats "k8s.io/kubelet/pkg/apis/stats/v1alpha1"
 )
 
 const (
@@ -42,7 +43,7 @@ var (
 */
 
 // MockProvider implements the virtual-kubelet provider interface and stores pods in memory.
-type MockProvider struct { //nolint:golint
+type MockProvider struct {
 	nodeName           string
 	operatingSystem    string
 	internalIP         string
@@ -54,10 +55,12 @@ type MockProvider struct { //nolint:golint
 }
 
 // MockConfig contains a mock virtual-kubelet's configurable parameters.
-type MockConfig struct { //nolint:golint
-	CPU    string `json:"cpu,omitempty"`
-	Memory string `json:"memory,omitempty"`
-	Pods   string `json:"pods,omitempty"`
+type MockConfig struct {
+	CPU        string            `json:"cpu,omitempty"`
+	Memory     string            `json:"memory,omitempty"`
+	Pods       string            `json:"pods,omitempty"`
+	Others     map[string]string `json:"others,omitempty"`
+	ProviderID string            `json:"providerID,omitempty"`
 }
 
 // NewMockProviderMockConfig creates a new MockV0Provider. Mock legacy provider does not implement the new asynchronous podnotifier interface
@@ -99,7 +102,7 @@ func NewMockProvider(providerConfig, nodeName, operatingSystem string, internalI
 func loadConfig(providerConfig, nodeName string) (config MockConfig, err error) {
 	data, err := os.ReadFile(providerConfig)
 	if err != nil {
-		return config, err
+		return config, fmt.Errorf("error reaeding provider config: %w", err)
 	}
 	configMap := map[string]MockConfig{}
 	err = json.Unmarshal(data, &configMap)
@@ -120,13 +123,18 @@ func loadConfig(providerConfig, nodeName string) (config MockConfig, err error)
 	}
 
 	if _, err = resource.ParseQuantity(config.CPU); err != nil {
-		return config, fmt.Errorf("Invalid CPU value %v", config.CPU)
+		return config, fmt.Errorf("invalid CPU value %v", config.CPU)
 	}
 	if _, err = resource.ParseQuantity(config.Memory); err != nil {
-		return config, fmt.Errorf("Invalid memory value %v", config.Memory)
+		return config, fmt.Errorf("invalid memory value %v", config.Memory)
 	}
 	if _, err = resource.ParseQuantity(config.Pods); err != nil {
-		return config, fmt.Errorf("Invalid pods value %v", config.Pods)
+		return config, fmt.Errorf("invalid pods value %v", config.Pods)
+	}
+	for _, v := range config.Others {
+		if _, err = resource.ParseQuantity(v); err != nil {
+			return config, fmt.Errorf("invalid other value %v", v)
+		}
 	}
 	return config, nil
 }
@@ -293,6 +301,19 @@ func (p *MockProvider) RunInContainer(ctx context.Context, namespace, name, cont
 	return nil
 }
 
+// AttachToContainer attaches to the executing process of a container in the pod, copying data
+// between in/out/err and the container's stdin/stdout/stderr.
+func (p *MockProvider) AttachToContainer(ctx context.Context, namespace, name, container string, attach api.AttachIO) error {
+	log.G(ctx).Infof("receive AttachToContainer %q", container)
+	return nil
+}
+
+// PortForward forwards a local port to a port on the pod
+func (p *MockProvider) PortForward(ctx context.Context, namespace, pod string, port int32, stream io.ReadWriteCloser) error {
+	log.G(ctx).Infof("receive PortForward %q", pod)
+	return nil
+}
+
 // GetPodStatus returns the status of a pod by name that is "running".
 // returns nil if a pod by that name is not found.
 func (p *MockProvider) GetPodStatus(ctx context.Context, namespace, name string) (*v1.PodStatus, error) {
@@ -328,10 +349,13 @@ func (p *MockProvider) GetPods(ctx context.Context) ([]*v1.Pod, error) {
 	return pods, nil
 }
 
-func (p *MockProvider) ConfigureNode(ctx context.Context, n *v1.Node) { //nolint:golint
+func (p *MockProvider) ConfigureNode(ctx context.Context, n *v1.Node) {
 	ctx, span := trace.StartSpan(ctx, "mock.ConfigureNode") //nolint:staticcheck,ineffassign
 	defer span.End()
 
+	if p.config.ProviderID != "" {
+		n.Spec.ProviderID = p.config.ProviderID
+	}
 	n.Status.Capacity = p.capacity()
 	n.Status.Allocatable = p.capacity()
 	n.Status.Conditions = p.nodeConditions()
@@ -343,17 +367,21 @@ func (p *MockProvider) ConfigureNode(ctx context.Context, n *v1.Node) { //nolint
 	}
 	n.Status.NodeInfo.OperatingSystem = os
 	n.Status.NodeInfo.Architecture = "amd64"
-	n.ObjectMeta.Labels["alpha.service-controller.kubernetes.io/exclude-balancer"] = "true"
-	n.ObjectMeta.Labels["node.kubernetes.io/exclude-from-external-load-balancers"] = "true"
+	n.Labels["alpha.service-controller.kubernetes.io/exclude-balancer"] = "true"
+	n.Labels["node.kubernetes.io/exclude-from-external-load-balancers"] = "true"
 }
 
 // Capacity returns a resource list containing the capacity limits.
 func (p *MockProvider) capacity() v1.ResourceList {
-	return v1.ResourceList{
+	rl := v1.ResourceList{
 		"cpu":    resource.MustParse(p.config.CPU),
 		"memory": resource.MustParse(p.config.Memory),
 		"pods":   resource.MustParse(p.config.Pods),
 	}
+	for k, v := range p.config.Others {
+		rl[v1.ResourceName(k)] = resource.MustParse(v)
+	}
+	return rl
 }
 
 // NodeConditions returns a list of conditions (Ready, OutOfDisk, etc), for updates to the node status
@@ -508,6 +536,129 @@ func (p *MockProvider) GetStatsSummary(ctx context.Context) (*stats.Summary, err
 	return res, nil
 }
 
+func (p *MockProvider) generateMockMetrics(metricsMap map[string][]*dto.Metric, resourceType string, label []*dto.LabelPair) map[string][]*dto.Metric {
+	var (
+		cpuMetricSuffix    = "_cpu_usage_seconds_total"
+		memoryMetricSuffix = "_memory_working_set_bytes"
+		dummyValue         = float64(100)
+	)
+
+	if metricsMap == nil {
+		metricsMap = map[string][]*dto.Metric{}
+	}
+
+	finalCpuMetricName := resourceType + cpuMetricSuffix
+	finalMemoryMetricName := resourceType + memoryMetricSuffix
+
+	newCPUMetric := dto.Metric{
+		Label: label,
+		Counter: &dto.Counter{
+			Value: &dummyValue,
+		},
+	}
+	newMemoryMetric := dto.Metric{
+		Label: label,
+		Gauge: &dto.Gauge{
+			Value: &dummyValue,
+		},
+	}
+	// if metric family exists add to metric array
+	if cpuMetrics, ok := metricsMap[finalCpuMetricName]; ok {
+		metricsMap[finalCpuMetricName] = append(cpuMetrics, &newCPUMetric)
+	} else {
+		metricsMap[finalCpuMetricName] = []*dto.Metric{&newCPUMetric}
+	}
+	if memoryMetrics, ok := metricsMap[finalMemoryMetricName]; ok {
+		metricsMap[finalMemoryMetricName] = append(memoryMetrics, &newMemoryMetric)
+	} else {
+		metricsMap[finalMemoryMetricName] = []*dto.Metric{&newMemoryMetric}
+	}
+
+	return metricsMap
+}
+
+func (p *MockProvider) getMetricType(metricName string) *dto.MetricType {
+	var (
+		dtoCounterMetricType = dto.MetricType_COUNTER
+		dtoGaugeMetricType   = dto.MetricType_GAUGE
+		cpuMetricSuffix      = "_cpu_usage_seconds_total"
+		memoryMetricSuffix   = "_memory_working_set_bytes"
+	)
+	if strings.HasSuffix(metricName, cpuMetricSuffix) {
+		return &dtoCounterMetricType
+	}
+	if strings.HasSuffix(metricName, memoryMetricSuffix) {
+		return &dtoGaugeMetricType
+	}
+
+	return nil
+}
+
+func (p *MockProvider) GetMetricsResource(ctx context.Context) ([]*dto.MetricFamily, error) {
+	var span trace.Span
+	ctx, span = trace.StartSpan(ctx, "GetMetricsResource") //nolint: ineffassign,staticcheck
+	defer span.End()
+
+	var (
+		nodeNameStr      = "NodeName"
+		podNameStr       = "PodName"
+		containerNameStr = "containerName"
+	)
+	nodeLabels := []*dto.LabelPair{
+		{
+			Name:  &nodeNameStr,
+			Value: &p.nodeName,
+		},
+	}
+
+	metricsMap := p.generateMockMetrics(nil, "node", nodeLabels)
+	for _, pod := range p.pods {
+		podLabels := []*dto.LabelPair{
+			{
+				Name:  &nodeNameStr,
+				Value: &p.nodeName,
+			},
+			{
+				Name:  &podNameStr,
+				Value: &pod.Name,
+			},
+		}
+		metricsMap = p.generateMockMetrics(metricsMap, "pod", podLabels)
+		for _, container := range pod.Spec.Containers {
+			containerLabels := []*dto.LabelPair{
+				{
+					Name:  &nodeNameStr,
+					Value: &p.nodeName,
+				},
+				{
+					Name:  &podNameStr,
+					Value: &pod.Name,
+				},
+				{
+					Name:  &containerNameStr,
+					Value: &container.Name,
+				},
+			}
+			metricsMap = p.generateMockMetrics(metricsMap, "container", containerLabels)
+		}
+	}
+
+	res := []*dto.MetricFamily{}
+	for metricName := range metricsMap {
+		tempName := metricName
+		tempMetrics := metricsMap[tempName]
+
+		metricFamily := dto.MetricFamily{
+			Name:   &tempName,
+			Type:   p.getMetricType(tempName),
+			Metric: tempMetrics,
+		}
+		res = append(res, &metricFamily)
+	}
+
+	return res, nil
+}
+
 // NotifyPods is called to set a pod notifier callback function. This should be called before any operations are done
 // within the provider.
 func (p *MockProvider) NotifyPods(ctx context.Context, notifier func(*v1.Pod)) {
@@ -520,15 +671,15 @@ func buildKeyFromNames(namespace string, name string) (string, error) {
 
 // buildKey is a helper for building the "key" for the providers pod store.
 func buildKey(pod *v1.Pod) (string, error) {
-	if pod.ObjectMeta.Namespace == "" {
+	if pod.Namespace == "" {
 		return "", fmt.Errorf("pod namespace not found")
 	}
 
-	if pod.ObjectMeta.Name == "" {
+	if pod.Name == "" {
 		return "", fmt.Errorf("pod name not found")
 	}
 
-	return buildKeyFromNames(pod.ObjectMeta.Namespace, pod.ObjectMeta.Name)
+	return buildKeyFromNames(pod.Namespace, pod.Name)
 }
 
 // addAttributes adds the specified attributes to the provided span.

cmd/virtual-kubelet/internal/provider/store.go

@@ -13,7 +13,7 @@ type Store struct {
 	ls map[string]InitFunc
 }
 
-func NewStore() *Store { //nolint:golint
+func NewStore() *Store {
 	return &Store{
 		ls: make(map[string]InitFunc),
 	}
@@ -71,4 +71,4 @@ type InitConfig struct {
 	ResourceManager   *manager.ResourceManager
 }
 
-type InitFunc func(InitConfig) (Provider, error) //nolint:golint
+type InitFunc func(InitConfig) (Provider, error)

cmd/virtual-kubelet/internal/provider/types.go

@@ -7,7 +7,7 @@ const (
 	OperatingSystemWindows = "windows"
 )
 
-type OperatingSystems map[string]bool //nolint:golint
+type OperatingSystems map[string]bool
 
 var (
 	// ValidOperatingSystems defines the group of operating systems
@@ -18,7 +18,7 @@ var (
 	}
 )
 
-func (o OperatingSystems) Names() []string { //nolint:golint
+func (o OperatingSystems) Names() []string {
 	keys := make([]string, 0, len(o))
 	for k := range o {
 		keys = append(keys, k)

cmd/virtual-kubelet/main.go

@@ -38,7 +38,7 @@ import (
 var (
 	buildVersion = "N/A"
 	buildTime    = "N/A"
-	k8sVersion   = "v1.15.2" // This should follow the version of k8s.io/kubernetes we are importing
+	k8sVersion   = "v1.31.4" // This should follow the version of k8s.io/client-go we are importing
 )
 
 func main() {

docs/proposals/MetricsUpdateProposal.md

@@ -0,0 +1,296 @@
+#  Virtual Kubelet Metrics Update
+
+<!-- toc -->
+- [Summary](#summary)
+- [Motivation](#motivation)
+  - [Goals](#goals)
+  - [Non-Goals](#non-goals)
+- [Proposal](#proposal)
+- [Design Details](#design-details)
+  - [API](#api)
+  - [Data](#data)
+  - [Changes to the Provider](#changes-to-the-provider)
+  - [Test Plan](#test-plan)
+<!-- /toc -->
+
+## Summary
+
+Add the new /metrics/resource endpoint in the virtual-kubelet to support the metrics server update for new Kubernetes versions `>=1.24`
+
+
+## Motivation
+
+The Kubernetes metrics server now tries to get metrics from the kubelet using the new metrics endpoint [/metrics/resource](https://github.com/kubernetes-sigs/metrics-server/commit/a2d732e5cdbfd93a6ebce221e8df0e8b463eecc6#diff-6e5b914d1403a14af1cc43582a2c9af727113037a3c6a77d8729aaefba084fb5R88),
+while Virtual Kubelet is still exposing the earlier metrics endpoint [/stats/summary](https://github.com/virtual-kubelet/virtual-kubelet/blob/master/node/api/server.go#L90). 
+This causes metrics to break when using virtual kubelet with newer Kubernetes versions (>=1.24). 
+To support the new metrics server, this document proposes adding a new handler to handle the updated metrics endpoint. 
+This will be an additive update, and the old 
+[/stats/summary](https://github.com/virtual-kubelet/virtual-kubelet/blob/master/node/api/server.go#L90) endpoint will still be available to maintain backward compatibility with 
+the older metrics server version.
+
+
+### Goals
+
+- Support metrics for kubernetes version `>=1.24` through adding /metrics/resource endpoint handler.
+
+### Non-Goals
+
+- Ensure pod autoscaling works as expected with the newer kubernetes versions `>=1.24` as expected
+
+## Proposal
+
+Add a new handler for `/metrics/resource` endpoint that calls a new `GetMetricsResource` method in the provider,
+which in-turn returns metrics using the prometheus `model.Samples` data structure as expected by the new metrics server.
+The provider will need to implement the `GetMetricsResource` method in order to add support for the new `/metrics/resource` endpoint with Kubernetes version >=1.24
+
+
+## Design Details
+Currently the virtual kubelet code uses the `PodStatsSummaryHandler` method to set up a http handler for serving pod metrics via the `/stats/summary` endpoint. 
+To support the updated metrics server, we need to add another handler `PodMetricsResourceHandler`  which can serve metrics via the `/metrics/resource` endpoint. 
+The `PodMetricsResourceHandler` calls the new `GetMetricsResource` method of the provider to get the metrics from the specific provider. 
+
+### API
+Add `GetMetricsResource` to `PodHandlerConfig`
+```go
+type PodHandlerConfig struct { //nolint:golint
+	RunInContainer   ContainerExecHandlerFunc
+	GetContainerLogs ContainerLogsHandlerFunc
+	// GetPods is meant to enumerate the pods that the provider knows about
+	GetPods PodListerFunc
+	// GetPodsFromKubernetes is meant to enumerate the pods that the node is meant to be running
+	GetPodsFromKubernetes PodListerFunc
+	GetStatsSummary       PodStatsSummaryHandlerFunc
+	GetMetricsResource    PodMetricsResourceHandlerFunc
+	StreamIdleTimeout     time.Duration
+	StreamCreationTimeout time.Duration
+}
+```
+Add endpoint to `PodHandler` method 
+```go
+const MetricsResourceRouteSuffix = "/metrics/resource"
+
+func PodHandler(p PodHandlerConfig, debug bool) http.Handler {
+	r := mux.NewRouter()
+
+	// This matches the behaviour in the reference kubelet
+	r.StrictSlash(true)
+	if debug {
+		r.HandleFunc("/runningpods/", HandleRunningPods(p.GetPods)).Methods("GET")
+	}
+
+	r.HandleFunc("/pods", HandleRunningPods(p.GetPodsFromKubernetes)).Methods("GET")
+	r.HandleFunc("/containerLogs/{namespace}/{pod}/{container}", HandleContainerLogs(p.GetContainerLogs)).Methods("GET")
+	r.HandleFunc(
+		"/exec/{namespace}/{pod}/{container}",
+		HandleContainerExec(
+			p.RunInContainer,
+			WithExecStreamCreationTimeout(p.StreamCreationTimeout),
+			WithExecStreamIdleTimeout(p.StreamIdleTimeout),
+		),
+	).Methods("POST", "GET")
+
+	if p.GetStatsSummary != nil {
+		f := HandlePodStatsSummary(p.GetStatsSummary)
+		r.HandleFunc("/stats/summary", f).Methods("GET")
+		r.HandleFunc("/stats/summary/", f).Methods("GET")
+	}
+
+	if p.GetMetricsResource != nil {
+		f := HandlePodMetricsResource(p.GetMetricsResource)
+		r.HandleFunc(MetricsResourceRouteSuffix, f).Methods("GET")
+		r.HandleFunc(MetricsResourceRouteSuffix+"/", f).Methods("GET")
+	}
+	r.NotFoundHandler = http.HandlerFunc(NotFound)
+	return r
+}
+```
+  
+New `PodMetricsResourceHandler` method, that uses the new `PodMetricsResourceHandlerFunc` definition.
+```go
+// PodMetricsResourceHandler creates an http handler for serving pod metrics.
+//
+// If the passed in handler func is nil this will create handlers which only
+// serves http.StatusNotImplemented
+func PodMetricsResourceHandler(f PodMetricsResourceHandlerFunc) http.Handler {
+	if f == nil {
+		return http.HandlerFunc(NotImplemented)
+	}
+
+	r := mux.NewRouter()
+
+	h := HandlePodMetricsResource(f)
+
+	r.Handle(MetricsResourceRouteSuffix, ochttp.WithRouteTag(h, "PodMetricsResourceHandler")).Methods("GET")
+	r.Handle(MetricsResourceRouteSuffix+"/", ochttp.WithRouteTag(h, "PodMetricsResourceHandler")).Methods("GET")
+
+	r.NotFoundHandler = http.HandlerFunc(NotFound)
+	return r
+}
+
+```
+ 
+ 
+`HandlePodMetricsResource` method returns a HandlerFunc which serves the metrics encoded in prometheus' text format encoding as expected by the  metrics-server
+```go
+// HandlePodMetricsResource makes an HTTP handler for implementing the kubelet /metrics/resource endpoint
+func HandlePodMetricsResource(h PodMetricsResourceHandlerFunc) http.HandlerFunc {
+	if h == nil {
+		return NotImplemented
+	}
+	return handleError(func(w http.ResponseWriter, req *http.Request) error {
+		metrics, err := h(req.Context())
+		if err != nil {
+			if isCancelled(err) {
+				return err
+			}
+			return errors.Wrap(err, "error getting status from provider")
+		}
+
+		b, err := json.Marshal(metrics)
+		if err != nil {
+			return errors.Wrap(err, "error marshalling metrics")
+		}
+
+		if _, err := w.Write(b); err != nil {
+			return errors.Wrap(err, "could not write to client")
+		}
+		return nil
+	})
+}
+```
+ 
+The `PodMetricsResourceHandlerFunc` returns the metrics data using Prometheus' `MetricFamily` data structure. More details are provided in the Data subsection
+```go
+// PodMetricsResourceHandlerFunc defines the handler for getting pod metrics
+type PodMetricsResourceHandlerFunc func(context.Context) ([]*dto.MetricFamily, error)
+```
+ 
+### Data
+
+The updated metrics server does not add any new fields to the metrics data but uses the Prometheus textparse series parser to parse and reconstruct the [MetricsBatch](https://github.com/kubernetes-sigs/metrics-server/blob/83b2e01f9825849ae5f562e47aa1a4178b5d06e5/pkg/storage/types.go#L31) data structure.  
+Currently virtual-kubelet is sending data to the server using the [summary](https://github.com/virtual-kubelet/virtual-kubelet/blob/be0a062aec9a5eeea3ad6fbe5aec557a235558f6/node/api/statsv1alpha1/types.go#L24) data structure. The Prometheus text parser expects a series of bytes as in the Prometheus [model.Samples](https://github.com/kubernetes/kubernetes/blob/a93eda9db305611cacd8b6ee930ab3149a08f9b0/vendor/github.com/prometheus/common/model/value.go#L184) data structure, similar to the test [here](https://github.com/prometheus/prometheus/blob/c70d85baed260f6013afd18d6cd0ffcac4339861/model/textparse/promparse_test.go#L31).
+ 
+Examples of how the new metrics are defined may be seen in the Kubernetes e2e test that calls the /metrics/resource endpoint [here](https://github.com/kubernetes/kubernetes/blob/a93eda9db305611cacd8b6ee930ab3149a08f9b0/test/e2e_node/resource_metrics_test.go#L76), and the kubelet metrics defined in the Kubernetes/kubelet code [here](https://github.com/kubernetes/kubernetes/blob/master/pkg/kubelet/metrics/collectors/resource_metrics.go) .
+ 
+```go
+var (
+	nodeCPUUsageDesc = metrics.NewDesc("node_cpu_usage_seconds_total",
+		"Cumulative cpu time consumed by the node in core-seconds",
+		nil,
+		nil,
+		metrics.ALPHA,
+		"")
+
+	nodeMemoryUsageDesc = metrics.NewDesc("node_memory_working_set_bytes",
+		"Current working set of the node in bytes",
+		nil,
+		nil,
+		metrics.ALPHA,
+		"")
+
+	containerCPUUsageDesc = metrics.NewDesc("container_cpu_usage_seconds_total",
+		"Cumulative cpu time consumed by the container in core-seconds",
+		[]string{"container", "pod", "namespace"},
+		nil,
+		metrics.ALPHA,
+		"")
+
+	containerMemoryUsageDesc = metrics.NewDesc("container_memory_working_set_bytes",
+		"Current working set of the container in bytes",
+		[]string{"container", "pod", "namespace"},
+		nil,
+		metrics.ALPHA,
+		"")
+
+	podCPUUsageDesc = metrics.NewDesc("pod_cpu_usage_seconds_total",
+		"Cumulative cpu time consumed by the pod in core-seconds",
+		[]string{"pod", "namespace"},
+		nil,
+		metrics.ALPHA,
+		"")
+
+	podMemoryUsageDesc = metrics.NewDesc("pod_memory_working_set_bytes",
+		"Current working set of the pod in bytes",
+		[]string{"pod", "namespace"},
+		nil,
+		metrics.ALPHA,
+		"")
+
+	resourceScrapeResultDesc = metrics.NewDesc("scrape_error",
+		"1 if there was an error while getting container metrics, 0 otherwise",
+		nil,
+		nil,
+		metrics.ALPHA,
+		"")
+
+	containerStartTimeDesc = metrics.NewDesc("container_start_time_seconds",
+		"Start time of the container since unix epoch in seconds",
+		[]string{"container", "pod", "namespace"},
+		nil,
+		metrics.ALPHA,
+		"")
+)
+```
+ 
+The kubernetes/kubelet code implements Prometheus' [collector](https://github.com/kubernetes/kubernetes/blob/master/pkg/kubelet/metrics/collectors/resource_metrics.go#L88) interface which is used along with the k8s.io/component-base implementation of the [registry](https://github.com/kubernetes/component-base/blob/40d14bdbd62f9e2ea697f97d81d4abc72839901e/metrics/registry.go#L114) interface in order to collect and return the metrics data using the Prometheus' [MetricFamily](https://github.com/prometheus/client_model/blob/master/go/metrics.pb.go#L773) data structure.
+ 
+The Gather method in the registry calls the kubelet collector's Collect method, and returns the data using the MetricFamily data structure. The metrics server expects metrics to be encoded in prometheus'
+text format, and the kubelet uses the http handler from prometheus' promhttp module which returns the metrics data encoded in prometheus' text format encoding.
+```go
+type KubeRegistry interface {
+	// Deprecated
+	RawMustRegister(...prometheus.Collector)
+	// CustomRegister is our internal variant of Prometheus registry.Register
+	CustomRegister(c StableCollector) error
+	// CustomMustRegister is our internal variant of Prometheus registry.MustRegister
+	CustomMustRegister(cs ...StableCollector)
+	// Register conforms to Prometheus registry.Register
+	Register(Registerable) error
+	// MustRegister conforms to Prometheus registry.MustRegister
+	MustRegister(...Registerable)
+	// Unregister conforms to Prometheus registry.Unregister
+	Unregister(collector Collector) bool
+	// Gather conforms to Prometheus gatherer.Gather
+	Gather() ([]*dto.MetricFamily, error)
+	// Reset invokes the Reset() function on all items in the registry
+	// which are added as resettables.
+	Reset()
+}
+```
+
+Prometheus’ MetricsFamily data structure:  
+```go
+type MetricFamily struct {
+	Name                 *string     `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"`
+	Help                 *string     `protobuf:"bytes,2,opt,name=help" json:"help,omitempty"`
+	Type                 *MetricType `protobuf:"varint,3,opt,name=type,enum=io.prometheus.client.MetricType" json:"type,omitempty"`
+	Metric               []*Metric   `protobuf:"bytes,4,rep,name=metric" json:"metric,omitempty"`
+	XXX_NoUnkeyedLiteral struct{}    `json:"-"`
+	XXX_unrecognized     []byte      `json:"-"`
+	XXX_sizecache        int32       `json:"-"`
+}
+```
+
+Therefore the provider's GetMetricsResource method should use the same return type as the Gather method in the registry interface.
+ 
+### Changes to the Provider.
+
+In order to support the new metrics endpoint the Provider must implement the GetMetricsResource method with definition
+
+```golang
+
+import (
+  dto "github.com/prometheus/client_model/go"
+  "context"
+)
+
+func GetMetricsResource(context.Context) ([]*dto.MetricsFamily, error) {
+...
+}
+```
+
+### Test Plan
+
+- Write a provider implementation for GetMetricsResource method in ACI Provider and deploy pods get metrics using kubectl 
+- Run end-to-end tests with the provider implementation
+

errdefs/auth.go

@@ -0,0 +1,33 @@
+package errdefs
+
+import (
+	"errors"
+	"fmt"
+)
+
+var (
+	// ErrForbidden is returned when the user is not authorized to perform the operation.
+	ErrForbidden = errors.New("forbidden")
+	// ErrUnauthorized is returned when the user is not authenticated.
+	ErrUnauthorized = errors.New("unauthorized")
+)
+
+// Unauthorized wraps ErrUnauthorized with a message.
+func Unauthorized(msg string) error {
+	return fmt.Errorf("%w: %s", ErrUnauthorized, msg)
+}
+
+// Forbidden wraps ErrForbidden with a message.
+func Forbidden(msg string) error {
+	return fmt.Errorf("%w: %s", ErrForbidden, msg)
+}
+
+// IsForbidden returns true if the error has ErrForbidden in the error chain.
+func IsForbidden(err error) bool {
+	return errors.Is(err, ErrForbidden)
+}
+
+// IsUnauthorized returns true if the error has ErrUnauthorized in the error chain.
+func IsUnauthorized(err error) bool {
+	return errors.Is(err, ErrUnauthorized)
+}

go.mod

@@ -1,99 +1,125 @@
 module github.com/virtual-kubelet/virtual-kubelet
 
-go 1.17
+go 1.23.0
+
+toolchain go1.23.4
 
 require (
 	contrib.go.opencensus.io/exporter/jaeger v0.2.1
 	contrib.go.opencensus.io/exporter/ocagent v0.7.0
-	github.com/bombsimon/logrusr/v3 v3.0.0
-	github.com/elazarl/goproxy v0.0.0-20190421051319-9d40249d3c2f // indirect
-	github.com/google/go-cmp v0.5.8
-	github.com/gorilla/mux v1.8.0
+	github.com/bombsimon/logrusr/v3 v3.1.0
+	github.com/google/go-cmp v0.7.0
+	github.com/gorilla/mux v1.8.1
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/pkg/errors v0.9.1
-	github.com/prometheus/client_golang v1.13.0
-	github.com/sirupsen/logrus v1.9.0
-	github.com/spf13/cobra v1.5.0
+	github.com/prometheus/client_model v0.6.2
+	github.com/prometheus/common v0.64.0
+	github.com/sirupsen/logrus v1.9.3
+	github.com/spf13/cobra v1.8.1
 	github.com/spf13/pflag v1.0.5
-	go.opencensus.io v0.23.0
-	go.opentelemetry.io/otel v0.20.0
-	go.opentelemetry.io/otel/sdk v0.20.0
-	go.opentelemetry.io/otel/trace v0.20.0
-	golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4
-	golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f
-	golang.org/x/time v0.0.0-20220609170525-579cf78fd858
+	github.com/stretchr/testify v1.10.0
+	go.opencensus.io v0.24.0
+	go.opentelemetry.io/otel v1.36.0
+	go.opentelemetry.io/otel/sdk v1.36.0
+	go.opentelemetry.io/otel/trace v1.36.0
+	golang.org/x/sync v0.14.0
+	golang.org/x/time v0.9.0
+	google.golang.org/protobuf v1.36.6
 	gotest.tools v2.2.0+incompatible
-	k8s.io/api v0.25.0
-	k8s.io/apimachinery v0.25.0
-	k8s.io/apiserver v0.25.0
-	k8s.io/client-go v0.25.0
-	k8s.io/klog/v2 v2.80.1
-	k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed
-	sigs.k8s.io/controller-runtime v0.13.0
+	k8s.io/api v0.31.4
+	k8s.io/apimachinery v0.31.4
+	k8s.io/apiserver v0.31.4
+	k8s.io/client-go v0.31.4
+	k8s.io/klog/v2 v2.130.1
+	k8s.io/kubelet v0.31.4
+	k8s.io/utils v0.0.0-20240711033017-18e509b52bc8
+	sigs.k8s.io/controller-runtime v0.19.4
 )
 
 require (
-	github.com/PuerkitoBio/purell v1.1.1 // indirect
-	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
+	github.com/NYTimes/gziphandler v1.1.1 // indirect
+	github.com/antlr4-go/antlr/v4 v4.13.0 // indirect
+	github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/blang/semver/v4 v4.0.0 // indirect
-	github.com/census-instrumentation/opencensus-proto v0.2.1 // indirect
-	github.com/cespare/xxhash/v2 v2.1.2 // indirect
-	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/emicklei/go-restful/v3 v3.8.0 // indirect
-	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
-	github.com/evanphx/json-patch/v5 v5.6.0 // indirect
-	github.com/felixge/httpsnoop v1.0.1 // indirect
-	github.com/fsnotify/fsnotify v1.5.4 // indirect
-	github.com/go-logr/logr v1.2.3 // indirect
-	github.com/go-openapi/jsonpointer v0.19.5 // indirect
-	github.com/go-openapi/jsonreference v0.19.5 // indirect
-	github.com/go-openapi/swag v0.19.14 // indirect
+	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
+	github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/coreos/go-semver v0.3.1 // indirect
+	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
+	github.com/evanphx/json-patch/v5 v5.9.0 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/fsnotify/fsnotify v1.7.0 // indirect
+	github.com/fxamacker/cbor/v2 v2.7.0 // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-openapi/jsonpointer v0.19.6 // indirect
+	github.com/go-openapi/jsonreference v0.20.2 // indirect
+	github.com/go-openapi/swag v0.22.4 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
-	github.com/golang/protobuf v1.5.2 // indirect
-	github.com/google/gnostic v0.5.7-v3refs // indirect
-	github.com/google/gofuzz v1.1.0 // indirect
-	github.com/google/uuid v1.1.2 // indirect
-	github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
+	github.com/google/cel-go v0.20.1 // indirect
+	github.com/google/gnostic-models v0.6.8 // indirect
+	github.com/google/gofuzz v1.2.0 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/gorilla/websocket v1.5.0 // indirect
+	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect
 	github.com/imdario/mergo v0.3.12 // indirect
-	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/mailru/easyjson v0.7.6 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
-	github.com/moby/spdystream v0.2.0 // indirect
+	github.com/klauspost/compress v1.17.9 // indirect
+	github.com/kylelemons/godebug v1.1.0 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/moby/spdystream v0.4.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
-	github.com/prometheus/client_model v0.2.0 // indirect
-	github.com/prometheus/common v0.37.0 // indirect
-	github.com/prometheus/procfs v0.8.0 // indirect
+	github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+	github.com/prometheus/client_golang v1.20.5 // indirect
+	github.com/prometheus/procfs v0.15.1 // indirect
+	github.com/stoewer/go-strcase v1.2.0 // indirect
 	github.com/uber/jaeger-client-go v2.25.0+incompatible // indirect
-	go.opentelemetry.io/contrib v0.20.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp v0.20.0 // indirect
-	go.opentelemetry.io/otel/metric v0.20.0 // indirect
-	go.opentelemetry.io/otel/sdk/export/metric v0.20.0 // indirect
-	go.opentelemetry.io/otel/sdk/metric v0.20.0 // indirect
-	go.opentelemetry.io/proto/otlp v0.7.0 // indirect
-	golang.org/x/net v0.0.0-20220722155237-a158d28d115b // indirect
-	golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b // indirect
-	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
-	golang.org/x/text v0.3.7 // indirect
-	google.golang.org/api v0.43.0 // indirect
-	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect
-	google.golang.org/grpc v1.47.0 // indirect
-	google.golang.org/protobuf v1.28.1 // indirect
+	github.com/x448/float16 v0.8.4 // indirect
+	go.etcd.io/etcd/api/v3 v3.5.14 // indirect
+	go.etcd.io/etcd/client/pkg/v3 v3.5.14 // indirect
+	go.etcd.io/etcd/client/v3 v3.5.14 // indirect
+	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0 // indirect
+	go.opentelemetry.io/otel/metric v1.36.0 // indirect
+	go.opentelemetry.io/proto/otlp v1.3.1 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	go.uber.org/zap v1.26.0 // indirect
+	golang.org/x/crypto v0.38.0 // indirect
+	golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc // indirect
+	golang.org/x/net v0.40.0 // indirect
+	golang.org/x/oauth2 v0.30.0 // indirect
+	golang.org/x/sys v0.33.0 // indirect
+	golang.org/x/term v0.32.0 // indirect
+	golang.org/x/text v0.25.0 // indirect
+	google.golang.org/api v0.30.0 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect
+	google.golang.org/grpc v1.65.0 // indirect
+	gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/apiextensions-apiserver v0.25.0 // indirect
-	k8s.io/component-base v0.25.0 // indirect
-	k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 // indirect
-	sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.32 // indirect
-	sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect
-	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
-	sigs.k8s.io/yaml v1.3.0 // indirect
+	k8s.io/apiextensions-apiserver v0.31.0 // indirect
+	k8s.io/component-base v0.31.4 // indirect
+	k8s.io/kms v0.31.4 // indirect
+	k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
+	sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3 // indirect
+	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
+	sigs.k8s.io/yaml v1.4.0 // indirect
 )

hack/skaffold/virtual-kubelet/Dockerfile

@@ -1,8 +1,8 @@
 FROM gcr.io/distroless/base
 
-ENV APISERVER_CERT_LOCATION /vkubelet-mock-0-crt.pem
-ENV APISERVER_KEY_LOCATION /vkubelet-mock-0-key.pem
-ENV KUBELET_PORT 10250
+ENV APISERVER_CERT_LOCATION=/vkubelet-mock-0-crt.pem
+ENV APISERVER_KEY_LOCATION=/vkubelet-mock-0-key.pem
+ENV KUBELET_PORT=10250
 
 # Use the pre-built binary in "bin/virtual-kubelet".
 COPY bin/e2e/virtual-kubelet /virtual-kubelet

hack/skaffold/virtual-kubelet/skaffold.yml

@@ -1,18 +1,17 @@
-apiVersion: skaffold/v2beta10
+apiVersion: skaffold/v4beta11
 kind: Config
 build:
   artifacts:
-  - image: virtual-kubelet
-    docker:
-      # Use a Dockerfile specific for development only.
-      dockerfile: hack/skaffold/virtual-kubelet/Dockerfile
-deploy:
-  kubectl:
-    manifests:
+    - image: virtual-kubelet
+      docker:
+        dockerfile: hack/skaffold/virtual-kubelet/Dockerfile
+manifests:
+  rawYaml:
     - hack/skaffold/virtual-kubelet/base.yml
     - hack/skaffold/virtual-kubelet/pod.yml
+deploy:
+  kubectl: {}
 profiles:
-- name: local
-  build:
-    # For the "local" profile, we must perform the build locally.
-    local: {}
+  - name: local
+    build:
+      local: {}

internal/kubernetes/portforward/constants.go

@@ -0,0 +1,24 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package portforward contains server-side logic for handling port forwarding requests.
+package portforward
+
+// ProtocolV1Name is the name of the subprotocol used for port forwarding.
+const ProtocolV1Name = "portforward.k8s.io"
+
+// SupportedProtocols are the supported port forwarding protocols.
+var SupportedProtocols = []string{ProtocolV1Name}

internal/kubernetes/portforward/httpstream.go

@@ -0,0 +1,317 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package portforward
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/http"
+	"strconv"
+	"sync"
+	"time"
+
+	api "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/httpstream"
+	"k8s.io/apimachinery/pkg/util/httpstream/spdy"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+
+	"k8s.io/klog/v2"
+)
+
+func handleHTTPStreams(req *http.Request, w http.ResponseWriter, portForwarder PortForwarder, podName string, uid types.UID, supportedPortForwardProtocols []string, idleTimeout, streamCreationTimeout time.Duration) error {
+	_, err := httpstream.Handshake(req, w, supportedPortForwardProtocols)
+	// negotiated protocol isn't currently used server side, but could be in the future
+	if err != nil {
+		// Handshake writes the error to the client
+		return err
+	}
+	streamChan := make(chan httpstream.Stream, 1)
+
+	klog.V(5).InfoS("Upgrading port forward response")
+	
+	// TODO aka-somix: SPDY is deprecated and it should be replaced in order to support HTTP/2
+	upgrader := spdy.NewResponseUpgrader()
+	conn := upgrader.UpgradeResponse(w, req, httpStreamReceived(streamChan))
+	if conn == nil {
+		return errors.New("unable to upgrade httpstream connection")
+	}
+	defer conn.Close()
+
+	klog.V(5).InfoS("Connection setting port forwarding streaming connection idle timeout", "connection", conn, "idleTimeout", idleTimeout)
+	conn.SetIdleTimeout(idleTimeout)
+
+	h := &httpStreamHandler{
+		conn:                  conn,
+		streamChan:            streamChan,
+		streamPairs:           make(map[string]*httpStreamPair),
+		streamCreationTimeout: streamCreationTimeout,
+		pod:                   podName,
+		uid:                   uid,
+		forwarder:             portForwarder,
+	}
+	h.run()
+
+	return nil
+}
+
+// httpStreamReceived is the httpstream.NewStreamHandler for port
+// forward streams. It checks each stream's port and stream type headers,
+// rejecting any streams that with missing or invalid values. Each valid
+// stream is sent to the streams channel.
+func httpStreamReceived(streams chan httpstream.Stream) func(httpstream.Stream, <-chan struct{}) error {
+	return func(stream httpstream.Stream, replySent <-chan struct{}) error {
+		// make sure it has a valid port header
+		portString := stream.Headers().Get(api.PortHeader)
+		if len(portString) == 0 {
+			return fmt.Errorf("%q header is required", api.PortHeader)
+		}
+		port, err := strconv.ParseUint(portString, 10, 16)
+		if err != nil {
+			return fmt.Errorf("unable to parse %q as a port: %v", portString, err)
+		}
+		if port < 1 {
+			return fmt.Errorf("port %q must be > 0", portString)
+		}
+
+		// make sure it has a valid stream type header
+		streamType := stream.Headers().Get(api.StreamType)
+		if len(streamType) == 0 {
+			return fmt.Errorf("%q header is required", api.StreamType)
+		}
+		if streamType != api.StreamTypeError && streamType != api.StreamTypeData {
+			return fmt.Errorf("invalid stream type %q", streamType)
+		}
+
+		streams <- stream
+		return nil
+	}
+}
+
+// httpStreamHandler is capable of processing multiple port forward
+// requests over a single httpstream.Connection.
+type httpStreamHandler struct {
+	conn                  httpstream.Connection
+	streamChan            chan httpstream.Stream
+	streamPairsLock       sync.RWMutex
+	streamPairs           map[string]*httpStreamPair
+	streamCreationTimeout time.Duration
+	pod                   string
+	uid                   types.UID
+	forwarder             PortForwarder
+}
+
+// getStreamPair returns a httpStreamPair for requestID. This creates a
+// new pair if one does not yet exist for the requestID. The returned bool is
+// true if the pair was created.
+func (h *httpStreamHandler) getStreamPair(requestID string) (*httpStreamPair, bool) {
+	h.streamPairsLock.Lock()
+	defer h.streamPairsLock.Unlock()
+
+	if p, ok := h.streamPairs[requestID]; ok {
+		klog.V(5).InfoS("Connection request found existing stream pair", "connection", h.conn, "request", requestID)
+		return p, false
+	}
+
+	klog.V(5).InfoS("Connection request creating new stream pair", "connection", h.conn, "request", requestID)
+
+	p := newPortForwardPair(requestID)
+	h.streamPairs[requestID] = p
+
+	return p, true
+}
+
+// monitorStreamPair waits for the pair to receive both its error and data
+// streams, or for the timeout to expire (whichever happens first), and then
+// removes the pair.
+func (h *httpStreamHandler) monitorStreamPair(p *httpStreamPair, timeout <-chan time.Time) {
+	select {
+	case <-timeout:
+		err := fmt.Errorf("(conn=%v, request=%s) timed out waiting for streams", h.conn, p.requestID)
+		utilruntime.HandleError(err)
+		p.printError(err.Error())
+	case <-p.complete:
+		klog.V(5).InfoS("Connection request successfully received error and data streams", "connection", h.conn, "request", p.requestID)
+	}
+	h.removeStreamPair(p.requestID)
+}
+
+// hasStreamPair returns a bool indicating if a stream pair for requestID
+// exists.
+func (h *httpStreamHandler) hasStreamPair(requestID string) bool {
+	h.streamPairsLock.RLock()
+	defer h.streamPairsLock.RUnlock()
+
+	_, ok := h.streamPairs[requestID]
+	return ok
+}
+
+// removeStreamPair removes the stream pair identified by requestID from streamPairs.
+func (h *httpStreamHandler) removeStreamPair(requestID string) {
+	h.streamPairsLock.Lock()
+	defer h.streamPairsLock.Unlock()
+
+	if h.conn != nil {
+		pair := h.streamPairs[requestID]
+		h.conn.RemoveStreams(pair.dataStream, pair.errorStream)
+	}
+	delete(h.streamPairs, requestID)
+}
+
+// requestID returns the request id for stream.
+func (h *httpStreamHandler) requestID(stream httpstream.Stream) string {
+	requestID := stream.Headers().Get(api.PortForwardRequestIDHeader)
+	if len(requestID) == 0 {
+		klog.V(5).InfoS("Connection stream received without requestID header", "connection", h.conn)
+		// If we get here, it's because the connection came from an older client
+		// that isn't generating the request id header
+		// (https://github.com/kubernetes/kubernetes/blob/843134885e7e0b360eb5441e85b1410a8b1a7a0c/pkg/client/unversioned/portforward/portforward.go#L258-L287)
+		//
+		// This is a best-effort attempt at supporting older clients.
+		//
+		// When there aren't concurrent new forwarded connections, each connection
+		// will have a pair of streams (data, error), and the stream IDs will be
+		// consecutive odd numbers, e.g. 1 and 3 for the first connection. Convert
+		// the stream ID into a pseudo-request id by taking the stream type and
+		// using id = stream.Identifier() when the stream type is error,
+		// and id = stream.Identifier() - 2 when it's data.
+		//
+		// NOTE: this only works when there are not concurrent new streams from
+		// multiple forwarded connections; it's a best-effort attempt at supporting
+		// old clients that don't generate request ids.  If there are concurrent
+		// new connections, it's possible that 1 connection gets streams whose IDs
+		// are not consecutive (e.g. 5 and 9 instead of 5 and 7).
+		streamType := stream.Headers().Get(api.StreamType)
+		switch streamType {
+		case api.StreamTypeError:
+			requestID = strconv.Itoa(int(stream.Identifier()))
+		case api.StreamTypeData:
+			requestID = strconv.Itoa(int(stream.Identifier()) - 2)
+		}
+
+		klog.V(5).InfoS("Connection automatically assigning request ID from stream type and stream ID", "connection", h.conn, "request", requestID, "streamType", streamType, "stream", stream.Identifier())
+	}
+	return requestID
+}
+
+// run is the main loop for the httpStreamHandler. It processes new
+// streams, invoking portForward for each complete stream pair. The loop exits
+// when the httpstream.Connection is closed.
+func (h *httpStreamHandler) run() {
+	klog.V(5).InfoS("Connection waiting for port forward streams", "connection", h.conn)
+Loop:
+	for {
+		select {
+		case <-h.conn.CloseChan():
+			klog.V(5).InfoS("Connection upgraded connection closed", "connection", h.conn)
+			break Loop
+		case stream := <-h.streamChan:
+			requestID := h.requestID(stream)
+			streamType := stream.Headers().Get(api.StreamType)
+			klog.V(5).InfoS("Connection request received new type of stream", "connection", h.conn, "request", requestID, "streamType", streamType)
+
+			p, created := h.getStreamPair(requestID)
+			if created {
+				go h.monitorStreamPair(p, time.After(h.streamCreationTimeout))
+			}
+			if complete, err := p.add(stream); err != nil {
+				msg := fmt.Sprintf("error processing stream for request %s: %v", requestID, err)
+				utilruntime.HandleError(errors.New(msg))
+				p.printError(msg)
+			} else if complete {
+				go h.portForward(p)
+			}
+		}
+	}
+}
+
+// portForward invokes the httpStreamHandler's forwarder.PortForward
+// function for the given stream pair.
+func (h *httpStreamHandler) portForward(p *httpStreamPair) {
+	ctx := context.Background()
+	defer p.dataStream.Close()
+	defer p.errorStream.Close()
+
+	portString := p.dataStream.Headers().Get(api.PortHeader)
+	port, _ := strconv.ParseInt(portString, 10, 32)
+
+	klog.V(5).InfoS("Connection request invoking forwarder.PortForward for port", "connection", h.conn, "request", p.requestID, "port", portString)
+	err := h.forwarder.PortForward(ctx, h.pod, h.uid, int32(port), p.dataStream)
+	klog.V(5).InfoS("Connection request done invoking forwarder.PortForward for port", "connection", h.conn, "request", p.requestID, "port", portString)
+
+	if err != nil {
+		msg := fmt.Errorf("error forwarding port %d to pod %s, uid %v: %v", port, h.pod, h.uid, err)
+		utilruntime.HandleError(msg)
+		fmt.Fprint(p.errorStream, msg.Error())
+	}
+}
+
+// httpStreamPair represents the error and data streams for a port
+// forwarding request.
+type httpStreamPair struct {
+	lock        sync.RWMutex
+	requestID   string
+	dataStream  httpstream.Stream
+	errorStream httpstream.Stream
+	complete    chan struct{}
+}
+
+// newPortForwardPair creates a new httpStreamPair.
+func newPortForwardPair(requestID string) *httpStreamPair {
+	return &httpStreamPair{
+		requestID: requestID,
+		complete:  make(chan struct{}),
+	}
+}
+
+// add adds the stream to the httpStreamPair. If the pair already
+// contains a stream for the new stream's type, an error is returned. add
+// returns true if both the data and error streams for this pair have been
+// received.
+func (p *httpStreamPair) add(stream httpstream.Stream) (bool, error) {
+	p.lock.Lock()
+	defer p.lock.Unlock()
+
+	switch stream.Headers().Get(api.StreamType) {
+	case api.StreamTypeError:
+		if p.errorStream != nil {
+			return false, errors.New("error stream already assigned")
+		}
+		p.errorStream = stream
+	case api.StreamTypeData:
+		if p.dataStream != nil {
+			return false, errors.New("data stream already assigned")
+		}
+		p.dataStream = stream
+	}
+
+	complete := p.errorStream != nil && p.dataStream != nil
+	if complete {
+		close(p.complete)
+	}
+	return complete, nil
+}
+
+// printError writes s to p.errorStream if p.errorStream has been set.
+func (p *httpStreamPair) printError(s string) {
+	p.lock.RLock()
+	defer p.lock.RUnlock()
+	if p.errorStream != nil {
+		fmt.Fprint(p.errorStream, s)
+	}
+}

internal/kubernetes/portforward/httpstream_test.go

@@ -0,0 +1,267 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package portforward
+
+import (
+	"net/http"
+	"testing"
+	"time"
+
+	api "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/util/httpstream"
+)
+
+func TestHTTPStreamReceived(t *testing.T) {
+	tests := map[string]struct {
+		port          string
+		streamType    string
+		expectedError string
+	}{
+		"missing port": {
+			expectedError: `"port" header is required`,
+		},
+		"unable to parse port": {
+			port:          "abc",
+			expectedError: `unable to parse "abc" as a port: strconv.ParseUint: parsing "abc": invalid syntax`,
+		},
+		"negative port": {
+			port:          "-1",
+			expectedError: `unable to parse "-1" as a port: strconv.ParseUint: parsing "-1": invalid syntax`,
+		},
+		"missing stream type": {
+			port:          "80",
+			expectedError: `"streamType" header is required`,
+		},
+		"valid port with error stream": {
+			port:       "80",
+			streamType: "error",
+		},
+		"valid port with data stream": {
+			port:       "80",
+			streamType: "data",
+		},
+		"invalid stream type": {
+			port:          "80",
+			streamType:    "foo",
+			expectedError: `invalid stream type "foo"`,
+		},
+	}
+	for name, test := range tests {
+		streams := make(chan httpstream.Stream, 1)
+		f := httpStreamReceived(streams)
+		stream := newFakeHTTPStream()
+		if len(test.port) > 0 {
+			stream.headers.Set("port", test.port)
+		}
+		if len(test.streamType) > 0 {
+			stream.headers.Set("streamType", test.streamType)
+		}
+		replySent := make(chan struct{})
+		err := f(stream, replySent)
+		close(replySent)
+		if len(test.expectedError) > 0 {
+			if err == nil {
+				t.Errorf("%s: expected err=%q, but it was nil", name, test.expectedError)
+			}
+			if e, a := test.expectedError, err.Error(); e != a {
+				t.Errorf("%s: expected err=%q, got %q", name, e, a)
+			}
+			continue
+		}
+		if err != nil {
+			t.Errorf("%s: unexpected error %v", name, err)
+			continue
+		}
+		if s := <-streams; s != stream {
+			t.Errorf("%s: expected stream %#v, got %#v", name, stream, s)
+		}
+	}
+}
+
+type fakeConn struct {
+	removeStreamsCalled bool
+}
+
+func (*fakeConn) CreateStream(headers http.Header) (httpstream.Stream, error) { return nil, nil }
+func (*fakeConn) Close() error                                                { return nil }
+func (*fakeConn) CloseChan() <-chan bool                                      { return nil }
+func (*fakeConn) SetIdleTimeout(timeout time.Duration)                        {}
+func (f *fakeConn) RemoveStreams(streams ...httpstream.Stream)                { f.removeStreamsCalled = true }
+
+func TestGetStreamPair(t *testing.T) {
+	timeout := make(chan time.Time)
+
+	conn := &fakeConn{}
+	h := &httpStreamHandler{
+		streamPairs: make(map[string]*httpStreamPair),
+		conn:        conn,
+	}
+
+	// test adding a new entry
+	p, created := h.getStreamPair("1")
+	if p == nil {
+		t.Fatalf("unexpected nil pair")
+	}
+	if !created {
+		t.Fatal("expected created=true")
+	}
+	if p.dataStream != nil {
+		t.Errorf("unexpected non-nil data stream")
+	}
+	if p.errorStream != nil {
+		t.Errorf("unexpected non-nil error stream")
+	}
+
+	// start the monitor for this pair
+	monitorDone := make(chan struct{})
+	go func() {
+		h.monitorStreamPair(p, timeout)
+		close(monitorDone)
+	}()
+
+	if !h.hasStreamPair("1") {
+		t.Fatal("This should still be true")
+	}
+
+	// make sure we can retrieve an existing entry
+	p2, created := h.getStreamPair("1")
+	if created {
+		t.Fatal("expected created=false")
+	}
+	if p != p2 {
+		t.Fatalf("retrieving an existing pair: expected %#v, got %#v", p, p2)
+	}
+
+	// removed via complete
+	dataStream := newFakeHTTPStream()
+	dataStream.headers.Set(api.StreamType, api.StreamTypeData)
+	complete, err := p.add(dataStream)
+	if err != nil {
+		t.Fatalf("unexpected error adding data stream to pair: %v", err)
+	}
+	if complete {
+		t.Fatalf("unexpected complete")
+	}
+
+	errorStream := newFakeHTTPStream()
+	errorStream.headers.Set(api.StreamType, api.StreamTypeError)
+	complete, err = p.add(errorStream)
+	if err != nil {
+		t.Fatalf("unexpected error adding error stream to pair: %v", err)
+	}
+	if !complete {
+		t.Fatal("unexpected incomplete")
+	}
+
+	// make sure monitorStreamPair completed
+	<-monitorDone
+
+	if !conn.removeStreamsCalled {
+		t.Fatalf("connection remove stream not called")
+	}
+	conn.removeStreamsCalled = false
+
+	// make sure the pair was removed
+	if h.hasStreamPair("1") {
+		t.Fatal("expected removal of pair after both data and error streams received")
+	}
+
+	// removed via timeout
+	p, created = h.getStreamPair("2")
+	if !created {
+		t.Fatal("expected created=true")
+	}
+	if p == nil {
+		t.Fatal("expected p not to be nil")
+	}
+
+	monitorDone = make(chan struct{})
+	go func() {
+		h.monitorStreamPair(p, timeout)
+		close(monitorDone)
+	}()
+	// cause the timeout
+	close(timeout)
+	// make sure monitorStreamPair completed
+	<-monitorDone
+	if h.hasStreamPair("2") {
+		t.Fatal("expected stream pair to be removed")
+	}
+	if !conn.removeStreamsCalled {
+		t.Fatalf("connection remove stream not called")
+	}
+}
+
+func TestRequestID(t *testing.T) {
+	h := &httpStreamHandler{}
+
+	s := newFakeHTTPStream()
+	s.headers.Set(api.StreamType, api.StreamTypeError)
+	s.id = 1
+	if e, a := "1", h.requestID(s); e != a {
+		t.Errorf("expected %q, got %q", e, a)
+	}
+
+	s.headers.Set(api.StreamType, api.StreamTypeData)
+	s.id = 3
+	if e, a := "1", h.requestID(s); e != a {
+		t.Errorf("expected %q, got %q", e, a)
+	}
+
+	s.id = 7
+	s.headers.Set(api.PortForwardRequestIDHeader, "2")
+	if e, a := "2", h.requestID(s); e != a {
+		t.Errorf("expected %q, got %q", e, a)
+	}
+}
+
+type fakeHTTPStream struct {
+	headers http.Header
+	id      uint32
+}
+
+func newFakeHTTPStream() *fakeHTTPStream {
+	return &fakeHTTPStream{
+		headers: make(http.Header),
+	}
+}
+
+var _ httpstream.Stream = &fakeHTTPStream{}
+
+func (s *fakeHTTPStream) Read(data []byte) (int, error) {
+	return 0, nil
+}
+
+func (s *fakeHTTPStream) Write(data []byte) (int, error) {
+	return 0, nil
+}
+
+func (s *fakeHTTPStream) Close() error {
+	return nil
+}
+
+func (s *fakeHTTPStream) Reset() error {
+	return nil
+}
+
+func (s *fakeHTTPStream) Headers() http.Header {
+	return s.headers
+}
+
+func (s *fakeHTTPStream) Identifier() uint32 {
+	return s.id
+}

internal/kubernetes/portforward/portforward.go

@@ -0,0 +1,54 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package portforward
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"time"
+
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apiserver/pkg/util/wsstream"
+)
+
+// PortForwarder knows how to forward content from a data stream to/from a port
+// in a pod.
+type PortForwarder interface {
+	// PortForwarder copies data between a data stream and a port in a pod.
+	PortForward(ctx context.Context, name string, uid types.UID, port int32, stream io.ReadWriteCloser) error
+}
+
+// ServePortForward handles a port forwarding request.  A single request is
+// kept alive as long as the client is still alive and the connection has not
+// been timed out due to idleness. This function handles multiple forwarded
+// connections; i.e., multiple `curl http://localhost:8888/` requests will be
+// handled by a single invocation of ServePortForward.
+func ServePortForward(w http.ResponseWriter, req *http.Request, portForwarder PortForwarder, podName string, uid types.UID, portForwardOptions *V4Options, idleTimeout time.Duration, streamCreationTimeout time.Duration, supportedProtocols []string) {
+	var err error
+	if wsstream.IsWebSocketRequest(req) {
+		err = handleWebSocketStreams(req, w, portForwarder, podName, uid, portForwardOptions, supportedProtocols, idleTimeout, streamCreationTimeout)
+	} else {
+		err = handleHTTPStreams(req, w, portForwarder, podName, uid, supportedProtocols, idleTimeout, streamCreationTimeout)
+	}
+
+	if err != nil {
+		runtime.HandleError(err)
+		return
+	}
+}

internal/kubernetes/portforward/websocket.go

@@ -0,0 +1,199 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package portforward
+
+import (
+	"context"
+	"encoding/binary"
+	"fmt"
+	"io"
+	"net/http"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"k8s.io/klog/v2"
+
+	api "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apiserver/pkg/endpoints/responsewriter"
+	"k8s.io/apiserver/pkg/util/wsstream"
+)
+
+const (
+	dataChannel = iota
+	errorChannel
+
+	v4BinaryWebsocketProtocol = "v4." + wsstream.ChannelWebSocketProtocol
+	v4Base64WebsocketProtocol = "v4." + wsstream.Base64ChannelWebSocketProtocol
+)
+
+// V4Options contains details about which streams are required for port
+// forwarding.
+// All fields included in V4Options need to be expressed explicitly in the
+// CRI (k8s.io/cri-api/pkg/apis/{version}/api.proto) PortForwardRequest.
+type V4Options struct {
+	Ports []int32
+}
+
+// NewV4Options creates a new options from the Request.
+func NewV4Options(req *http.Request) (*V4Options, error) {
+	if !wsstream.IsWebSocketRequest(req) {
+		return &V4Options{}, nil
+	}
+
+	portStrings := req.URL.Query()[api.PortHeader]
+	if len(portStrings) == 0 {
+		return nil, fmt.Errorf("query parameter %q is required", api.PortHeader)
+	}
+
+	ports := make([]int32, 0, len(portStrings))
+	for _, portString := range portStrings {
+		if len(portString) == 0 {
+			return nil, fmt.Errorf("query parameter %q cannot be empty", api.PortHeader)
+		}
+		for _, p := range strings.Split(portString, ",") {
+			port, err := strconv.ParseUint(p, 10, 16)
+			if err != nil {
+				return nil, fmt.Errorf("unable to parse %q as a port: %v", portString, err)
+			}
+			if port < 1 {
+				return nil, fmt.Errorf("port %q must be > 0", portString)
+			}
+			ports = append(ports, int32(port))
+		}
+	}
+
+	return &V4Options{
+		Ports: ports,
+	}, nil
+}
+
+// BuildV4Options returns a V4Options based on the given information.
+func BuildV4Options(ports []int32) (*V4Options, error) {
+	return &V4Options{Ports: ports}, nil
+}
+
+// handleWebSocketStreams handles requests to forward ports to a pod via
+// a PortForwarder. A pair of streams are created per port (DATA n,
+// ERROR n+1). The associated port is written to each stream as a unsigned 16
+// bit integer in little endian format.
+func handleWebSocketStreams(req *http.Request, w http.ResponseWriter, portForwarder PortForwarder, podName string, uid types.UID, opts *V4Options, supportedPortForwardProtocols []string, idleTimeout, streamCreationTimeout time.Duration) error {
+	channels := make([]wsstream.ChannelType, 0, len(opts.Ports)*2)
+	for i := 0; i < len(opts.Ports); i++ {
+		channels = append(channels, wsstream.ReadWriteChannel, wsstream.WriteChannel)
+	}
+	conn := wsstream.NewConn(map[string]wsstream.ChannelProtocolConfig{
+		"": {
+			Binary:   true,
+			Channels: channels,
+		},
+		v4BinaryWebsocketProtocol: {
+			Binary:   true,
+			Channels: channels,
+		},
+		v4Base64WebsocketProtocol: {
+			Binary:   false,
+			Channels: channels,
+		},
+	})
+	conn.SetIdleTimeout(idleTimeout)
+	_, streams, err := conn.Open(responsewriter.GetOriginal(w), req)
+	if err != nil {
+		err = fmt.Errorf("unable to upgrade websocket connection: %v", err)
+		return err
+	}
+	defer conn.Close()
+	streamPairs := make([]*websocketStreamPair, len(opts.Ports))
+	for i := range streamPairs {
+		streamPair := websocketStreamPair{
+			port:        opts.Ports[i],
+			dataStream:  streams[i*2+dataChannel],
+			errorStream: streams[i*2+errorChannel],
+		}
+		streamPairs[i] = &streamPair
+
+		portBytes := make([]byte, 2)
+		// port is always positive so conversion is allowable
+		binary.LittleEndian.PutUint16(portBytes, uint16(streamPair.port))
+		streamPair.dataStream.Write(portBytes)
+		streamPair.errorStream.Write(portBytes)
+	}
+	h := &websocketStreamHandler{
+		conn:        conn,
+		streamPairs: streamPairs,
+		pod:         podName,
+		uid:         uid,
+		forwarder:   portForwarder,
+	}
+	h.run()
+
+	return nil
+}
+
+// websocketStreamPair represents the error and data streams for a port
+// forwarding request.
+type websocketStreamPair struct {
+	port        int32
+	dataStream  io.ReadWriteCloser
+	errorStream io.WriteCloser
+}
+
+// websocketStreamHandler is capable of processing a single port forward
+// request over a websocket connection
+type websocketStreamHandler struct {
+	conn        *wsstream.Conn
+	streamPairs []*websocketStreamPair
+	pod         string
+	uid         types.UID
+	forwarder   PortForwarder
+}
+
+// run invokes the websocketStreamHandler's forwarder.PortForward
+// function for the given stream pair.
+func (h *websocketStreamHandler) run() {
+	wg := sync.WaitGroup{}
+	wg.Add(len(h.streamPairs))
+
+	for _, pair := range h.streamPairs {
+		p := pair
+		go func() {
+			defer wg.Done()
+			h.portForward(p)
+		}()
+	}
+
+	wg.Wait()
+}
+
+func (h *websocketStreamHandler) portForward(p *websocketStreamPair) {
+	ctx := context.Background()
+	defer p.dataStream.Close()
+	defer p.errorStream.Close()
+
+	klog.V(5).InfoS("Connection invoking forwarder.PortForward for port", "connection", h.conn, "port", p.port)
+	err := h.forwarder.PortForward(ctx, h.pod, h.uid, p.port, p.dataStream)
+	klog.V(5).InfoS("Connection done invoking forwarder.PortForward for port", "connection", h.conn, "port", p.port)
+
+	if err != nil {
+		msg := fmt.Errorf("error forwarding port %d to pod %s, uid %v: %v", p.port, h.pod, h.uid, err)
+		runtime.HandleError(msg)
+		fmt.Fprint(p.errorStream, msg.Error())
+	}
+}

internal/kubernetes/portforward/websocket_test.go

@@ -0,0 +1,101 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package portforward
+
+import (
+	"net/http"
+	"reflect"
+	"testing"
+)
+
+func TestV4Options(t *testing.T) {
+	tests := map[string]struct {
+		url           string
+		websocket     bool
+		expectedOpts  *V4Options
+		expectedError string
+	}{
+		"non-ws request": {
+			url:          "http://example.com",
+			expectedOpts: &V4Options{},
+		},
+		"missing port": {
+			url:           "http://example.com",
+			websocket:     true,
+			expectedError: `query parameter "port" is required`,
+		},
+		"unable to parse port": {
+			url:           "http://example.com?port=abc",
+			websocket:     true,
+			expectedError: `unable to parse "abc" as a port: strconv.ParseUint: parsing "abc": invalid syntax`,
+		},
+		"negative port": {
+			url:           "http://example.com?port=-1",
+			websocket:     true,
+			expectedError: `unable to parse "-1" as a port: strconv.ParseUint: parsing "-1": invalid syntax`,
+		},
+		"one port": {
+			url:       "http://example.com?port=80",
+			websocket: true,
+			expectedOpts: &V4Options{
+				Ports: []int32{80},
+			},
+		},
+		"multiple ports": {
+			url:       "http://example.com?port=80,90,100",
+			websocket: true,
+			expectedOpts: &V4Options{
+				Ports: []int32{80, 90, 100},
+			},
+		},
+		"multiple port": {
+			url:       "http://example.com?port=80&port=90",
+			websocket: true,
+			expectedOpts: &V4Options{
+				Ports: []int32{80, 90},
+			},
+		},
+	}
+	for name, test := range tests {
+		req, err := http.NewRequest(http.MethodGet, test.url, nil)
+		if err != nil {
+			t.Errorf("%s: invalid url %q err=%q", name, test.url, err)
+			continue
+		}
+		if test.websocket {
+			req.Header.Set("Connection", "Upgrade")
+			req.Header.Set("Upgrade", "websocket")
+		}
+		opts, err := NewV4Options(req)
+		if len(test.expectedError) > 0 {
+			if err == nil {
+				t.Errorf("%s: expected err=%q, but it was nil", name, test.expectedError)
+			}
+			if e, a := test.expectedError, err.Error(); e != a {
+				t.Errorf("%s: expected err=%q, got %q", name, e, a)
+			}
+			continue
+		}
+		if err != nil {
+			t.Errorf("%s: unexpected error %v", name, err)
+			continue
+		}
+		if !reflect.DeepEqual(test.expectedOpts, opts) {
+			t.Errorf("%s: expected options %#v, got %#v", name, test.expectedOpts, err)
+		}
+	}
+}

internal/kubernetes/remotecommand/attach.go

@@ -0,0 +1,79 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package remotecommand
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"time"
+
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	remotecommandconsts "k8s.io/apimachinery/pkg/util/remotecommand"
+	"k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/client-go/tools/remotecommand"
+	utilexec "k8s.io/utils/exec"
+)
+
+// Attacher knows how to attach to a container in a pod.
+type Attacher interface {
+	// AttachToContainer attaches to a container in the pod, copying data
+	// between in/out/err and the container's stdin/stdout/stderr.
+	AttachToContainer(name string, uid types.UID, container string, in io.Reader, out, err io.WriteCloser, tty bool, resize <-chan remotecommand.TerminalSize, timeout time.Duration) error
+}
+
+// ServeAttach handles requests to attach to a container. After
+// creating/receiving the required streams, it delegates the actual attachment
+// to the attacher.
+func ServeAttach(w http.ResponseWriter, req *http.Request, attacher Attacher, podName string, uid types.UID, container string, streamOpts *Options, idleTimeout, streamCreationTimeout time.Duration, supportedProtocols []string) {
+	ctx, ok := createStreams(req, w, streamOpts, supportedProtocols, idleTimeout, streamCreationTimeout)
+	if !ok {
+		// error is handled by createStreams
+		return
+	}
+	defer ctx.conn.Close()
+
+	err := attacher.AttachToContainer(podName, uid, container, ctx.stdinStream, ctx.stdoutStream, ctx.stderrStream, ctx.tty, ctx.resizeChan, 0)
+	if err != nil {
+		if exitErr, ok := err.(utilexec.ExitError); ok && exitErr.Exited() {
+			rc := exitErr.ExitStatus()
+			ctx.writeStatus(&apierrors.StatusError{ErrStatus: metav1.Status{
+				Status: metav1.StatusFailure,
+				Reason: remotecommandconsts.NonZeroExitCodeReason,
+				Details: &metav1.StatusDetails{
+					Causes: []metav1.StatusCause{
+						{
+							Type:    remotecommandconsts.ExitCodeCauseType,
+							Message: fmt.Sprintf("%d", rc),
+						},
+					},
+				},
+				Message: fmt.Sprintf("command terminated with non-zero exit code: %v", exitErr),
+			}})
+			return
+		}
+		err = fmt.Errorf("error attaching to container: %v", err)
+		runtime.HandleError(err)
+		ctx.writeStatus(apierrors.NewInternalError(err))
+		return
+	}
+	ctx.writeStatus(&apierrors.StatusError{ErrStatus: metav1.Status{
+		Status: metav1.StatusSuccess,
+	}})
+}

internal/kubernetes/remotecommand/httpstream.go

@@ -148,6 +148,8 @@ func createHTTPStreamStreams(req *http.Request, w http.ResponseWriter, opts *Opt
 
 	var handler protocolHandler
 	switch protocol {
+	case remotecommandconsts.StreamProtocolV5Name:
+		handler = &v5ProtocolHandler{}
 	case remotecommandconsts.StreamProtocolV4Name:
 		handler = &v4ProtocolHandler{}
 	case remotecommandconsts.StreamProtocolV3Name:
@@ -159,6 +161,9 @@ func createHTTPStreamStreams(req *http.Request, w http.ResponseWriter, opts *Opt
 		fallthrough
 	case remotecommandconsts.StreamProtocolV1Name:
 		handler = &v1ProtocolHandler{}
+	default:
+		klog.Errorf("unable to create HTTP stream: unknown protocol %q", protocol)
+		return nil, false
 	}
 
 	// count the streams client asked for, starting with 1
@@ -199,6 +204,57 @@ type protocolHandler interface {
 	supportsTerminalResizing() bool
 }
 
+// v5ProtocolHandler implements the V5 protocol version for streaming command execution.
+type v5ProtocolHandler struct{}
+
+func (*v5ProtocolHandler) waitForStreams(streams <-chan streamAndReply, expectedStreams int, expired <-chan time.Time) (*context, error) {
+	ctx := &context{}
+	receivedStreams := 0
+	replyChan := make(chan struct{})
+	stop := make(chan struct{})
+	defer close(stop)
+WaitForStreams:
+	for {
+		select {
+		case stream := <-streams:
+			streamType := stream.Headers().Get(api.StreamType)
+			switch streamType {
+			case api.StreamTypeError:
+				ctx.writeStatus = v4WriteStatusFunc(stream) // write json errors
+				go waitStreamReply(stream.replySent, replyChan, stop)
+			case api.StreamTypeStdin:
+				ctx.stdinStream = stream
+				go waitStreamReply(stream.replySent, replyChan, stop)
+			case api.StreamTypeStdout:
+				ctx.stdoutStream = stream
+				go waitStreamReply(stream.replySent, replyChan, stop)
+			case api.StreamTypeStderr:
+				ctx.stderrStream = stream
+				go waitStreamReply(stream.replySent, replyChan, stop)
+			case api.StreamTypeResize:
+				ctx.resizeStream = stream
+				go waitStreamReply(stream.replySent, replyChan, stop)
+			default:
+				runtime.HandleError(fmt.Errorf("unexpected stream type: %q", streamType))
+			}
+		case <-replyChan:
+			receivedStreams++
+			if receivedStreams == expectedStreams {
+				break WaitForStreams
+			}
+		case <-expired:
+			// TODO find a way to return the error to the user. Maybe use a separate
+			// stream to report errors?
+			return nil, errors.New("timed out waiting for client to create streams")
+		}
+	}
+
+	return ctx, nil
+}
+
+// supportsTerminalResizing returns true because v5ProtocolHandler supports it
+func (*v5ProtocolHandler) supportsTerminalResizing() bool { return true }
+
 // v4ProtocolHandler implements the V4 protocol version for streaming command execution. It only differs
 // in from v3 in the error stream format using an json-marshaled metav1.Status which carries
 // the process' exit code.

internal/lockdeps/lockdeps.go

@@ -1,9 +0,0 @@
-package lockdeps
-
-import (
-	// TODO(Sargun): Remove in Go1.13
-	// This is a dep that `go mod tidy` keeps removing, because it's a transitive dep that's pulled in via a test
-	// See: https://github.com/golang/go/issues/29702
-	_ "github.com/prometheus/client_golang/prometheus"
-	_ "golang.org/x/sys/unix"
-)

internal/manager/resource.go

@@ -32,7 +32,12 @@ type ResourceManager struct {
 }
 
 // NewResourceManager returns a ResourceManager with the internal maps initialized.
-func NewResourceManager(podLister corev1listers.PodLister, secretLister corev1listers.SecretLister, configMapLister corev1listers.ConfigMapLister, serviceLister corev1listers.ServiceLister) (*ResourceManager, error) {
+func NewResourceManager(
+	podLister corev1listers.PodLister,
+	secretLister corev1listers.SecretLister,
+	configMapLister corev1listers.ConfigMapLister,
+	serviceLister corev1listers.ServiceLister,
+) (*ResourceManager, error) {
 	rm := ResourceManager{
 		podLister:       podLister,
 		secretLister:    secretLister,

internal/podutils/env.go

@@ -29,7 +29,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/sets"
 	apivalidation "k8s.io/apimachinery/pkg/util/validation"
 	"k8s.io/client-go/tools/record"
-	"k8s.io/utils/pointer"
+	"k8s.io/utils/ptr"
 )
 
 const (
@@ -334,7 +334,7 @@ func getEnvironmentVariableValue(ctx context.Context, env *corev1.EnvVar, mappin
 		return getEnvironmentVariableValueWithValueFrom(ctx, env, mappingFunc, pod, container, rm, recorder)
 	}
 	// Handle values that have been directly provided after expanding variable references.
-	return pointer.StringPtr(expansion.Expand(env.Value, mappingFunc)), nil
+	return ptr.To(expansion.Expand(env.Value, mappingFunc)), nil
 }
 
 func getEnvironmentVariableValueWithValueFrom(ctx context.Context, env *corev1.EnvVar, mappingFunc func(string) string, pod *corev1.Pod, container *corev1.Container, rm *manager.ResourceManager, recorder record.EventRecorder) (*string, error) {
@@ -411,7 +411,7 @@ func getEnvironmentVariableValueWithValueFromConfigMapKeyRef(ctx context.Context
 		return nil, fmt.Errorf("configmap %q doesn't contain the %q key required by pod %s", vf.Name, vf.Key, pod.Name)
 	}
 	// Populate the environment variable and continue on to the next reference.
-	return pointer.StringPtr(keyValue), nil
+	return ptr.To(keyValue), nil
 }
 
 func getEnvironmentVariableValueWithValueFromSecretKeyRef(ctx context.Context, env *corev1.EnvVar, mappingFunc func(string) string, pod *corev1.Pod, container *corev1.Container, rm *manager.ResourceManager, recorder record.EventRecorder) (*string, error) {
@@ -463,7 +463,7 @@ func getEnvironmentVariableValueWithValueFromSecretKeyRef(ctx context.Context, e
 		return nil, fmt.Errorf("secret %q doesn't contain the %q key required by pod %s", vf.Name, vf.Key, pod.Name)
 	}
 	// Populate the environment variable and continue on to the next reference.
-	return pointer.StringPtr(string(keyValue)), nil
+	return ptr.To(string(keyValue)), nil
 }
 
 // Handle population from a field (downward API).
@@ -476,7 +476,7 @@ func getEnvironmentVariableValueWithValueFromFieldRef(ctx context.Context, env *
 		return nil, err
 	}
 
-	return pointer.StringPtr(runtimeVal), nil
+	return ptr.To(runtimeVal), nil
 }
 
 // podFieldSelectorRuntimeValue returns the runtime value of the given

internal/podutils/helper.go

@@ -111,15 +111,15 @@ func ExtractFieldPathAsString(obj interface{}, fieldPath string) (string, error)
 
 // SplitMaybeSubscriptedPath checks whether the specified fieldPath is
 // subscripted, and
-//  - if yes, this function splits the fieldPath into path and subscript, and
-//    returns (path, subscript, true).
-//  - if no, this function returns (fieldPath, "", false).
+//   - if yes, this function splits the fieldPath into path and subscript, and
+//     returns (path, subscript, true).
+//   - if no, this function returns (fieldPath, "", false).
 //
 // Example inputs and outputs:
-//  - "metadata.annotations['myKey']" --> ("metadata.annotations", "myKey", true)
-//  - "metadata.annotations['a[b]c']" --> ("metadata.annotations", "a[b]c", true)
-//  - "metadata.labels['']"           --> ("metadata.labels", "", true)
-//  - "metadata.labels"               --> ("metadata.labels", "", false)
+//   - "metadata.annotations['myKey']" --> ("metadata.annotations", "myKey", true)
+//   - "metadata.annotations['a[b]c']" --> ("metadata.annotations", "a[b]c", true)
+//   - "metadata.labels[”]"           --> ("metadata.labels", "", true)
+//   - "metadata.labels"               --> ("metadata.labels", "", false)
 func SplitMaybeSubscriptedPath(fieldPath string) (string, string, bool) {
 	if !strings.HasSuffix(fieldPath, "']") {
 		return fieldPath, "", false

internal/queue/queue.go

@@ -51,7 +51,7 @@ type Queue struct {
 	name    string
 	handler ItemHandler
 
-	ratelimiter workqueue.RateLimiter
+	ratelimiter workqueue.TypedRateLimiter[any]
 	// items are items that are marked dirty waiting for processing.
 	items *list.List
 	// itemInQueue is a map of (string) key -> item while it is in the items list
@@ -90,7 +90,7 @@ func (item *queueItem) String() string {
 //
 // It expects to get a item rate limiter, and a friendly name which is used in logs, and in the internal kubernetes
 // metrics. If retryFunc is nil, the default retry function.
-func New(ratelimiter workqueue.RateLimiter, name string, handler ItemHandler, retryFunc ShouldRetryFunc) *Queue {
+func New(ratelimiter workqueue.TypedRateLimiter[any], name string, handler ItemHandler, retryFunc ShouldRetryFunc) *Queue {
 	if retryFunc == nil {
 		retryFunc = DefaultRetryFunc
 	}
@@ -288,6 +288,24 @@ func (q *Queue) Len() int {
 	return q.items.Len() + len(q.itemsBeingProcessed)
 }
 
+// UnprocessedLen returns the count of items yet to be processed in the queue
+func (q *Queue) UnprocessedLen() int {
+	q.lock.Lock()
+	defer q.lock.Unlock()
+	if q.items.Len() != len(q.itemsInQueue) {
+		panic("Internally inconsistent state")
+	}
+
+	return len(q.itemsInQueue)
+}
+
+// ProcessedLen returns the count items that are being processed
+func (q *Queue) ItemsBeingProcessedLen() int {
+	q.lock.Lock()
+	defer q.lock.Unlock()
+	return len(q.itemsBeingProcessed)
+}
+
 // Run starts the workers
 //
 // It blocks until context is cancelled, and all of the workers exit.

internal/queue/queue_test.go

@@ -35,10 +35,10 @@ func TestQueueMaxRetries(t *testing.T) {
 		n++
 		return knownErr
 	}
-	wq := New(workqueue.NewMaxOfRateLimiter(
+	wq := New(workqueue.NewTypedMaxOfRateLimiter[any](
 		// The default upper bound is 1000 seconds. Let's not use that.
-		workqueue.NewItemExponentialFailureRateLimiter(5*time.Millisecond, 10*time.Millisecond),
-		&workqueue.BucketRateLimiter{Limiter: rate.NewLimiter(rate.Limit(10), 100)},
+		workqueue.NewTypedItemExponentialFailureRateLimiter[any](5*time.Millisecond, 10*time.Millisecond),
+		&workqueue.TypedBucketRateLimiter[any]{Limiter: rate.NewLimiter(rate.Limit(10), 100)},
 	), t.Name(), handler, nil)
 	wq.Enqueue(context.TODO(), "test")
 
@@ -77,7 +77,7 @@ func TestQueueCustomRetries(t *testing.T) {
 		return sleepTime, retErr
 	}
 
-	wq := New(&workqueue.BucketRateLimiter{Limiter: rate.NewLimiter(rate.Limit(1000), 1000)}, t.Name(), handler, shouldRetryFunc)
+	wq := New(&workqueue.TypedBucketRateLimiter[any]{Limiter: rate.NewLimiter(rate.Limit(1000), 1000)}, t.Name(), handler, shouldRetryFunc)
 
 	timeTaken := func(key string) time.Duration {
 		start := time.Now()
@@ -106,7 +106,7 @@ func TestForget(t *testing.T) {
 	handler := func(ctx context.Context, key string) error {
 		panic("Should never be called")
 	}
-	wq := New(workqueue.DefaultItemBasedRateLimiter(), t.Name(), handler, nil)
+	wq := New(workqueue.DefaultTypedItemBasedRateLimiter[any](), t.Name(), handler, nil)
 
 	wq.Forget(context.TODO(), "val")
 	assert.Assert(t, is.Equal(0, wq.Len()))
@@ -121,7 +121,7 @@ func TestQueueEmpty(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
 	defer cancel()
 
-	q := New(workqueue.DefaultItemBasedRateLimiter(), t.Name(), func(ctx context.Context, key string) error {
+	q := New(workqueue.DefaultTypedItemBasedRateLimiter[any](), t.Name(), func(ctx context.Context, key string) error {
 		return nil
 	}, nil)
 
@@ -136,7 +136,7 @@ func TestQueueItemNoSleep(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), 1000*time.Millisecond)
 	defer cancel()
 
-	q := New(workqueue.DefaultItemBasedRateLimiter(), t.Name(), func(ctx context.Context, key string) error {
+	q := New(workqueue.DefaultTypedItemBasedRateLimiter[any](), t.Name(), func(ctx context.Context, key string) error {
 		return nil
 	}, nil)
 
@@ -160,7 +160,7 @@ func TestQueueItemSleep(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), 1000*time.Millisecond)
 	defer cancel()
 
-	q := New(workqueue.DefaultItemBasedRateLimiter(), t.Name(), func(ctx context.Context, key string) error {
+	q := New(workqueue.DefaultTypedItemBasedRateLimiter[any](), t.Name(), func(ctx context.Context, key string) error {
 		return nil
 	}, nil)
 	q.lock.Lock()
@@ -179,7 +179,7 @@ func TestQueueBackgroundAdd(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), 5000*time.Millisecond)
 	defer cancel()
 
-	q := New(workqueue.DefaultItemBasedRateLimiter(), t.Name(), func(ctx context.Context, key string) error {
+	q := New(workqueue.DefaultTypedItemBasedRateLimiter[any](), t.Name(), func(ctx context.Context, key string) error {
 		return nil
 	}, nil)
 	start := time.Now()
@@ -201,7 +201,7 @@ func TestQueueBackgroundAdvance(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), 5000*time.Millisecond)
 	defer cancel()
 
-	q := New(workqueue.DefaultItemBasedRateLimiter(), t.Name(), func(ctx context.Context, key string) error {
+	q := New(workqueue.DefaultTypedItemBasedRateLimiter[any](), t.Name(), func(ctx context.Context, key string) error {
 		return nil
 	}, nil)
 	start := time.Now()
@@ -230,7 +230,7 @@ func TestQueueRedirty(t *testing.T) {
 
 	var times int64
 	var q *Queue
-	q = New(workqueue.DefaultItemBasedRateLimiter(), t.Name(), func(ctx context.Context, key string) error {
+	q = New(workqueue.DefaultTypedItemBasedRateLimiter[any](), t.Name(), func(ctx context.Context, key string) error {
 		assert.Assert(t, is.Equal(key, "foo"))
 		if atomic.AddInt64(&times, 1) == 1 {
 			q.EnqueueWithoutRateLimit(context.TODO(), "foo")
@@ -256,7 +256,7 @@ func TestHeapConcurrency(t *testing.T) {
 
 	start := time.Now()
 	seen := sync.Map{}
-	q := New(workqueue.DefaultItemBasedRateLimiter(), t.Name(), func(ctx context.Context, key string) error {
+	q := New(workqueue.DefaultTypedItemBasedRateLimiter[any](), t.Name(), func(ctx context.Context, key string) error {
 		seen.Store(key, struct{}{})
 		time.Sleep(time.Second)
 		return nil
@@ -291,7 +291,7 @@ func checkConsistency(t *testing.T, q *Queue) {
 }
 
 func TestHeapOrder(t *testing.T) {
-	q := New(workqueue.DefaultItemBasedRateLimiter(), t.Name(), func(ctx context.Context, key string) error {
+	q := New(workqueue.DefaultTypedItemBasedRateLimiter[any](), t.Name(), func(ctx context.Context, key string) error {
 		return nil
 	}, nil)
 	q.clock = nonmovingClock{}
@@ -317,7 +317,7 @@ func TestHeapOrder(t *testing.T) {
 type rateLimitWrapper struct {
 	addedMap     sync.Map
 	forgottenMap sync.Map
-	rl           workqueue.RateLimiter
+	rl           workqueue.TypedRateLimiter[any]
 }
 
 func (r *rateLimitWrapper) When(item interface{}) time.Duration {
@@ -356,7 +356,7 @@ func TestRateLimiter(t *testing.T) {
 
 	start := time.Now()
 	ratelimiter := &rateLimitWrapper{
-		rl: workqueue.NewItemFastSlowRateLimiter(1*time.Millisecond, 100*time.Millisecond, 1),
+		rl: workqueue.NewTypedItemFastSlowRateLimiter[any](1*time.Millisecond, 100*time.Millisecond, 1),
 	}
 
 	q := New(ratelimiter, t.Name(), func(ctx context.Context, key string) error {
@@ -421,7 +421,7 @@ func TestQueueForgetInProgress(t *testing.T) {
 
 	var times int64
 	var q *Queue
-	q = New(workqueue.DefaultItemBasedRateLimiter(), t.Name(), func(ctx context.Context, key string) error {
+	q = New(workqueue.DefaultTypedItemBasedRateLimiter[any](), t.Name(), func(ctx context.Context, key string) error {
 		assert.Assert(t, is.Equal(key, "foo"))
 		atomic.AddInt64(&times, 1)
 		q.Forget(context.TODO(), key)
@@ -441,7 +441,7 @@ func TestQueueForgetBeforeStart(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	q := New(workqueue.DefaultItemBasedRateLimiter(), t.Name(), func(ctx context.Context, key string) error {
+	q := New(workqueue.DefaultTypedItemBasedRateLimiter[any](), t.Name(), func(ctx context.Context, key string) error {
 		panic("shouldn't be called")
 	}, nil)
 
@@ -458,7 +458,7 @@ func TestQueueMoveItem(t *testing.T) {
 
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	q := New(workqueue.DefaultItemBasedRateLimiter(), t.Name(), func(ctx context.Context, key string) error {
+	q := New(workqueue.DefaultTypedItemBasedRateLimiter[any](), t.Name(), func(ctx context.Context, key string) error {
 		panic("shouldn't be called")
 	}, nil)
 	q.clock = nonmovingClock{}

internal/test/e2e/framework/pod.go

@@ -6,7 +6,6 @@ import (
 	"strings"
 
 	corev1 "k8s.io/api/core/v1"
-	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -106,9 +105,9 @@ func (f *Framework) WaitUntilPodReady(namespace, name string) (*corev1.Pod, erro
 }
 
 // IsPodReady returns true if a pod is ready.
-func IsPodReady(pod *v1.Pod) bool {
+func IsPodReady(pod *corev1.Pod) bool {
 	for _, cond := range pod.Status.Conditions {
-		if cond.Type == v1.PodReady && cond.Status == v1.ConditionTrue {
+		if cond.Type == corev1.PodReady && cond.Status == corev1.ConditionTrue {
 			return true
 		}
 	}
@@ -119,7 +118,7 @@ func IsPodReady(pod *v1.Pod) bool {
 func (f *Framework) WaitUntilPodDeleted(namespace, name string) (*corev1.Pod, error) {
 	return f.WaitUntilPodCondition(namespace, name, func(event watchapi.Event) (bool, error) {
 		pod := event.Object.(*corev1.Pod)
-		return event.Type == watchapi.Deleted || pod.ObjectMeta.DeletionTimestamp != nil, nil
+		return event.Type == watchapi.Deleted || pod.DeletionTimestamp != nil, nil
 	})
 }
 

internal/test/e2e/framework/stats.go

@@ -4,8 +4,9 @@ import (
 	"context"
 	"encoding/json"
 
-	stats "github.com/virtual-kubelet/virtual-kubelet/node/api/statsv1alpha1"
+	api "github.com/virtual-kubelet/virtual-kubelet/node/api"
 	"k8s.io/apimachinery/pkg/util/net"
+	stats "k8s.io/kubelet/pkg/apis/stats/v1alpha1"
 )
 
 // GetStatsSummary queries the /stats/summary endpoint of the virtual-kubelet and returns the Summary object obtained as a response.
@@ -29,3 +30,21 @@ func (f *Framework) GetStatsSummary(ctx context.Context) (*stats.Summary, error)
 	}
 	return res, nil
 }
+
+// GetStatsSummary queries the /metrics/resource endpoint of the virtual-kubelet and returns the Summary object obtained as a response.
+func (f *Framework) GetMetricsResource(ctx context.Context) ([]byte, error) {
+	// Query the /stats/summary endpoint.
+	b, err := f.KubeClient.CoreV1().
+		RESTClient().
+		Get().
+		Namespace(f.Namespace).
+		Resource("pods").
+		SubResource("proxy").
+		Name(net.JoinSchemeNamePort("https", f.NodeName, "10250")).
+		Suffix(api.MetricsResourceRouteSuffix).DoRaw(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	return b, nil
+}

internal/test/e2e/main_test.go

@@ -1,3 +1,4 @@
+//go:build e2e
 // +build e2e
 
 package e2e
@@ -48,7 +49,7 @@ func teardown() error {
 	return nil
 }
 
-//  Provider-specific shouldSkipTest function
+// Provider-specific shouldSkipTest function
 func shouldSkipTest(testName string) bool {
 	return false
 }

log/slog/slog.go

@@ -0,0 +1,109 @@
+// Copyright © 2021 The virtual-kubelet authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Package slog implements a virtual-kubelet/log.Logger using slog as a backend
+// You can use this by creating a slog logger and calling `FromSlog(logger)`
+// If you want this to be the default logger for virtual-kubelet, set `log.L` to the value returned by `FromSlog(logger)`
+package slog
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+
+	"github.com/virtual-kubelet/virtual-kubelet/log"
+)
+
+// Ensure log.Logger is fully implemented during compile time.
+var _ log.Logger = (*adapter)(nil)
+
+// Create a custom logging level for the Fatal level as slog does not
+// have this level by default
+const LevelFatal = slog.Level(12)
+
+// adapter implements the `log.Logger` interface for slog
+type adapter struct {
+	inner *slog.Logger
+}
+
+// FromSlog creates a new `log.Logger` from a slog logger
+func FromSlog(logger *slog.Logger) log.Logger {
+	return &adapter{inner: logger}
+}
+
+func (l *adapter) Debug(args ...interface{}) {
+	msg := args[0].(string)
+	l.inner.Debug(msg)
+}
+
+func (l *adapter) Debugf(format string, args ...interface{}) {
+	formattedArgs := fmt.Sprintf(format, args...)
+	l.inner.Debug(formattedArgs)
+}
+
+func (l *adapter) Info(args ...interface{}) {
+	msg := args[0].(string)
+	l.inner.Info(msg)
+}
+
+func (l *adapter) Infof(format string, args ...interface{}) {
+	formattedArgs := fmt.Sprintf(format, args...)
+	l.inner.Info(formattedArgs)
+}
+
+func (l *adapter) Warn(args ...interface{}) {
+	msg := args[0].(string)
+	l.inner.Warn(msg)
+}
+
+func (l *adapter) Warnf(format string, args ...interface{}) {
+	formattedArgs := fmt.Sprintf(format, args...)
+	l.inner.Warn(formattedArgs)
+}
+
+func (l *adapter) Error(args ...interface{}) {
+	msg := args[0].(string)
+	l.inner.Error(msg)
+}
+
+func (l *adapter) Errorf(format string, args ...interface{}) {
+	formattedArgs := fmt.Sprintf(format, args...)
+	l.inner.Error(formattedArgs)
+}
+
+func (l *adapter) Fatal(args ...interface{}) {
+	msg := args[0].(string)
+	l.inner.Log(context.Background(), LevelFatal, msg)
+}
+
+func (l *adapter) Fatalf(format string, args ...interface{}) {
+	formattedArgs := fmt.Sprintf(format, args...)
+	l.inner.Log(context.Background(), LevelFatal, formattedArgs)
+}
+
+func (l *adapter) WithField(key string, val interface{}) log.Logger {
+	return &adapter{inner: l.inner.With(key, val)}
+}
+
+func (l *adapter) WithFields(f log.Fields) log.Logger {
+	logger := l.inner
+	for k, v := range f {
+		logger = logger.With(k, v)
+	}
+	return &adapter{inner: logger}
+}
+
+func (l *adapter) WithError(err error) log.Logger {
+	return &adapter{inner: l.inner.With("error", err)}
+}

node/api/attach.go

@@ -0,0 +1,148 @@
+// Copyright © 2017 The virtual-kubelet authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package api
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/gorilla/mux"
+	"github.com/virtual-kubelet/virtual-kubelet/errdefs"
+	"github.com/virtual-kubelet/virtual-kubelet/internal/kubernetes/remotecommand"
+	"k8s.io/apimachinery/pkg/types"
+	remoteutils "k8s.io/client-go/tools/remotecommand"
+)
+
+// ContainerAttachHandlerFunc defines the handler function used for "execing" into a
+// container in a pod.
+type ContainerAttachHandlerFunc func(ctx context.Context, namespace, podName, containerName string, attach AttachIO) error
+
+// HandleContainerAttach makes an http handler func from a Provider which execs a command in a pod's container
+// Note that this handler currently depends on gorrilla/mux to get url parts as variables.
+// TODO(@cpuguy83): don't force gorilla/mux on consumers of this function
+func HandleContainerAttach(h ContainerAttachHandlerFunc, opts ...ContainerExecHandlerOption) http.HandlerFunc {
+	if h == nil {
+		return NotImplemented
+	}
+
+	var cfg ContainerExecHandlerConfig
+	for _, o := range opts {
+		o(&cfg)
+	}
+
+	if cfg.StreamIdleTimeout == 0 {
+		cfg.StreamIdleTimeout = 30 * time.Second
+	}
+	if cfg.StreamCreationTimeout == 0 {
+		cfg.StreamCreationTimeout = 30 * time.Second
+	}
+
+	return handleError(func(w http.ResponseWriter, req *http.Request) error {
+		vars := mux.Vars(req)
+
+		namespace := vars["namespace"]
+		pod := vars["pod"]
+		container := vars["container"]
+
+		supportedStreamProtocols := strings.Split(req.Header.Get("X-Stream-Protocol-Version"), ",")
+
+		streamOpts, err := getExecOptions(req)
+		if err != nil {
+			return errdefs.AsInvalidInput(err)
+		}
+
+		ctx, cancel := context.WithCancel(req.Context())
+		defer cancel()
+
+		attach := &containerAttachContext{ctx: ctx, h: h, pod: pod, namespace: namespace, container: container}
+		remotecommand.ServeAttach(
+			w,
+			req,
+			attach,
+			"",
+			"",
+			container,
+			streamOpts,
+			cfg.StreamIdleTimeout,
+			cfg.StreamCreationTimeout,
+			supportedStreamProtocols,
+		)
+
+		return nil
+	})
+}
+
+type containerAttachContext struct {
+	h                         ContainerAttachHandlerFunc
+	namespace, pod, container string
+	ctx                       context.Context
+}
+
+// AttachToContainer Implements remotecommand.Attacher
+// This is called by remotecommand.ServeAttach
+func (c *containerAttachContext) AttachToContainer(
+	name string,
+	uid types.UID,
+	container string,
+	in io.Reader,
+	out, err io.WriteCloser,
+	tty bool,
+	resize <-chan remoteutils.TerminalSize,
+	timeout time.Duration,
+) error {
+
+	eio := &execIO{
+		tty:    tty,
+		stdin:  in,
+		stdout: out,
+		stderr: err,
+	}
+
+	if tty {
+		eio.chResize = make(chan TermSize)
+	}
+
+	ctx, cancel := context.WithCancel(c.ctx)
+	defer cancel()
+
+	if tty {
+		go func() {
+			send := func(s remoteutils.TerminalSize) bool {
+				select {
+				case eio.chResize <- TermSize{Width: s.Width, Height: s.Height}:
+					return false
+				case <-ctx.Done():
+					return true
+				}
+			}
+
+			for {
+				select {
+				case s := <-resize:
+					if send(s) {
+						return
+					}
+				case <-ctx.Done():
+					return
+				}
+			}
+		}()
+	}
+
+	return c.h(c.ctx, c.namespace, c.pod, c.container, eio)
+}

node/api/exec.go

@@ -171,7 +171,17 @@ type containerExecContext struct {
 
 // ExecInContainer Implements remotecommand.Executor
 // This is called by remotecommand.ServeExec
-func (c *containerExecContext) ExecInContainer(name string, uid types.UID, container string, cmd []string, in io.Reader, out, err io.WriteCloser, tty bool, resize <-chan remoteutils.TerminalSize, timeout time.Duration) error {
+func (c *containerExecContext) ExecInContainer(
+	name string,
+	uid types.UID,
+	container string,
+	cmd []string,
+	in io.Reader,
+	out, err io.WriteCloser,
+	tty bool,
+	resize <-chan remoteutils.TerminalSize,
+	timeout time.Duration,
+) error {
 
 	eio := &execIO{
 		tty:    tty,

node/api/metrics.go

@@ -0,0 +1,67 @@
+// Copyright © 2017 The virtual-kubelet authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package api
+
+import (
+	"bytes"
+	"context"
+	"net/http"
+
+	"github.com/pkg/errors"
+	dto "github.com/prometheus/client_model/go"
+	"github.com/prometheus/common/expfmt"
+)
+
+const (
+	PrometheusTextFormatContentType = "text/plain; version=0.0.4"
+)
+
+// PodMetricsResourceHandlerFunc defines the handler for getting pod metrics
+type PodMetricsResourceHandlerFunc func(context.Context) ([]*dto.MetricFamily, error)
+
+// HandlePodMetricsResource makes an HTTP handler for implementing the kubelet /metrics/resource endpoint
+func HandlePodMetricsResource(h PodMetricsResourceHandlerFunc) http.HandlerFunc {
+	if h == nil {
+		return NotImplemented
+	}
+	return handleError(func(w http.ResponseWriter, req *http.Request) error {
+		metrics, err := h(req.Context())
+		if err != nil {
+			if isCancelled(err) {
+				return err
+			}
+			return errors.Wrap(err, "error getting status from provider")
+		}
+
+		// Convert metrics to Prometheus text format.
+		var buffer bytes.Buffer
+		enc := expfmt.NewEncoder(&buffer, expfmt.NewFormat(expfmt.TypeTextPlain))
+		for _, mf := range metrics {
+			if err := enc.Encode(mf); err != nil {
+				return errors.Wrap(err, "could not convert metrics to prometheus text format")
+			}
+		}
+
+		// Set the response content type to "text/plain; version=0.0.4".
+		w.Header().Set("Content-Type", PrometheusTextFormatContentType)
+
+		// Write the metrics in Prometheus text format to the response writer.
+		if _, err := w.Write(buffer.Bytes()); err != nil {
+			return errors.Wrap(err, "could not write to client")
+		}
+
+		return nil
+	})
+}

node/api/metrics_test.go

@@ -0,0 +1,149 @@
+// Copyright © 2017 The virtual-kubelet authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package api_test
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/pkg/errors"
+	dto "github.com/prometheus/client_model/go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/virtual-kubelet/virtual-kubelet/node/api"
+	"google.golang.org/protobuf/proto"
+)
+
+const (
+	prometheusContentType = "text/plain; version=0.0.4"
+)
+
+func TestHandlePodMetricsResource(t *testing.T) {
+	testCases := []struct {
+		name               string
+		handler            api.PodMetricsResourceHandlerFunc
+		expectedStatusCode int
+		expectedError      error
+	}{
+		{
+			name: "Valid PodMetricsResourceHandlerFunc",
+			handler: func(_ context.Context) ([]*dto.MetricFamily, error) {
+				// Create the expected metrics.
+				cpuUsageMetric := &dto.MetricFamily{
+					Name:   proto.String("container_cpu_usage_seconds_total"),
+					Help:   proto.String("[ALPHA] Cumulative cpu time consumed by the container in core-seconds"),
+					Type:   dto.MetricType_GAUGE.Enum(),
+					Metric: []*dto.Metric{},
+				}
+				memoryUsageMetric := &dto.MetricFamily{
+					Name:   proto.String("container_memory_working_set_bytes"),
+					Help:   proto.String("[ALPHA] Current working set of the container in bytes"),
+					Type:   dto.MetricType_GAUGE.Enum(),
+					Metric: []*dto.Metric{},
+				}
+
+				// Add the sample metrics to the metric families.
+				cpuUsageMetric.Metric = append(cpuUsageMetric.Metric, createSampleMetric(
+					map[string]string{
+						"container": "simple-hello-world-container",
+						"namespace": "k8se-apps",
+						"pod":       "test-pod--zruwatj-86454fdc54-2wwpw",
+					},
+					0.1104636, 1680536423102,
+				))
+				cpuUsageMetric.Metric = append(cpuUsageMetric.Metric, createSampleMetric(
+					map[string]string{
+						"container": "simple-hello-world-container",
+						"namespace": "k8se-apps",
+						"pod":       "test-pod--zruwatj-86454fdc54-4mzd4",
+					},
+					0.11322, 1680536423103,
+				))
+				memoryUsageMetric.Metric = append(memoryUsageMetric.Metric, createSampleMetric(
+					map[string]string{
+						"container": "simple-hello-world-container",
+						"namespace": "k8se-apps",
+						"pod":       "test-pod--zruwatj-86454fdc54-2wwpw",
+					},
+					2.3277568e+07, 1680536423102,
+				))
+				memoryUsageMetric.Metric = append(memoryUsageMetric.Metric, createSampleMetric(
+					map[string]string{
+						"container": "simple-hello-world-container",
+						"namespace": "k8se-apps",
+						"pod":       "test-pod--zruwatj-86454fdc54-4mzd4",
+					},
+					2.2450176e+07, 1680536423104,
+				))
+
+				return []*dto.MetricFamily{cpuUsageMetric, memoryUsageMetric}, nil
+			},
+			expectedStatusCode: http.StatusOK,
+		},
+		{
+			name:               "Nil PodMetricsResourceHandlerFunc",
+			handler:            nil,
+			expectedStatusCode: http.StatusNotImplemented,
+		},
+		{
+			name: "Error in PodMetricsResourceHandlerFunc",
+			handler: func(_ context.Context) ([]*dto.MetricFamily, error) {
+				return nil, errors.New("test error")
+			},
+			expectedStatusCode: http.StatusInternalServerError,
+			expectedError:      errors.New("error getting status from provider: test error"),
+		},
+		// Add more test cases as needed
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			h := api.HandlePodMetricsResource(tc.handler)
+			require.NotNil(t, h)
+
+			rr := httptest.NewRecorder()
+			req, err := http.NewRequest("GET", "/metrics/resource", nil)
+			require.NoError(t, err)
+
+			h.ServeHTTP(rr, req)
+
+			assert.Equal(t, tc.expectedStatusCode, rr.Code)
+
+			if tc.expectedError != nil {
+				bodyBytes, err := io.ReadAll(rr.Body)
+				require.NoError(t, err)
+				assert.Contains(t, string(bodyBytes), tc.expectedError.Error())
+			} else if tc.expectedStatusCode == http.StatusOK {
+				contentType := rr.Header().Get("Content-Type")
+				assert.Equal(t, prometheusContentType, contentType)
+			}
+		})
+	}
+}
+
+func createSampleMetric(labels map[string]string, value float64, timestamp int64) *dto.Metric {
+	labelPairs := []*dto.LabelPair{}
+	for k, v := range labels {
+		labelPairs = append(labelPairs, &dto.LabelPair{
+			Name:  proto.String(k),
+			Value: proto.String(v),
+		})
+	}
+
+	return &dto.Metric{Label: labelPairs, Gauge: &dto.Gauge{Value: &value}, TimestampMs: &timestamp}
+}

node/api/pods.go

@@ -24,9 +24,9 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/serializer"
 )
 
-type PodListerFunc func(context.Context) ([]*v1.Pod, error) //nolint:golint
+type PodListerFunc func(context.Context) ([]*v1.Pod, error)
 
-func HandleRunningPods(getPods PodListerFunc) http.HandlerFunc { //nolint:golint
+func HandleRunningPods(getPods PodListerFunc) http.HandlerFunc {
 	if getPods == nil {
 		return NotImplemented
 	}

node/api/portforward.go

@@ -0,0 +1,116 @@
+// Copyright © 2017 The virtual-kubelet authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package api
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/gorilla/mux"
+	"github.com/virtual-kubelet/virtual-kubelet/internal/kubernetes/portforward"
+	"k8s.io/apimachinery/pkg/types"
+)
+
+// PortForwardHandlerFunc defines the handler function used to
+// portforward, passing through the original dataStream
+type PortForwardHandlerFunc func(ctx context.Context, namespace, pod string, port int32, stream io.ReadWriteCloser) error
+
+// PortForwardHandlerConfig is used to pass options to options to the container exec handler.
+type PortForwardHandlerConfig struct {
+	// StreamIdleTimeout is the maximum time a streaming connection
+	// can be idle before the connection is automatically closed.
+	StreamIdleTimeout time.Duration
+	// StreamCreationTimeout is the maximum time for streaming connection
+	StreamCreationTimeout time.Duration
+}
+
+// PortForwardHandlerOption configures a PortForwardHandlerConfig
+// It is used as functional options passed to `HandlePortForward`
+type PortForwardHandlerOption func(*PortForwardHandlerConfig)
+
+// WithPortForwardStreamIdleTimeout sets the idle timeout for a container port forward streaming
+func WithPortForwardStreamIdleTimeout(dur time.Duration) PortForwardHandlerOption {
+	return func(cfg *PortForwardHandlerConfig) {
+		cfg.StreamIdleTimeout = dur
+	}
+}
+
+// WithPortForwardCreationTimeout sets the creation timeout for a container exec stream
+func WithPortForwardCreationTimeout(dur time.Duration) PortForwardHandlerOption {
+	return func(cfg *PortForwardHandlerConfig) {
+		cfg.StreamCreationTimeout = dur
+	}
+}
+
+// HandlePortForward makes an http handler func from a Provider which forward ports to a container
+// Note that this handler currently depends on gorrilla/mux to get url parts as variables.
+func HandlePortForward(h PortForwardHandlerFunc, opts ...PortForwardHandlerOption) http.HandlerFunc {
+	if h == nil {
+		return NotImplemented
+	}
+
+	var cfg PortForwardHandlerConfig
+	for _, o := range opts {
+		o(&cfg)
+	}
+
+	if cfg.StreamIdleTimeout == 0 {
+		cfg.StreamIdleTimeout = 30 * time.Second
+	}
+	if cfg.StreamCreationTimeout == 0 {
+		cfg.StreamCreationTimeout = 30 * time.Second
+	}
+
+	return handleError(func(w http.ResponseWriter, req *http.Request) error {
+		vars := mux.Vars(req)
+
+		namespace := vars["namespace"]
+
+		pod := vars["pod"]
+
+		supportedStreamProtocols := strings.Split(req.Header.Get("X-Stream-Protocol-Version"), ",")
+
+		portfwd := &portForwardContext{h: h, pod: pod, namespace: namespace}
+		portforward.ServePortForward(
+			w,
+			req,
+			portfwd,
+			pod,
+			"",
+			&portforward.V4Options{}, // This is only used for websocket connection
+			cfg.StreamIdleTimeout,
+			cfg.StreamCreationTimeout,
+			supportedStreamProtocols,
+		)
+
+		return nil
+	})
+
+}
+
+type portForwardContext struct {
+	h         PortForwardHandlerFunc
+	pod       string
+	namespace string
+}
+
+// PortForward Implements portforward.Portforwarder
+// This is called by portforward.ServePortForward
+func (p *portForwardContext) PortForward(ctx context.Context, name string, uid types.UID, port int32, stream io.ReadWriteCloser) error {
+	return p.h(ctx, p.namespace, p.pod, port, stream)
+}

node/api/server.go

@@ -34,17 +34,22 @@ type ServeMux interface {
 }
 
 type PodHandlerConfig struct { //nolint:golint
-	RunInContainer   ContainerExecHandlerFunc
-	GetContainerLogs ContainerLogsHandlerFunc
+	RunInContainer    ContainerExecHandlerFunc
+	AttachToContainer ContainerAttachHandlerFunc
+	PortForward       PortForwardHandlerFunc
+	GetContainerLogs  ContainerLogsHandlerFunc
 	// GetPods is meant to enumerate the pods that the provider knows about
 	GetPods PodListerFunc
 	// GetPodsFromKubernetes is meant to enumerate the pods that the node is meant to be running
 	GetPodsFromKubernetes PodListerFunc
 	GetStatsSummary       PodStatsSummaryHandlerFunc
+	GetMetricsResource    PodMetricsResourceHandlerFunc
 	StreamIdleTimeout     time.Duration
 	StreamCreationTimeout time.Duration
 }
 
+const MetricsResourceRouteSuffix = "/metrics/resource"
+
 // PodHandler creates an http handler for interacting with pods/containers.
 func PodHandler(p PodHandlerConfig, debug bool) http.Handler {
 	r := mux.NewRouter()
@@ -54,7 +59,6 @@ func PodHandler(p PodHandlerConfig, debug bool) http.Handler {
 	if debug {
 		r.HandleFunc("/runningpods/", HandleRunningPods(p.GetPods)).Methods("GET")
 	}
-
 	r.HandleFunc("/pods", HandleRunningPods(p.GetPodsFromKubernetes)).Methods("GET")
 	r.HandleFunc("/containerLogs/{namespace}/{pod}/{container}", HandleContainerLogs(p.GetContainerLogs)).Methods("GET")
 	r.HandleFunc(
@@ -65,6 +69,22 @@ func PodHandler(p PodHandlerConfig, debug bool) http.Handler {
 			WithExecStreamIdleTimeout(p.StreamIdleTimeout),
 		),
 	).Methods("POST", "GET")
+	r.HandleFunc(
+		"/attach/{namespace}/{pod}/{container}",
+		HandleContainerAttach(
+			p.AttachToContainer,
+			WithExecStreamCreationTimeout(p.StreamCreationTimeout),
+			WithExecStreamIdleTimeout(p.StreamIdleTimeout),
+		),
+	).Methods("POST", "GET")
+	r.HandleFunc(
+		"/portForward/{namespace}/{pod}",
+		HandlePortForward(
+			p.PortForward,
+			WithPortForwardStreamIdleTimeout(p.StreamCreationTimeout),
+			WithPortForwardCreationTimeout(p.StreamIdleTimeout),
+		),
+	).Methods("POST", "GET")
 
 	if p.GetStatsSummary != nil {
 		f := HandlePodStatsSummary(p.GetStatsSummary)
@@ -72,6 +92,11 @@ func PodHandler(p PodHandlerConfig, debug bool) http.Handler {
 		r.HandleFunc("/stats/summary/", f).Methods("GET")
 	}
 
+	if p.GetMetricsResource != nil {
+		f := HandlePodMetricsResource(p.GetMetricsResource)
+		r.HandleFunc(MetricsResourceRouteSuffix, f).Methods("GET")
+		r.HandleFunc(MetricsResourceRouteSuffix+"/", f).Methods("GET")
+	}
 	r.NotFoundHandler = http.HandlerFunc(NotFound)
 	return r
 }
@@ -97,6 +122,26 @@ func PodStatsSummaryHandler(f PodStatsSummaryHandlerFunc) http.Handler {
 	return r
 }
 
+// PodMetricsResourceHandler creates an http handler for serving pod metrics.
+//
+// If the passed in handler func is nil this will create handlers which only
+// serves http.StatusNotImplemented
+func PodMetricsResourceHandler(f PodMetricsResourceHandlerFunc) http.Handler {
+	if f == nil {
+		return http.HandlerFunc(NotImplemented)
+	}
+
+	r := mux.NewRouter()
+
+	h := HandlePodMetricsResource(f)
+
+	r.Handle(MetricsResourceRouteSuffix, ochttp.WithRouteTag(h, "PodMetricsResourceHandler")).Methods("GET")
+	r.Handle(MetricsResourceRouteSuffix+"/", ochttp.WithRouteTag(h, "PodMetricsResourceHandler")).Methods("GET")
+
+	r.NotFoundHandler = http.HandlerFunc(NotFound)
+	return r
+}
+
 // AttachPodRoutes adds the http routes for pod stuff to the passed in serve mux.
 //
 // Callers should take care to namespace the serve mux as they see fit, however
@@ -111,7 +156,8 @@ func AttachPodRoutes(p PodHandlerConfig, mux ServeMux, debug bool) {
 // The main reason for this struct is in case of expansion we do not need to break
 // the package level API.
 type PodMetricsConfig struct {
-	GetStatsSummary PodStatsSummaryHandlerFunc
+	GetStatsSummary    PodStatsSummaryHandlerFunc
+	GetMetricsResource PodMetricsResourceHandlerFunc
 }
 
 // AttachPodMetricsRoutes adds the http routes for pod/node metrics to the passed in serve mux.
@@ -120,6 +166,7 @@ type PodMetricsConfig struct {
 // these routes get called by the Kubernetes API server.
 func AttachPodMetricsRoutes(p PodMetricsConfig, mux ServeMux) {
 	mux.Handle("/", InstrumentHandler(HandlePodStatsSummary(p.GetStatsSummary)))
+	mux.Handle("/", InstrumentHandler(HandlePodMetricsResource(p.GetMetricsResource)))
 }
 
 func instrumentRequest(r *http.Request) *http.Request {

node/api/stats.go

@@ -20,7 +20,7 @@ import (
 	"net/http"
 
 	"github.com/pkg/errors"
-	"github.com/virtual-kubelet/virtual-kubelet/node/api/statsv1alpha1"
+	statsv1alpha1 "k8s.io/kubelet/pkg/apis/stats/v1alpha1"
 )
 
 // PodStatsSummaryHandlerFunc defines the handler for getting pod stats summaries
@@ -48,6 +48,7 @@ func HandlePodStatsSummary(h PodStatsSummaryHandlerFunc) http.HandlerFunc {
 		if _, err := w.Write(b); err != nil {
 			return errors.Wrap(err, "could not write to client")
 		}
+		_, _ = w.Write([]byte("\n"))
 		return nil
 	})
 }

node/api/statsv1alpha1/README.md

@@ -1,7 +0,0 @@
-These types are copied from the [k8s.io/kubelet](https://pkg.go.dev/k8s.io/kubelet@v0.21.0/pkg/apis/stats/v1alpha1) module.
-They are used from a type alias in the API package.
-
-It is being used this way because the module is only available from 1.20 and on, but currently we are pinned to v1.19 and plan to continue to support v1.19 for some time.
-Likewise we want to stop importing k8s.io/kubernetes (where the older type def is) since this transatively imports all of kubernetes.
-
-After the min version is v1.20 we can update the type alias to point the the module and remove these type definitions.

node/api/statsv1alpha1/types.go

@@ -1,345 +0,0 @@
-/*
-Copyright 2015 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package statsv1alpha1
-
-import (
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-)
-
-// Summary is a top-level container for holding NodeStats and PodStats.
-type Summary struct {
-	// Overall node stats.
-	Node NodeStats `json:"node"`
-	// Per-pod stats.
-	Pods []PodStats `json:"pods"`
-}
-
-// NodeStats holds node-level unprocessed sample stats.
-type NodeStats struct {
-	// Reference to the measured Node.
-	NodeName string `json:"nodeName"`
-	// Stats of system daemons tracked as raw containers.
-	// The system containers are named according to the SystemContainer* constants.
-	// +optional
-	// +patchMergeKey=name
-	// +patchStrategy=merge
-	SystemContainers []ContainerStats `json:"systemContainers,omitempty" patchStrategy:"merge" patchMergeKey:"name"`
-	// The time at which data collection for the node-scoped (i.e. aggregate) stats was (re)started.
-	StartTime metav1.Time `json:"startTime"`
-	// Stats pertaining to CPU resources.
-	// +optional
-	CPU *CPUStats `json:"cpu,omitempty"`
-	// Stats pertaining to memory (RAM) resources.
-	// +optional
-	Memory *MemoryStats `json:"memory,omitempty"`
-	// Stats pertaining to network resources.
-	// +optional
-	Network *NetworkStats `json:"network,omitempty"`
-	// Stats pertaining to total usage of filesystem resources on the rootfs used by node k8s components.
-	// NodeFs.Used is the total bytes used on the filesystem.
-	// +optional
-	Fs *FsStats `json:"fs,omitempty"`
-	// Stats about the underlying container runtime.
-	// +optional
-	Runtime *RuntimeStats `json:"runtime,omitempty"`
-	// Stats about the rlimit of system.
-	// +optional
-	Rlimit *RlimitStats `json:"rlimit,omitempty"`
-}
-
-// RlimitStats are stats rlimit of OS.
-type RlimitStats struct {
-	Time metav1.Time `json:"time"`
-
-	// The max PID of OS.
-	MaxPID *int64 `json:"maxpid,omitempty"`
-	// The number of running process in the OS.
-	NumOfRunningProcesses *int64 `json:"curproc,omitempty"`
-}
-
-// RuntimeStats are stats pertaining to the underlying container runtime.
-type RuntimeStats struct {
-	// Stats about the underlying filesystem where container images are stored.
-	// This filesystem could be the same as the primary (root) filesystem.
-	// Usage here refers to the total number of bytes occupied by images on the filesystem.
-	// +optional
-	ImageFs *FsStats `json:"imageFs,omitempty"`
-}
-
-const (
-	// SystemContainerKubelet is the container name for the system container tracking Kubelet usage.
-	SystemContainerKubelet = "kubelet"
-	// SystemContainerRuntime is the container name for the system container tracking the runtime (e.g. docker) usage.
-	SystemContainerRuntime = "runtime"
-	// SystemContainerMisc is the container name for the system container tracking non-kubernetes processes.
-	SystemContainerMisc = "misc"
-	// SystemContainerPods is the container name for the system container tracking user pods.
-	SystemContainerPods = "pods"
-)
-
-// ProcessStats are stats pertaining to processes.
-type ProcessStats struct {
-	// Number of processes
-	// +optional
-	ProcessCount *uint64 `json:"process_count,omitempty"`
-}
-
-// PodStats holds pod-level unprocessed sample stats.
-type PodStats struct {
-	// Reference to the measured Pod.
-	PodRef PodReference `json:"podRef"`
-	// The time at which data collection for the pod-scoped (e.g. network) stats was (re)started.
-	StartTime metav1.Time `json:"startTime"`
-	// Stats of containers in the measured pod.
-	// +patchMergeKey=name
-	// +patchStrategy=merge
-	Containers []ContainerStats `json:"containers" patchStrategy:"merge" patchMergeKey:"name"`
-	// Stats pertaining to CPU resources consumed by pod cgroup (which includes all containers' resource usage and pod overhead).
-	// +optional
-	CPU *CPUStats `json:"cpu,omitempty"`
-	// Stats pertaining to memory (RAM) resources consumed by pod cgroup (which includes all containers' resource usage and pod overhead).
-	// +optional
-	Memory *MemoryStats `json:"memory,omitempty"`
-	// Stats pertaining to network resources.
-	// +optional
-	Network *NetworkStats `json:"network,omitempty"`
-	// Stats pertaining to volume usage of filesystem resources.
-	// VolumeStats.UsedBytes is the number of bytes used by the Volume
-	// +optional
-	// +patchMergeKey=name
-	// +patchStrategy=merge
-	VolumeStats []VolumeStats `json:"volume,omitempty" patchStrategy:"merge" patchMergeKey:"name"`
-	// EphemeralStorage reports the total filesystem usage for the containers and emptyDir-backed volumes in the measured Pod.
-	// +optional
-	EphemeralStorage *FsStats `json:"ephemeral-storage,omitempty"`
-	// ProcessStats pertaining to processes.
-	// +optional
-	ProcessStats *ProcessStats `json:"process_stats,omitempty"`
-}
-
-// ContainerStats holds container-level unprocessed sample stats.
-type ContainerStats struct {
-	// Reference to the measured container.
-	Name string `json:"name"`
-	// The time at which data collection for this container was (re)started.
-	StartTime metav1.Time `json:"startTime"`
-	// Stats pertaining to CPU resources.
-	// +optional
-	CPU *CPUStats `json:"cpu,omitempty"`
-	// Stats pertaining to memory (RAM) resources.
-	// +optional
-	Memory *MemoryStats `json:"memory,omitempty"`
-	// Metrics for Accelerators. Each Accelerator corresponds to one element in the array.
-	Accelerators []AcceleratorStats `json:"accelerators,omitempty"`
-	// Stats pertaining to container rootfs usage of filesystem resources.
-	// Rootfs.UsedBytes is the number of bytes used for the container write layer.
-	// +optional
-	Rootfs *FsStats `json:"rootfs,omitempty"`
-	// Stats pertaining to container logs usage of filesystem resources.
-	// Logs.UsedBytes is the number of bytes used for the container logs.
-	// +optional
-	Logs *FsStats `json:"logs,omitempty"`
-	// User defined metrics that are exposed by containers in the pod. Typically, we expect only one container in the pod to be exposing user defined metrics. In the event of multiple containers exposing metrics, they will be combined here.
-	// +patchMergeKey=name
-	// +patchStrategy=merge
-	UserDefinedMetrics []UserDefinedMetric `json:"userDefinedMetrics,omitempty" patchStrategy:"merge" patchMergeKey:"name"`
-}
-
-// PodReference contains enough information to locate the referenced pod.
-type PodReference struct {
-	Name      string `json:"name"`
-	Namespace string `json:"namespace"`
-	UID       string `json:"uid"`
-}
-
-// InterfaceStats contains resource value data about interface.
-type InterfaceStats struct {
-	// The name of the interface
-	Name string `json:"name"`
-	// Cumulative count of bytes received.
-	// +optional
-	RxBytes *uint64 `json:"rxBytes,omitempty"`
-	// Cumulative count of receive errors encountered.
-	// +optional
-	RxErrors *uint64 `json:"rxErrors,omitempty"`
-	// Cumulative count of bytes transmitted.
-	// +optional
-	TxBytes *uint64 `json:"txBytes,omitempty"`
-	// Cumulative count of transmit errors encountered.
-	// +optional
-	TxErrors *uint64 `json:"txErrors,omitempty"`
-}
-
-// NetworkStats contains data about network resources.
-type NetworkStats struct {
-	// The time at which these stats were updated.
-	Time metav1.Time `json:"time"`
-
-	// Stats for the default interface, if found
-	InterfaceStats `json:",inline"`
-
-	Interfaces []InterfaceStats `json:"interfaces,omitempty"`
-}
-
-// CPUStats contains data about CPU usage.
-type CPUStats struct {
-	// The time at which these stats were updated.
-	Time metav1.Time `json:"time"`
-	// Total CPU usage (sum of all cores) averaged over the sample window.
-	// The "core" unit can be interpreted as CPU core-nanoseconds per second.
-	// +optional
-	UsageNanoCores *uint64 `json:"usageNanoCores,omitempty"`
-	// Cumulative CPU usage (sum of all cores) since object creation.
-	// +optional
-	UsageCoreNanoSeconds *uint64 `json:"usageCoreNanoSeconds,omitempty"`
-}
-
-// MemoryStats contains data about memory usage.
-type MemoryStats struct {
-	// The time at which these stats were updated.
-	Time metav1.Time `json:"time"`
-	// Available memory for use.  This is defined as the memory limit - workingSetBytes.
-	// If memory limit is undefined, the available bytes is omitted.
-	// +optional
-	AvailableBytes *uint64 `json:"availableBytes,omitempty"`
-	// Total memory in use. This includes all memory regardless of when it was accessed.
-	// +optional
-	UsageBytes *uint64 `json:"usageBytes,omitempty"`
-	// The amount of working set memory. This includes recently accessed memory,
-	// dirty memory, and kernel memory. WorkingSetBytes is <= UsageBytes
-	// +optional
-	WorkingSetBytes *uint64 `json:"workingSetBytes,omitempty"`
-	// The amount of anonymous and swap cache memory (includes transparent
-	// hugepages).
-	// +optional
-	RSSBytes *uint64 `json:"rssBytes,omitempty"`
-	// Cumulative number of minor page faults.
-	// +optional
-	PageFaults *uint64 `json:"pageFaults,omitempty"`
-	// Cumulative number of major page faults.
-	// +optional
-	MajorPageFaults *uint64 `json:"majorPageFaults,omitempty"`
-}
-
-// AcceleratorStats contains stats for accelerators attached to the container.
-type AcceleratorStats struct {
-	// Make of the accelerator (nvidia, amd, google etc.)
-	Make string `json:"make"`
-
-	// Model of the accelerator (tesla-p100, tesla-k80 etc.)
-	Model string `json:"model"`
-
-	// ID of the accelerator.
-	ID string `json:"id"`
-
-	// Total accelerator memory.
-	// unit: bytes
-	MemoryTotal uint64 `json:"memoryTotal"`
-
-	// Total accelerator memory allocated.
-	// unit: bytes
-	MemoryUsed uint64 `json:"memoryUsed"`
-
-	// Percent of time over the past sample period (10s) during which
-	// the accelerator was actively processing.
-	DutyCycle uint64 `json:"dutyCycle"`
-}
-
-// VolumeStats contains data about Volume filesystem usage.
-type VolumeStats struct {
-	// Embedded FsStats
-	FsStats `json:",inline"`
-	// Name is the name given to the Volume
-	// +optional
-	Name string `json:"name,omitempty"`
-	// Reference to the PVC, if one exists
-	// +optional
-	PVCRef *PVCReference `json:"pvcRef,omitempty"`
-}
-
-// PVCReference contains enough information to describe the referenced PVC.
-type PVCReference struct {
-	Name      string `json:"name"`
-	Namespace string `json:"namespace"`
-}
-
-// FsStats contains data about filesystem usage.
-type FsStats struct {
-	// The time at which these stats were updated.
-	Time metav1.Time `json:"time"`
-	// AvailableBytes represents the storage space available (bytes) for the filesystem.
-	// +optional
-	AvailableBytes *uint64 `json:"availableBytes,omitempty"`
-	// CapacityBytes represents the total capacity (bytes) of the filesystems underlying storage.
-	// +optional
-	CapacityBytes *uint64 `json:"capacityBytes,omitempty"`
-	// UsedBytes represents the bytes used for a specific task on the filesystem.
-	// This may differ from the total bytes used on the filesystem and may not equal CapacityBytes - AvailableBytes.
-	// e.g. For ContainerStats.Rootfs this is the bytes used by the container rootfs on the filesystem.
-	// +optional
-	UsedBytes *uint64 `json:"usedBytes,omitempty"`
-	// InodesFree represents the free inodes in the filesystem.
-	// +optional
-	InodesFree *uint64 `json:"inodesFree,omitempty"`
-	// Inodes represents the total inodes in the filesystem.
-	// +optional
-	Inodes *uint64 `json:"inodes,omitempty"`
-	// InodesUsed represents the inodes used by the filesystem
-	// This may not equal Inodes - InodesFree because this filesystem may share inodes with other "filesystems"
-	// e.g. For ContainerStats.Rootfs, this is the inodes used only by that container, and does not count inodes used by other containers.
-	InodesUsed *uint64 `json:"inodesUsed,omitempty"`
-}
-
-// UserDefinedMetricType defines how the metric should be interpreted by the user.
-type UserDefinedMetricType string
-
-const (
-	// MetricGauge is an instantaneous value. May increase or decrease.
-	MetricGauge UserDefinedMetricType = "gauge"
-
-	// MetricCumulative is a counter-like value that is only expected to increase.
-	MetricCumulative UserDefinedMetricType = "cumulative"
-
-	// MetricDelta is a rate over a time period.
-	MetricDelta UserDefinedMetricType = "delta"
-)
-
-// UserDefinedMetricDescriptor contains metadata that describes a user defined metric.
-type UserDefinedMetricDescriptor struct {
-	// The name of the metric.
-	Name string `json:"name"`
-
-	// Type of the metric.
-	Type UserDefinedMetricType `json:"type"`
-
-	// Display Units for the stats.
-	Units string `json:"units"`
-
-	// Metadata labels associated with this metric.
-	// +optional
-	Labels map[string]string `json:"labels,omitempty"`
-}
-
-// UserDefinedMetric represents a metric defined and generated by users.
-type UserDefinedMetric struct {
-	UserDefinedMetricDescriptor `json:",inline"`
-	// The time at which these stats were updated.
-	Time metav1.Time `json:"time"`
-	// Value of the metric. Float64s have 53 bit precision.
-	// We do not foresee any metrics exceeding that value.
-	Value float64 `json:"value"`
-}

node/lease_controller_v1.go

@@ -30,7 +30,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/wait"
 	coordclientset "k8s.io/client-go/kubernetes/typed/coordination/v1"
 	"k8s.io/utils/clock"
-	"k8s.io/utils/pointer"
+	"k8s.io/utils/ptr"
 )
 
 // Code heavily borrowed from: https://github.com/kubernetes/kubernetes/blob/v1.18.13/pkg/kubelet/nodelease/controller.go
@@ -78,15 +78,15 @@ func newLeaseControllerWithRenewInterval(
 	nodeController *NodeController) (*leaseController, error) {
 
 	if leaseDurationSeconds <= 0 {
-		return nil, fmt.Errorf("Lease duration seconds %d is invalid, it must be > 0", leaseDurationSeconds)
+		return nil, fmt.Errorf("lease duration seconds %d is invalid, it must be > 0", leaseDurationSeconds)
 	}
 
 	if renewInterval == 0 {
-		return nil, fmt.Errorf("Lease renew interval %s is invalid, it must be > 0", renewInterval.String())
+		return nil, fmt.Errorf("lease renew interval %s is invalid, it must be > 0", renewInterval.String())
 	}
 
 	if float64(leaseDurationSeconds) <= renewInterval.Seconds() {
-		return nil, fmt.Errorf("Lease renew interval %s is invalid, it must be less than lease duration seconds %d", renewInterval.String(), leaseDurationSeconds)
+		return nil, fmt.Errorf("lease renew interval %s is invalid, it must be less than lease duration seconds %d", renewInterval.String(), leaseDurationSeconds)
 	}
 
 	return &leaseController{
@@ -128,7 +128,7 @@ func (c *leaseController) sync(ctx context.Context) {
 		return
 	}
 	if node == nil {
-		err = errors.New("Servernode is null")
+		err = errors.New("server node is null")
 		log.G(ctx).WithError(err).Error("servernode is null")
 		span.SetStatus(err)
 		return
@@ -275,8 +275,8 @@ func (c *leaseController) newLease(ctx context.Context, node *corev1.Node, base
 				Namespace: corev1.NamespaceNodeLease,
 			},
 			Spec: coordinationv1.LeaseSpec{
-				HolderIdentity:       pointer.StringPtr(node.Name),
-				LeaseDurationSeconds: pointer.Int32Ptr(c.leaseDurationSeconds),
+				HolderIdentity:       ptr.To(node.Name),
+				LeaseDurationSeconds: ptr.To(c.leaseDurationSeconds),
 			},
 		}
 	} else {

node/lifecycle_test.go

@@ -58,7 +58,7 @@ type fakeDiscardingRecorder struct {
 }
 
 func (r *fakeDiscardingRecorder) Event(object runtime.Object, eventType, reason, message string) {
-	r.Eventf(object, eventType, reason, message)
+	r.Eventf(object, eventType, reason, "%s", message)
 }
 
 func (r *fakeDiscardingRecorder) Eventf(object runtime.Object, eventType, reason, messageFmt string, args ...interface{}) {
@@ -441,7 +441,7 @@ func testCreateStartDeleteScenario(ctx context.Context, t *testing.T, s *system,
 			// TODO(Sargun): Make this "smarter" about the status the pod is in.
 			func(ev watch.Event) (bool, error) {
 				pod := ev.Object.(*corev1.Pod)
-				return pod.Name == p.ObjectMeta.Name, nil
+				return pod.Name == p.Name, nil
 			})
 
 		sendErr(ctx, watchErrCh, watchErr)
@@ -628,7 +628,7 @@ func benchmarkCreatePods(ctx context.Context, b *testing.B, s *system) {
 type podModifier func(*corev1.Pod)
 
 func randomizeUID(pod *corev1.Pod) {
-	pod.ObjectMeta.UID = uuid.NewUUID()
+	pod.UID = uuid.NewUUID()
 }
 
 func randomizeName(pod *corev1.Pod) {
@@ -638,7 +638,7 @@ func randomizeName(pod *corev1.Pod) {
 
 func forRealAPIServer(pod *corev1.Pod) {
 	pod.ResourceVersion = ""
-	pod.ObjectMeta.UID = ""
+	pod.UID = ""
 }
 
 func nameBasedOnTest(t *testing.T) podModifier {

node/mock_test.go

@@ -243,15 +243,15 @@ func buildKeyFromNames(namespace string, name string) (string, error) {
 
 // buildKey is a helper for building the "key" for the providers pod store.
 func buildKey(pod *v1.Pod) (string, error) {
-	if pod.ObjectMeta.Namespace == "" {
+	if pod.Namespace == "" {
 		return "", fmt.Errorf("pod namespace not found")
 	}
 
-	if pod.ObjectMeta.Name == "" {
+	if pod.Name == "" {
 		return "", fmt.Errorf("pod name not found")
 	}
 
-	return buildKeyFromNames(pod.ObjectMeta.Namespace, pod.ObjectMeta.Name)
+	return buildKeyFromNames(pod.Namespace, pod.Name)
 }
 
 type mockProviderAsync struct {

node/node.go

@@ -146,7 +146,7 @@ func WithNodeEnableLeaseV1WithRenewInterval(client coordclientset.LeaseInterface
 			n,
 		)
 		if err != nil {
-			return fmt.Errorf("Unable to configure lease controller: %w", err)
+			return fmt.Errorf("unable to configure lease controller: %w", err)
 		}
 
 		n.leaseController = leaseController
@@ -327,9 +327,9 @@ func (n *NodeController) ensureNode(ctx context.Context, providerNode *corev1.No
 	n.serverNodeLock.Unlock()
 	// Bad things will happen if the node is deleted in k8s and recreated by someone else
 	// we rely on this persisting
-	providerNode.ObjectMeta.Name = node.Name
-	providerNode.ObjectMeta.Namespace = node.Namespace
-	providerNode.ObjectMeta.UID = node.UID
+	providerNode.Name = node.Name
+	providerNode.Namespace = node.Namespace
+	providerNode.UID = node.UID
 
 	return nil
 }
@@ -346,7 +346,9 @@ func (n *NodeController) controlLoop(ctx context.Context, providerNode *corev1.N
 
 	var sleepInterval time.Duration
 	if n.leaseController == nil {
-		log.G(ctx).WithField("pingInterval", n.pingInterval).Debug("lease controller is not enabled, updating node status in Kube API server at Ping Time Interval")
+		log.G(ctx).
+			WithField("pingInterval", n.pingInterval).
+			Debug("lease controller is not enabled, updating node status in Kube API server at Ping Time Interval")
 		sleepInterval = n.pingInterval
 	} else {
 		log.G(ctx).WithField("statusInterval", n.statusInterval).Debug("lease controller in use, updating at statusInterval")
@@ -369,8 +371,8 @@ func (n *NodeController) controlLoop(ctx context.Context, providerNode *corev1.N
 			log.G(ctx).Debug("Received node status update")
 
 			providerNode.Status = updated.Status
-			providerNode.ObjectMeta.Annotations = updated.Annotations
-			providerNode.ObjectMeta.Labels = updated.Labels
+			providerNode.Annotations = updated.Annotations
+			providerNode.Labels = updated.Labels
 			if err := n.updateStatus(ctx, providerNode, false); err != nil {
 				log.G(ctx).WithError(err).Error("Error handling node status update")
 			}
@@ -401,7 +403,7 @@ func (n *NodeController) updateStatus(ctx context.Context, providerNode *corev1.
 	if result, err := n.nodePingController.getResult(ctx); err != nil {
 		return err
 	} else if result.error != nil {
-		return fmt.Errorf("Not updating node status because node ping failed: %w", result.error)
+		return fmt.Errorf("not updating node status because node ping failed: %w", result.error)
 	}
 
 	updateNodeStatusHeartbeat(providerNode)
@@ -429,7 +431,7 @@ func (n *NodeController) updateStatus(ctx context.Context, providerNode *corev1.
 }
 
 // Returns a copy of the server node object
-func (n *NodeController) getServerNode(ctx context.Context) (*corev1.Node, error) {
+func (n *NodeController) getServerNode(_ context.Context) (*corev1.Node, error) {
 	n.serverNodeLock.Lock()
 	defer n.serverNodeLock.Unlock()
 	if n.serverNode == nil {

node/node_test.go

@@ -25,7 +25,6 @@ import (
 
 	"gotest.tools/assert"
 	"gotest.tools/assert/cmp"
-	is "gotest.tools/assert/cmp"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -394,8 +393,8 @@ func TestBeforeAnnotationsPreserved(t *testing.T) {
 	newNode, err := nodes.Get(ctx, testNodeCopy.Name, emptyGetOptions)
 	assert.NilError(t, err)
 
-	assert.Assert(t, is.Contains(newNode.Annotations, "testAnnotation"))
-	assert.Assert(t, is.Contains(newNode.Annotations, "beforeAnnotation"))
+	assert.Assert(t, cmp.Contains(newNode.Annotations, "testAnnotation"))
+	assert.Assert(t, cmp.Contains(newNode.Annotations, "beforeAnnotation"))
 }
 
 // Are conditions set by systems outside of VK preserved?
@@ -444,7 +443,7 @@ func TestManualConditionsPreserved(t *testing.T) {
 
 	newNode, err := nodes.Get(ctx, testNodeCopy.Name, emptyGetOptions)
 	assert.NilError(t, err)
-	assert.Assert(t, is.Len(newNode.Status.Conditions, 0))
+	assert.Assert(t, cmp.Len(newNode.Status.Conditions, 0))
 
 	baseCondition := corev1.NodeCondition{
 		Type:    "BaseCondition",
@@ -479,8 +478,8 @@ func TestManualConditionsPreserved(t *testing.T) {
 
 	newNode, err = nodes.Get(ctx, testNodeCopy.Name, emptyGetOptions)
 	assert.NilError(t, err)
-	assert.Assert(t, is.Len(newNode.Status.Conditions, 1))
-	assert.Assert(t, is.Contains(newNode.Annotations, "testAnnotation"))
+	assert.Assert(t, cmp.Len(newNode.Status.Conditions, 1))
+	assert.Assert(t, cmp.Contains(newNode.Annotations, "testAnnotation"))
 
 	// Add a new event manually
 	manuallyAddedCondition := corev1.NodeCondition{
@@ -507,8 +506,8 @@ func TestManualConditionsPreserved(t *testing.T) {
 				return true
 			}
 		}
-		assert.Assert(t, is.Contains(receivedNode.Annotations, "testAnnotation"))
-		assert.Assert(t, is.Contains(newNode.Annotations, "manuallyAddedAnnotation"))
+		assert.Assert(t, cmp.Contains(receivedNode.Annotations, "testAnnotation"))
+		assert.Assert(t, cmp.Contains(newNode.Annotations, "manuallyAddedAnnotation"))
 
 		return false
 	}))
@@ -553,11 +552,11 @@ func TestManualConditionsPreserved(t *testing.T) {
 	for idx := range newNode.Status.Conditions {
 		seenConditionTypes[idx] = newNode.Status.Conditions[idx].Type
 	}
-	assert.Assert(t, is.Contains(seenConditionTypes, baseCondition.Type))
-	assert.Assert(t, is.Contains(seenConditionTypes, newCondition.Type))
-	assert.Assert(t, is.Contains(seenConditionTypes, manuallyAddedCondition.Type))
-	assert.Assert(t, is.Equal(newNode.Annotations["testAnnotation"], ""))
-	assert.Assert(t, is.Contains(newNode.Annotations, "manuallyAddedAnnotation"))
+	assert.Assert(t, cmp.Contains(seenConditionTypes, baseCondition.Type))
+	assert.Assert(t, cmp.Contains(seenConditionTypes, newCondition.Type))
+	assert.Assert(t, cmp.Contains(seenConditionTypes, manuallyAddedCondition.Type))
+	assert.Assert(t, cmp.Equal(newNode.Annotations["testAnnotation"], ""))
+	assert.Assert(t, cmp.Contains(newNode.Annotations, "manuallyAddedAnnotation"))
 
 	t.Log(newNode.Status.Conditions)
 }
@@ -607,15 +606,15 @@ func TestNodePingSingleInflight(t *testing.T) {
 	assert.Assert(t, timeTakenToCompleteFirstPing < pingTimeout*5, "Time taken to complete first ping: %v", timeTakenToCompleteFirstPing)
 
 	assert.Assert(t, cmp.Error(firstPing.error, context.DeadlineExceeded.Error()))
-	assert.Assert(t, is.Equal(1, calls.read()))
-	assert.Assert(t, is.Equal(0, finished.read()))
+	assert.Assert(t, cmp.Equal(1, calls.read()))
+	assert.Assert(t, cmp.Equal(0, finished.read()))
 
 	// Wait until the first sleep finishes (the test context is done)
 	assert.NilError(t, finished.until(testCtx, func(i int) bool { return i > 0 }))
 
 	// Assert we didn't stack up goroutines, and that the one goroutine in flight finishd
-	assert.Assert(t, is.Equal(1, calls.read()))
-	assert.Assert(t, is.Equal(1, finished.read()))
+	assert.Assert(t, cmp.Equal(1, calls.read()))
+	assert.Assert(t, cmp.Equal(1, finished.read()))
 
 }
 

node/nodeutil/auth.go

@@ -6,6 +6,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/virtual-kubelet/virtual-kubelet/errdefs"
 	"github.com/virtual-kubelet/virtual-kubelet/log"
 	"github.com/virtual-kubelet/virtual-kubelet/trace"
 	"k8s.io/apiserver/pkg/authentication/authenticator"
@@ -14,6 +15,7 @@ import (
 	"k8s.io/apiserver/pkg/authentication/user"
 	"k8s.io/apiserver/pkg/authorization/authorizer"
 	"k8s.io/apiserver/pkg/authorization/authorizerfactory"
+	"k8s.io/apiserver/pkg/server/options"
 	"k8s.io/client-go/kubernetes"
 )
 
@@ -49,7 +51,7 @@ func InstrumentAuth(auth Auth) Auth {
 // NoAuth creates an Auth which allows anonymous access to all resouorces
 func NoAuth() Auth {
 	return &authWrapper{
-		Request:                 anonymous.NewAuthenticator(),
+		Request:                 anonymous.NewAuthenticator(nil),
 		RequestAttributesGetter: &NodeRequestAttr{},
 		Authorizer:              authorizerfactory.NewAlwaysAllowAuthorizer(),
 	}
@@ -68,14 +70,24 @@ func handleAuth(auth Auth, w http.ResponseWriter, r *http.Request, next http.Han
 	defer span.End()
 	r = r.WithContext(ctx)
 
+	logger := log.G(r.Context())
 	info, ok, err := auth.AuthenticateRequest(r)
-	if err != nil || !ok {
-		log.G(r.Context()).WithError(err).Error("Authorization error")
+	if err != nil {
+		logger.WithError(err).Error("Error authenticating request")
 		http.Error(w, "Unauthorized", http.StatusUnauthorized)
+		span.SetStatus(err)
 		return
 	}
 
-	logger := log.G(ctx).WithFields(log.Fields{
+	if !ok {
+		logger.Error("Request not authenticated")
+		log.G(r.Context()).Infof("Unauthorized: RequestURI: %s", r.RequestURI)
+		http.Error(w, "Unauthorized", http.StatusUnauthorized)
+		span.SetStatus(errdefs.ErrUnauthorized)
+		return
+	}
+
+	logger = logger.WithFields(log.Fields{
 		"user-name": info.User.GetName(),
 		"user-id":   info.User.GetUID(),
 	})
@@ -89,11 +101,13 @@ func handleAuth(auth Auth, w http.ResponseWriter, r *http.Request, next http.Han
 	if err != nil {
 		log.G(r.Context()).WithError(err).Error("Authorization error")
 		http.Error(w, err.Error(), http.StatusInternalServerError)
+		span.SetStatus(err)
 		return
 	}
 
 	if decision != authorizer.DecisionAllow {
 		http.Error(w, "Forbidden", http.StatusForbidden)
+		span.SetStatus(errdefs.ErrForbidden)
 		return
 	}
 
@@ -114,13 +128,13 @@ type WebhookAuthConfig struct {
 func WebhookAuth(client kubernetes.Interface, nodeName string, opts ...WebhookAuthOption) (Auth, error) {
 	cfg := WebhookAuthConfig{
 		AuthnConfig: authenticatorfactory.DelegatingAuthenticatorConfig{
-			CacheTTL: 2 * time.Minute, // default taken from k8s.io/kubernetes/pkg/kubelet/apis/config/v1beta1
-			// TODO: After upgrading k8s libs, we need to add the retry backoff option
+			CacheTTL:            2 * time.Minute, // default taken from k8s.io/kubernetes/pkg/kubelet/apis/config/v1beta1
+			WebhookRetryBackoff: options.DefaultAuthWebhookRetryBackoff(),
 		},
 		AuthzConfig: authorizerfactory.DelegatingAuthorizerConfig{
-			AllowCacheTTL: 5 * time.Minute,  // default taken from k8s.io/kubernetes/pkg/kubelet/apis/config/v1beta1
-			DenyCacheTTL:  30 * time.Second, // default taken from k8s.io/kubernetes/pkg/kubelet/apis/config/v1beta1
-			// TODO: After upgrading k8s libs, we need to add the retry backoff option
+			AllowCacheTTL:       5 * time.Minute,  // default taken from k8s.io/kubernetes/pkg/kubelet/apis/config/v1beta1
+			DenyCacheTTL:        30 * time.Second, // default taken from k8s.io/kubernetes/pkg/kubelet/apis/config/v1beta1
+			WebhookRetryBackoff: options.DefaultAuthWebhookRetryBackoff(),
 		},
 	}
 

node/nodeutil/client.go

@@ -24,7 +24,7 @@ func ClientsetFromEnv(kubeConfigPath string) (*kubernetes.Clientset, error) {
 	)
 
 	if kubeConfigPath != "" {
-		config, err = clientsetFromEnvKubeConfigPath(kubeConfigPath)
+		config, err = RestConfigFromEnv(kubeConfigPath)
 	} else {
 		config, err = rest.InClusterConfig()
 	}
@@ -36,7 +36,8 @@ func ClientsetFromEnv(kubeConfigPath string) (*kubernetes.Clientset, error) {
 	return kubernetes.NewForConfig(config)
 }
 
-func clientsetFromEnvKubeConfigPath(kubeConfigPath string) (*rest.Config, error) {
+// RestConfigFromEnv is like ClientsetFromEnv except it returns a rest config instead of a full client.
+func RestConfigFromEnv(kubeConfigPath string) (*rest.Config, error) {
 	_, err := os.Stat(kubeConfigPath)
 	if os.IsNotExist(err) {
 		return rest.InClusterConfig()

node/nodeutil/controller.go

@@ -48,7 +48,8 @@ type Node struct {
 
 	workers int
 
-	eb record.EventBroadcaster
+	eb           record.EventBroadcaster
+	caController caController
 }
 
 // NodeController returns the configured node controller.
@@ -107,6 +108,10 @@ func (n *Node) Run(ctx context.Context) (retErr error) {
 		log.G(ctx).Debug("Started event broadcaster")
 	}
 
+	if n.caController != nil {
+		go n.caController.Run(ctx, 1)
+	}
+
 	cancelHTTP, err := n.runHTTP(ctx)
 	if err != nil {
 		return err
@@ -212,6 +217,8 @@ func (n *Node) Err() error {
 // NodeOpt is used as functional options when configuring a new node in NewNodeFromClient
 type NodeOpt func(c *NodeConfig) error
 
+type caController interface{ Run(context.Context, int) }
+
 // NodeConfig is used to hold configuration items for a Node.
 // It gets used in conjection with NodeOpt in NewNodeFromClient
 type NodeConfig struct {
@@ -251,23 +258,17 @@ type NodeConfig struct {
 	// The default value is derived from the number of cores available.
 	NumWorkers int
 
+	// Set the error handler for node status update failures
+	NodeStatusUpdateErrorHandler node.ErrorHandler
+
+	// SkipDownwardAPIResolution can be used to skip any attempts at resolving downward API references
+	// in pods before calling CreatePod on the provider.
+	// Providers need this if they need to do their own custom resolving
+	SkipDownwardAPIResolution bool
+
 	routeAttacher func(Provider, NodeConfig, corev1listers.PodLister)
-}
 
-// WithNodeConfig returns a NodeOpt which replaces the NodeConfig with the passed in value.
-func WithNodeConfig(c NodeConfig) NodeOpt {
-	return func(orig *NodeConfig) error {
-		*orig = c
-		return nil
-	}
-}
-
-// WithClient return a NodeOpt that sets the client that will be used to create/manage the node.
-func WithClient(c kubernetes.Interface) NodeOpt {
-	return func(cfg *NodeConfig) error {
-		cfg.Client = c
-		return nil
-	}
+	caController caController
 }
 
 // NewNode creates a new node using the provided client and name.
@@ -318,10 +319,6 @@ func NewNode(name string, newProvider NewProviderFunc, opts ...NodeOpt) (*Node,
 		return nil, errors.Wrap(err, "error parsing http listen address")
 	}
 
-	if cfg.Client == nil {
-		return nil, errors.New("no client provided")
-	}
-
 	podInformerFactory := informers.NewSharedInformerFactoryWithOptions(
 		cfg.Client,
 		cfg.InformerResyncPeriod,
@@ -365,11 +362,19 @@ func NewNode(name string, newProvider NewProviderFunc, opts ...NodeOpt) (*Node,
 		}
 	}
 
+	nodeControllerOpts := []node.NodeControllerOpt{
+		node.WithNodeEnableLeaseV1(NodeLeaseV1Client(cfg.Client), node.DefaultLeaseDuration),
+	}
+
+	if cfg.NodeStatusUpdateErrorHandler != nil {
+		nodeControllerOpts = append(nodeControllerOpts, node.WithNodeStatusUpdateErrorHandler(cfg.NodeStatusUpdateErrorHandler))
+	}
+
 	nc, err := node.NewNodeController(
 		np,
 		&cfg.NodeSpec,
 		cfg.Client.CoreV1().Nodes(),
-		node.WithNodeEnableLeaseV1(NodeLeaseV1Client(cfg.Client), node.DefaultLeaseDuration),
+		nodeControllerOpts...,
 	)
 	if err != nil {
 		return nil, errors.Wrap(err, "error creating node controller")
@@ -382,13 +387,14 @@ func NewNode(name string, newProvider NewProviderFunc, opts ...NodeOpt) (*Node,
 	}
 
 	pc, err := node.NewPodController(node.PodControllerConfig{
-		PodClient:         cfg.Client.CoreV1(),
-		EventRecorder:     cfg.EventRecorder,
-		Provider:          p,
-		PodInformer:       podInformer,
-		SecretInformer:    secretInformer,
-		ConfigMapInformer: configMapInformer,
-		ServiceInformer:   serviceInformer,
+		PodClient:                 cfg.Client.CoreV1(),
+		EventRecorder:             cfg.EventRecorder,
+		Provider:                  p,
+		PodInformer:               podInformer,
+		SecretInformer:            secretInformer,
+		ConfigMapInformer:         configMapInformer,
+		ServiceInformer:           serviceInformer,
+		SkipDownwardAPIResolution: cfg.SkipDownwardAPIResolution,
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "error creating pod controller")
@@ -408,6 +414,7 @@ func NewNode(name string, newProvider NewProviderFunc, opts ...NodeOpt) (*Node,
 		h:                  cfg.Handler,
 		listenAddr:         cfg.HTTPListenAddr,
 		workers:            cfg.NumWorkers,
+		caController:       cfg.caController,
 	}, nil
 }
 

node/nodeutil/opts.go

@@ -0,0 +1,150 @@
+package nodeutil
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/virtual-kubelet/virtual-kubelet/errdefs"
+	"github.com/virtual-kubelet/virtual-kubelet/node/api"
+	"k8s.io/apiserver/pkg/server/dynamiccertificates"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+)
+
+// WithNodeConfig returns a NodeOpt which replaces the NodeConfig with the passed in value.
+func WithNodeConfig(c NodeConfig) NodeOpt {
+	return func(orig *NodeConfig) error {
+		*orig = c
+		return nil
+	}
+}
+
+// WithClient return a NodeOpt that sets the client that will be used to create/manage the node.
+func WithClient(c kubernetes.Interface) NodeOpt {
+	return func(cfg *NodeConfig) error {
+		cfg.Client = c
+		return nil
+	}
+}
+
+// BootstrapConfig is the configuration for bootstrapping a node.
+type BootstrapConfig struct {
+	// ClientCAConfigMapSpec is the config map information for getting the CA for authenticating clients
+	// If not set, the CA in the rest config will be used.
+	ClientCAConfigMapSpec *ConfigMapCASpec
+	// A set of options to pass when creating the webhook auth.
+	WebhookAuthOpts []WebhookAuthOption
+	// RestConfig is the rest config to use for the client
+	// If it is not provided a default one from the environment will be created.
+	RestConfig *rest.Config
+}
+
+// ConfigMapCASpec is the spec for a config map that contains a CA.
+type ConfigMapCASpec struct {
+	Namespace string
+	Name      string
+	Key       string
+}
+
+// BootstrapOpt is a functional option used to configure node bootstrapping
+type BootstrapOpt func(*BootstrapConfig) error
+
+// WithBootstrapFromRestConfig takes a reset config (or a default one from the environment) and returns a NodeOpt that will:
+// 1. Create a client from the rest config (if no client is set)
+// 2. Create webhook authn/authz from the rest config
+// 3. Configure the TLS config for the HTTP server from the certs in the rest config.
+func WithBootstrapFromRestConfig(opts ...BootstrapOpt) NodeOpt {
+	return func(cfg *NodeConfig) error {
+		var bOpts BootstrapConfig
+		if rCfg, err := RestConfigFromEnv(cfg.KubeconfigPath); err == nil {
+			bOpts.RestConfig = rCfg
+		}
+
+		for _, o := range opts {
+			if err := o(&bOpts); err != nil {
+				return err
+			}
+		}
+
+		if bOpts.RestConfig == nil {
+			return fmt.Errorf("no rest config provided")
+		}
+
+		if cfg.Client == nil {
+			client, err := kubernetes.NewForConfig(bOpts.RestConfig)
+			if err != nil {
+				return err
+			}
+			cfg.Client = client
+		}
+
+		if err := WithTLSConfig(tlsFromRestConfig(bOpts.RestConfig))(cfg); err != nil {
+			return err
+		}
+
+		if err := configureWebhookCA(cfg, &bOpts); err != nil {
+			return fmt.Errorf("error configure webhook auth: %w", err)
+		}
+
+		var mux *http.ServeMux
+		if cfg.Handler == nil {
+			mux = http.NewServeMux()
+			cfg.Handler = mux
+		}
+		if cfg.routeAttacher == nil && mux != nil {
+			if err := AttachProviderRoutes(mux)(cfg); err != nil {
+				return err
+			}
+		}
+
+		auth, err := WebhookAuth(cfg.Client, cfg.NodeSpec.Name, bOpts.WebhookAuthOpts...)
+		if err != nil {
+			return err
+		}
+
+		cfg.Handler = api.InstrumentHandler(WithAuth(auth, cfg.Handler))
+		return nil
+	}
+}
+
+func configureWebhookCA(cfg *NodeConfig, bCfg *BootstrapConfig) error {
+	if bCfg.ClientCAConfigMapSpec != nil {
+		bCfg.WebhookAuthOpts = append(bCfg.WebhookAuthOpts, func(auth *WebhookAuthConfig) error {
+			cmCA, err := dynamiccertificates.NewDynamicCAFromConfigMapController("client-ca", bCfg.ClientCAConfigMapSpec.Namespace, bCfg.ClientCAConfigMapSpec.Name, bCfg.ClientCAConfigMapSpec.Key, cfg.Client)
+			if err != nil {
+				return fmt.Errorf("error loading dynamic CA from config map: %w", err)
+			}
+			auth.AuthnConfig.ClientCertificateCAContentProvider = cmCA
+			cfg.caController = cmCA
+			return nil
+		})
+		return nil
+	}
+
+	if bCfg.RestConfig.CAFile != "" {
+		bCfg.WebhookAuthOpts = append(bCfg.WebhookAuthOpts, func(auth *WebhookAuthConfig) error {
+			caFile, err := dynamiccertificates.NewDynamicCAContentFromFile("ca-file", bCfg.RestConfig.CAFile)
+			if err != nil {
+				return fmt.Errorf("error loading dynamic CA file from rest config: %w", err)
+			}
+			auth.AuthnConfig.ClientCertificateCAContentProvider = caFile
+			cfg.caController = caFile
+			return nil
+		})
+		return nil
+	}
+
+	if bCfg.RestConfig.CAData != nil {
+		bCfg.WebhookAuthOpts = append(bCfg.WebhookAuthOpts, func(auth *WebhookAuthConfig) error {
+			caData, err := dynamiccertificates.NewStaticCAContent("ca-data", bCfg.RestConfig.CAData)
+			if err != nil {
+				return fmt.Errorf("error loading static ca from rest config: %w", err)
+			}
+			auth.AuthnConfig.ClientCertificateCAContentProvider = caData
+			return nil
+		})
+		return nil
+	}
+
+	return errdefs.InvalidInput("no client CA found")
+}

node/nodeutil/provider.go

@@ -4,12 +4,13 @@ import (
 	"context"
 	"io"
 
+	dto "github.com/prometheus/client_model/go"
 	"github.com/virtual-kubelet/virtual-kubelet/node"
 	"github.com/virtual-kubelet/virtual-kubelet/node/api"
-	"github.com/virtual-kubelet/virtual-kubelet/node/api/statsv1alpha1"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	corev1listers "k8s.io/client-go/listers/core/v1"
+	statsv1alpha1 "k8s.io/kubelet/pkg/apis/stats/v1alpha1"
 )
 
 // Provider contains the methods required to implement a virtual-kubelet provider.
@@ -27,8 +28,18 @@ type Provider interface {
 	// between in/out/err and the container's stdin/stdout/stderr.
 	RunInContainer(ctx context.Context, namespace, podName, containerName string, cmd []string, attach api.AttachIO) error
 
+	// AttachToContainer attaches to the executing process of a container in the pod, copying data
+	// between in/out/err and the container's stdin/stdout/stderr.
+	AttachToContainer(ctx context.Context, namespace, podName, containerName string, attach api.AttachIO) error
+
 	// GetStatsSummary gets the stats for the node, including running pods
 	GetStatsSummary(context.Context) (*statsv1alpha1.Summary, error)
+
+	// GetMetricsResource gets the metrics for the node, including running pods
+	GetMetricsResource(context.Context) ([]*dto.MetricFamily, error)
+
+	// PortForward forwards a local port to a port on the pod
+	PortForward(ctx context.Context, namespace, pod string, port int32, stream io.ReadWriteCloser) error
 }
 
 // ProviderConfig holds objects created by NewNodeFromClient that a provider may need to bootstrap itself.
@@ -54,15 +65,18 @@ func AttachProviderRoutes(mux api.ServeMux) NodeOpt {
 	return func(cfg *NodeConfig) error {
 		cfg.routeAttacher = func(p Provider, cfg NodeConfig, pods corev1listers.PodLister) {
 			mux.Handle("/", api.PodHandler(api.PodHandlerConfig{
-				RunInContainer:   p.RunInContainer,
-				GetContainerLogs: p.GetContainerLogs,
-				GetPods:          p.GetPods,
+				RunInContainer:    p.RunInContainer,
+				AttachToContainer: p.AttachToContainer,
+				GetContainerLogs:  p.GetContainerLogs,
+				GetPods:           p.GetPods,
 				GetPodsFromKubernetes: func(context.Context) ([]*v1.Pod, error) {
 					return pods.List(labels.Everything())
 				},
 				GetStatsSummary:       p.GetStatsSummary,
+				GetMetricsResource:    p.GetMetricsResource,
 				StreamIdleTimeout:     cfg.StreamIdleTimeout,
 				StreamCreationTimeout: cfg.StreamCreationTimeout,
+				PortForward:           p.PortForward,
 			}, true))
 		}
 		return nil

node/nodeutil/tls.go

@@ -5,25 +5,28 @@ import (
 	"crypto/x509"
 	"fmt"
 	"os"
+
+	"k8s.io/client-go/rest"
 )
 
 // WithTLSConfig returns a NodeOpt which creates a base TLSConfig with the default cipher suites and tls min verions.
 // The tls config can be modified through functional options.
 func WithTLSConfig(opts ...func(*tls.Config) error) NodeOpt {
 	return func(cfg *NodeConfig) error {
-		tlsCfg := &tls.Config{
-			MinVersion:               tls.VersionTLS12,
-			PreferServerCipherSuites: true,
-			CipherSuites:             DefaultServerCiphers(),
-			ClientAuth:               tls.RequestClientCert,
+		if cfg.TLSConfig == nil {
+			cfg.TLSConfig = &tls.Config{
+				MinVersion:               tls.VersionTLS12,
+				PreferServerCipherSuites: true,
+				CipherSuites:             DefaultServerCiphers(),
+				ClientAuth:               tls.RequestClientCert,
+			}
 		}
 		for _, o := range opts {
-			if err := o(tlsCfg); err != nil {
+			if err := o(cfg.TLSConfig); err != nil {
 				return err
 			}
 		}
 
-		cfg.TLSConfig = tlsCfg
 		return nil
 	}
 }
@@ -45,7 +48,7 @@ func WithKeyPairFromPath(cert, key string) func(*tls.Config) error {
 	return func(cfg *tls.Config) error {
 		cert, err := tls.LoadX509KeyPair(cert, key)
 		if err != nil {
-			return err
+			return fmt.Errorf("error loading x509 key pair: %w", err)
 		}
 		cfg.Certificates = append(cfg.Certificates, cert)
 		return nil
@@ -56,7 +59,7 @@ func WithKeyPairFromPath(cert, key string) func(*tls.Config) error {
 // If a cert pool is not defined on the tls config an empty one will be created.
 func WithCACert(pem []byte) func(*tls.Config) error {
 	return func(cfg *tls.Config) error {
-		if cfg.ClientCAs == nil {
+		if cfg.RootCAs == nil {
 			cfg.ClientCAs = x509.NewCertPool()
 		}
 		if !cfg.ClientCAs.AppendCertsFromPEM(pem) {
@@ -81,3 +84,48 @@ func DefaultServerCiphers() []uint16 {
 		tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
 	}
 }
+
+func tlsFromRestConfig(r *rest.Config) func(*tls.Config) error {
+	return func(cfg *tls.Config) error {
+		var err error
+
+		cfg.ClientAuth = tls.RequestClientCert
+
+		certData := r.CertData
+		if certData == nil && r.CertFile != "" {
+			certData, err = os.ReadFile(r.CertFile)
+			if err != nil {
+				return fmt.Errorf("error reading cert file from clientset: %w", err)
+			}
+		}
+
+		keyData := r.KeyData
+		if keyData == nil && r.KeyFile != "" {
+			keyData, err = os.ReadFile(r.KeyFile)
+			if err != nil {
+				return fmt.Errorf("error reading key file from clientset: %w", err)
+			}
+		}
+		if keyData != nil && certData != nil {
+			pem, err := tls.X509KeyPair(certData, keyData)
+			if err != nil {
+				return fmt.Errorf("error creating key pair from clientset: %w", err)
+			}
+			cfg.Certificates = append(cfg.Certificates, pem)
+			cfg.ClientAuth = tls.RequestClientCert
+		}
+
+		caData := r.CAData
+		if certData == nil && r.CAFile != "" {
+			caData, err = os.ReadFile(r.CAFile)
+			if err != nil {
+				return fmt.Errorf("error reading ca file from clientset: %w", err)
+			}
+		}
+		if caData != nil {
+			return WithCACert(caData)(cfg)
+		}
+
+		return nil
+	}
+}

node/pod.go

@@ -69,11 +69,13 @@ func (pc *PodController) createOrUpdatePod(ctx context.Context, pod *corev1.Pod)
 		"namespace": pod.GetNamespace(),
 	})
 
-	// We do this so we don't mutate the pod from the informer cache
-	pod = pod.DeepCopy()
-	if err := podutils.PopulateEnvironmentVariables(ctx, pod, pc.resourceManager, pc.recorder); err != nil {
-		span.SetStatus(err)
-		return err
+	if !pc.skipDownwardAPIResolution {
+		// We do this so we don't mutate the pod from the informer cache
+		pod = pod.DeepCopy()
+		if err := podutils.PopulateEnvironmentVariables(ctx, pod, pc.resourceManager, pc.recorder); err != nil {
+			span.SetStatus(err)
+			return err
+		}
 	}
 
 	// We have to use a  different pod that we pass to the provider than the one that gets used in handleProviderError
@@ -122,10 +124,11 @@ func podsEqual(pod1, pod2 *corev1.Pod) bool {
 
 	return cmp.Equal(pod1.Spec.Containers, pod2.Spec.Containers) &&
 		cmp.Equal(pod1.Spec.InitContainers, pod2.Spec.InitContainers) &&
+		cmp.Equal(pod1.Spec.EphemeralContainers, pod2.Spec.EphemeralContainers) &&
 		cmp.Equal(pod1.Spec.ActiveDeadlineSeconds, pod2.Spec.ActiveDeadlineSeconds) &&
 		cmp.Equal(pod1.Spec.Tolerations, pod2.Spec.Tolerations) &&
-		cmp.Equal(pod1.ObjectMeta.Labels, pod2.Labels) &&
-		cmp.Equal(pod1.ObjectMeta.Annotations, pod2.Annotations)
+		cmp.Equal(pod1.Labels, pod2.Labels) &&
+		cmp.Equal(pod1.Annotations, pod2.Annotations)
 
 }
 
@@ -139,7 +142,7 @@ func deleteGraceTimeEqual(old, new *int64) bool {
 	return false
 }
 
-// podShouldEnqueue checks if two pods equal according according to podsEqual func and DeleteTimeStamp
+// podShouldEnqueue checks if two pods equal according to podsEqual func and DeleteTimeStamp
 func podShouldEnqueue(oldPod, newPod *corev1.Pod) bool {
 	if !podsEqual(oldPod, newPod) {
 		return true
@@ -278,7 +281,7 @@ func (pc *PodController) enqueuePodStatusUpdate(ctx context.Context, pod *corev1
 	ctx = span.WithField(ctx, "key", key)
 
 	var obj interface{}
-	err = wait.PollImmediateUntil(notificationRetryPeriod, func() (bool, error) {
+	err = wait.PollUntilContextCancel(ctx, notificationRetryPeriod, true, func(ctx context.Context) (bool, error) {
 		var ok bool
 		obj, ok = pc.knownPods.Load(key)
 		if ok {
@@ -304,14 +307,14 @@ func (pc *PodController) enqueuePodStatusUpdate(ctx context.Context, pod *corev1
 		// that we're in some kind of startup synchronization issue where the provider knows about a pod (as it performs
 		// recover, that we do not yet know about).
 		return false, nil
-	}, ctx.Done())
+	})
 
 	if err != nil {
 		if errors.IsNotFound(err) {
-			err = fmt.Errorf("Pod %q not found in pod lister: %w", key, err)
-			log.G(ctx).WithError(err).Debug("Not enqueuing pod status update")
+			err = fmt.Errorf("pod %q not found in pod lister: %w", key, err)
+			log.G(ctx).WithError(err).Debug("not enqueuing pod status update")
 		} else {
-			log.G(ctx).WithError(err).Warn("Not enqueuing pod status update due to error from pod lister")
+			log.G(ctx).WithError(err).Warn("not enqueuing pod status update due to error from pod lister")
 		}
 		span.SetStatus(err)
 		return

node/pod_test.go

@@ -17,6 +17,7 @@ package node
 import (
 	"context"
 	"fmt"
+	"reflect"
 	"testing"
 	"time"
 
@@ -56,20 +57,20 @@ func newTestController() *TestController {
 		ConfigMapInformer: iFactory.Core().V1().ConfigMaps(),
 		SecretInformer:    iFactory.Core().V1().Secrets(),
 		ServiceInformer:   iFactory.Core().V1().Services(),
-		SyncPodsFromKubernetesRateLimiter: workqueue.NewMaxOfRateLimiter(
+		SyncPodsFromKubernetesRateLimiter: workqueue.NewTypedMaxOfRateLimiter[any](
 			// The default upper bound is 1000 seconds. Let's not use that.
-			workqueue.NewItemExponentialFailureRateLimiter(5*time.Millisecond, 10*time.Millisecond),
-			&workqueue.BucketRateLimiter{Limiter: rate.NewLimiter(rate.Limit(10), 100)},
+			workqueue.NewTypedItemExponentialFailureRateLimiter[any](5*time.Millisecond, 10*time.Millisecond),
+			&workqueue.TypedBucketRateLimiter[any]{Limiter: rate.NewLimiter(rate.Limit(10), 100)},
 		),
-		SyncPodStatusFromProviderRateLimiter: workqueue.NewMaxOfRateLimiter(
+		SyncPodStatusFromProviderRateLimiter: workqueue.NewTypedMaxOfRateLimiter[any](
 			// The default upper bound is 1000 seconds. Let's not use that.
-			workqueue.NewItemExponentialFailureRateLimiter(5*time.Millisecond, 10*time.Millisecond),
-			&workqueue.BucketRateLimiter{Limiter: rate.NewLimiter(rate.Limit(10), 100)},
+			workqueue.NewTypedItemExponentialFailureRateLimiter[any](5*time.Millisecond, 10*time.Millisecond),
+			&workqueue.TypedBucketRateLimiter[any]{Limiter: rate.NewLimiter(rate.Limit(10), 100)},
 		),
-		DeletePodsFromKubernetesRateLimiter: workqueue.NewMaxOfRateLimiter(
+		DeletePodsFromKubernetesRateLimiter: workqueue.NewTypedMaxOfRateLimiter[any](
 			// The default upper bound is 1000 seconds. Let's not use that.
-			workqueue.NewItemExponentialFailureRateLimiter(5*time.Millisecond, 10*time.Millisecond),
-			&workqueue.BucketRateLimiter{Limiter: rate.NewLimiter(rate.Limit(10), 100)},
+			workqueue.NewTypedItemExponentialFailureRateLimiter[any](5*time.Millisecond, 10*time.Millisecond),
+			&workqueue.TypedBucketRateLimiter[any]{Limiter: rate.NewLimiter(rate.Limit(10), 100)},
 		),
 	})
 
@@ -211,8 +212,8 @@ func TestPodCreateNewPod(t *testing.T) {
 	svr := newTestController()
 
 	pod := &corev1.Pod{}
-	pod.ObjectMeta.Namespace = "default" //nolint:goconst
-	pod.ObjectMeta.Name = "nginx"        //nolint:goconst
+	pod.Namespace = "default" //nolint:goconst
+	pod.Name = "nginx"        //nolint:goconst
 	pod.Spec = newPodSpec()
 
 	err := svr.createOrUpdatePod(context.Background(), pod.DeepCopy())
@@ -223,12 +224,48 @@ func TestPodCreateNewPod(t *testing.T) {
 	assert.Check(t, is.Equal(svr.mock.updates.read(), 0))
 }
 
+func TestPodCreateNewPodWithNoDownwardAPIResolution(t *testing.T) {
+	svr := newTestController()
+	svr.skipDownwardAPIResolution = true
+
+	pod := &corev1.Pod{}
+	pod.Namespace = "default" //nolint:goconst
+	pod.Name = "nginx"        //nolint:goconst
+	pod.Spec = newPodSpec()
+	pod.Spec.Containers[0].Env = []corev1.EnvVar{
+		{
+			Name: "MY_NODE_NAME",
+			ValueFrom: &corev1.EnvVarSource{
+				FieldRef: &corev1.ObjectFieldSelector{
+					FieldPath: "spec.nodeName",
+				},
+			},
+		},
+	}
+
+	err := svr.createOrUpdatePod(context.Background(), pod.DeepCopy())
+	assert.Check(t, is.Nil(err))
+
+	// createOrUpdate called CreatePod but did not call UpdatePod because the pod did not exist
+	assert.Check(t, is.Equal(svr.mock.creates.read(), 1))
+	assert.Check(t, is.Equal(svr.mock.updates.read(), 0))
+
+	// make sure the downward API field in env was left alone
+	key, err := buildKey(pod)
+	assert.Check(t, is.Nil(err))
+	createdPod, ok := svr.mock.pods.Load(key)
+	assert.Check(t, ok)
+
+	// createdPod went through the pod controller logic, make sure the downward API wasn't resolved
+	assert.Check(t, is.DeepEqual(createdPod.(*corev1.Pod).Spec.Containers[0].Env, pod.Spec.Containers[0].Env))
+}
+
 func TestPodUpdateExisting(t *testing.T) {
 	svr := newTestController()
 
 	pod := &corev1.Pod{}
-	pod.ObjectMeta.Namespace = "default"
-	pod.ObjectMeta.Name = "nginx"
+	pod.Namespace = "default"
+	pod.Name = "nginx"
 	pod.Spec = newPodSpec()
 
 	err := svr.createOrUpdatePod(context.Background(), pod.DeepCopy())
@@ -251,8 +288,8 @@ func TestPodNoSpecChange(t *testing.T) {
 	svr := newTestController()
 
 	pod := &corev1.Pod{}
-	pod.ObjectMeta.Namespace = "default"
-	pod.ObjectMeta.Name = "nginx"
+	pod.Namespace = "default"
+	pod.Name = "nginx"
 	pod.Spec = newPodSpec()
 
 	err := svr.createOrUpdatePod(context.Background(), pod.DeepCopy())
@@ -272,8 +309,8 @@ func TestPodStatusDelete(t *testing.T) {
 	ctx := context.Background()
 	c := newTestController()
 	pod := &corev1.Pod{}
-	pod.ObjectMeta.Namespace = "default"
-	pod.ObjectMeta.Name = "nginx"
+	pod.Namespace = "default"
+	pod.Name = "nginx"
 	pod.Spec = newPodSpec()
 	fk8s := fake.NewSimpleClientset(pod)
 	c.client = fk8s
@@ -293,7 +330,8 @@ func TestPodStatusDelete(t *testing.T) {
 	if err != nil && !errors.IsNotFound(err) {
 		t.Fatalf("Get pod %v failed", key)
 	}
-	if newPod != nil && newPod.DeletionTimestamp == nil {
+
+	if newPod != nil && !reflect.ValueOf(*newPod).IsZero() && newPod.DeletionTimestamp == nil {
 		t.Fatalf("Pod %v delete failed", key)
 	}
 	t.Logf("pod delete success")
@@ -337,8 +375,8 @@ func TestReCreatePodRace(t *testing.T) {
 	ctx := context.Background()
 	c := newTestController()
 	pod := &corev1.Pod{}
-	pod.ObjectMeta.Namespace = "default"
-	pod.ObjectMeta.Name = "nginx"
+	pod.Namespace = "default"
+	pod.Name = "nginx"
 	pod.Spec = newPodSpec()
 	pod.UID = "aaaaa"
 	podCopy := pod.DeepCopy()

node/podcontroller.go

@@ -140,6 +140,11 @@ type PodController struct {
 	// This is used since `pc.Run()` is typically called in a goroutine and managing
 	// this can be non-trivial for callers.
 	err error
+
+	// skipDownwardAPIResolution can be used to skip any attempts at resolving downward API references
+	// in pods before calling CreatePod on the provider.
+	// Providers need this if they need to do their own custom resolving
+	skipDownwardAPIResolution bool
 }
 
 type knownPod struct {
@@ -176,17 +181,17 @@ type PodControllerConfig struct {
 	ServiceInformer   corev1informers.ServiceInformer
 
 	// SyncPodsFromKubernetesRateLimiter defines the rate limit for the SyncPodsFromKubernetes queue
-	SyncPodsFromKubernetesRateLimiter workqueue.RateLimiter
+	SyncPodsFromKubernetesRateLimiter workqueue.TypedRateLimiter[any]
 	// SyncPodsFromKubernetesShouldRetryFunc allows for a custom retry policy for the SyncPodsFromKubernetes queue
 	SyncPodsFromKubernetesShouldRetryFunc ShouldRetryFunc
 
 	// DeletePodsFromKubernetesRateLimiter defines the rate limit for the DeletePodsFromKubernetesRateLimiter queue
-	DeletePodsFromKubernetesRateLimiter workqueue.RateLimiter
+	DeletePodsFromKubernetesRateLimiter workqueue.TypedRateLimiter[any]
 	// DeletePodsFromKubernetesShouldRetryFunc allows for a custom retry policy for the SyncPodsFromKubernetes queue
 	DeletePodsFromKubernetesShouldRetryFunc ShouldRetryFunc
 
 	// SyncPodStatusFromProviderRateLimiter defines the rate limit for the SyncPodStatusFromProviderRateLimiter queue
-	SyncPodStatusFromProviderRateLimiter workqueue.RateLimiter
+	SyncPodStatusFromProviderRateLimiter workqueue.TypedRateLimiter[any]
 	// SyncPodStatusFromProviderShouldRetryFunc allows for a custom retry policy for the SyncPodStatusFromProvider queue
 	SyncPodStatusFromProviderShouldRetryFunc ShouldRetryFunc
 
@@ -196,6 +201,11 @@ type PodControllerConfig struct {
 	// For example, if the pod informer is not filtering based on pod.Spec.NodeName, you should
 	// set that filter here so the pod controller does not handle events for pods assigned to other nodes.
 	PodEventFilterFunc PodEventFilterFunc
+
+	// SkipDownwardAPIResolution can be used to skip any attempts at resolving downward API references
+	// in pods before calling CreatePod on the provider.
+	// Providers need this if they need to do their own custom resolving
+	SkipDownwardAPIResolution bool
 }
 
 // NewPodController creates a new pod controller with the provided config.
@@ -222,13 +232,13 @@ func NewPodController(cfg PodControllerConfig) (*PodController, error) {
 		return nil, errdefs.InvalidInput("missing provider")
 	}
 	if cfg.SyncPodsFromKubernetesRateLimiter == nil {
-		cfg.SyncPodsFromKubernetesRateLimiter = workqueue.DefaultControllerRateLimiter()
+		cfg.SyncPodsFromKubernetesRateLimiter = workqueue.DefaultTypedControllerRateLimiter[any]()
 	}
 	if cfg.DeletePodsFromKubernetesRateLimiter == nil {
-		cfg.DeletePodsFromKubernetesRateLimiter = workqueue.DefaultControllerRateLimiter()
+		cfg.DeletePodsFromKubernetesRateLimiter = workqueue.DefaultTypedControllerRateLimiter[any]()
 	}
 	if cfg.SyncPodStatusFromProviderRateLimiter == nil {
-		cfg.SyncPodStatusFromProviderRateLimiter = workqueue.DefaultControllerRateLimiter()
+		cfg.SyncPodStatusFromProviderRateLimiter = workqueue.DefaultTypedControllerRateLimiter[any]()
 	}
 	rm, err := manager.NewResourceManager(cfg.PodInformer.Lister(), cfg.SecretInformer.Lister(), cfg.ConfigMapInformer.Lister(), cfg.ServiceInformer.Lister())
 	if err != nil {
@@ -236,15 +246,16 @@ func NewPodController(cfg PodControllerConfig) (*PodController, error) {
 	}
 
 	pc := &PodController{
-		client:             cfg.PodClient,
-		podsInformer:       cfg.PodInformer,
-		podsLister:         cfg.PodInformer.Lister(),
-		provider:           cfg.Provider,
-		resourceManager:    rm,
-		ready:              make(chan struct{}),
-		done:               make(chan struct{}),
-		recorder:           cfg.EventRecorder,
-		podEventFilterFunc: cfg.PodEventFilterFunc,
+		client:                    cfg.PodClient,
+		podsInformer:              cfg.PodInformer,
+		podsLister:                cfg.PodInformer.Lister(),
+		provider:                  cfg.Provider,
+		resourceManager:           rm,
+		ready:                     make(chan struct{}),
+		done:                      make(chan struct{}),
+		recorder:                  cfg.EventRecorder,
+		podEventFilterFunc:        cfg.PodEventFilterFunc,
+		skipDownwardAPIResolution: cfg.SkipDownwardAPIResolution,
 	}
 
 	pc.syncPodsFromKubernetes = queue.New(cfg.SyncPodsFromKubernetesRateLimiter, "syncPodsFromKubernetes", pc.syncPodFromKubernetesHandler, cfg.SyncPodsFromKubernetesShouldRetryFunc)
@@ -402,7 +413,10 @@ func (pc *PodController) Run(ctx context.Context, podSyncWorkers int) (retErr er
 		}
 	}
 
-	pc.podsInformer.Informer().AddEventHandler(eventHandler)
+	_, err := pc.podsInformer.Informer().AddEventHandler(eventHandler)
+	if err != nil {
+		log.G(ctx).Error(err)
+	}
 
 	// Perform a reconciliation step that deletes any dangling pods from the provider.
 	// This happens only when the virtual-kubelet is starting, and operates on a "best-effort" basis.
@@ -450,6 +464,51 @@ func (pc *PodController) Err() error {
 	return pc.err
 }
 
+// SyncPodsFromKubernetesQueueLen returns the length of the SyncPodsFromKubernetes queue, which include items being processed and unprocessed
+func (pc *PodController) SyncPodsFromKubernetesQueueLen() int {
+	return pc.syncPodsFromKubernetes.Len()
+}
+
+// SyncPodsFromKubernetesQueueUnprocessedLen returns the length of the unprocessed items in the SyncPodsFromKubernetes queue
+func (pc *PodController) SyncPodsFromKubernetesQueueUnprocessedLen() int {
+	return pc.syncPodsFromKubernetes.UnprocessedLen()
+}
+
+// SyncPodsFromKubernetesQueueItemsBeingProcessedLen returns the length of the items being processed in the SyncPodsFromKubernetes queue
+func (pc *PodController) SyncPodsFromKubernetesQueueItemsBeingProcessedLen() int {
+	return pc.syncPodsFromKubernetes.ItemsBeingProcessedLen()
+}
+
+// DeletePodsFromKubernetesQueueLen returns the length of the DeletePodsFromKubernetes queue, which include items being processed and unprocessed
+func (pc *PodController) DeletePodsFromKubernetesQueueLen() int {
+	return pc.deletePodsFromKubernetes.Len()
+}
+
+// DeletePodsFromKubernetesQueueUnprocessedLen returns the length of the unprocessed items in the DeletePodsFromKubernetes queue
+func (pc *PodController) DeletePodsFromKubernetesQueueUnprocessedLen() int {
+	return pc.deletePodsFromKubernetes.UnprocessedLen()
+}
+
+// DeletePodsFromKubernetesQueueItemsBeingProcessedLen returns the length of the items being processed in the DeletePodsFromKubernetes queue
+func (pc *PodController) DeletePodsFromKubernetesQueueItemsBeingProcessedLen() int {
+	return pc.deletePodsFromKubernetes.ItemsBeingProcessedLen()
+}
+
+// SyncPodStatusFromProviderQueueLen returns the length of the SyncPodStatusFromProvider queue, which include items being processed and unprocessed
+func (pc *PodController) SyncPodStatusFromProviderQueueLen() int {
+	return pc.syncPodStatusFromProvider.Len()
+}
+
+// SyncPodStatusFromProviderQueueUnprocessedLen returns the length of the unprocessed items in the SyncPodStatusFromProvider queue
+func (pc *PodController) SyncPodStatusFromProviderQueueUnprocessedLen() int {
+	return pc.syncPodStatusFromProvider.UnprocessedLen()
+}
+
+// SyncPodStatusFromProviderQueueItemsBeingProcessedLen returns the length of the items being processed in the SyncPodStatusFromProvider queue
+func (pc *PodController) SyncPodStatusFromProviderQueueItemsBeingProcessedLen() int {
+	return pc.syncPodStatusFromProvider.ItemsBeingProcessedLen()
+}
+
 // syncPodFromKubernetesHandler compares the actual state with the desired, and attempts to converge the two.
 func (pc *PodController) syncPodFromKubernetesHandler(ctx context.Context, key string) error {
 	ctx, span := trace.StartSpan(ctx, "syncPodFromKubernetesHandler")

node/podcontroller_test.go

@@ -123,8 +123,8 @@ func TestPodEventFilter(t *testing.T) {
 	}
 
 	pod := &corev1.Pod{}
-	pod.ObjectMeta.Namespace = "default"
-	pod.ObjectMeta.Name = "nginx"
+	pod.Namespace = "default"
+	pod.Name = "nginx"
 	pod.Spec = newPodSpec()
 
 	podC := tc.client.CoreV1().Pods(testNamespace)

node/sync.go

@@ -158,7 +158,7 @@ func (p *syncProviderWrapper) updatePodStatus(ctx context.Context, podFromKubern
 	ctx = addPodAttributes(ctx, span, podFromKubernetes)
 
 	var statusErr error
-	podStatus, err := p.PodLifecycleHandler.GetPodStatus(ctx, podFromKubernetes.Namespace, podFromKubernetes.Name)
+	podStatus, err := p.GetPodStatus(ctx, podFromKubernetes.Namespace, podFromKubernetes.Name)
 	if err != nil {
 		if !errdefs.IsNotFound(err) {
 			span.SetStatus(err)
@@ -184,7 +184,7 @@ func (p *syncProviderWrapper) updatePodStatus(ctx context.Context, podFromKubern
 		return nil
 	}
 
-	if podFromKubernetes.Status.Phase != corev1.PodRunning && time.Since(podFromKubernetes.ObjectMeta.CreationTimestamp.Time) <= time.Minute {
+	if podFromKubernetes.Status.Phase != corev1.PodRunning && time.Since(podFromKubernetes.CreationTimestamp.Time) <= time.Minute {
 		span.SetStatus(statusErr)
 		return statusErr
 	}

test/e2e/basic.go

@@ -1,16 +1,18 @@
 package e2e
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"testing"
 	"time"
 
+	"github.com/prometheus/common/expfmt"
 	"github.com/virtual-kubelet/virtual-kubelet/internal/podutils"
-	stats "github.com/virtual-kubelet/virtual-kubelet/node/api/statsv1alpha1"
 	"gotest.tools/assert"
 	v1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	stats "k8s.io/kubelet/pkg/apis/stats/v1alpha1"
 )
 
 const (
@@ -111,6 +113,72 @@ func (ts *EndToEndTestSuite) TestGetStatsSummary(t *testing.T) {
 	}
 }
 
+// TestGetMetricsResource creates a pod having two containers and queries the /metrics/resource endpoint of the virtual-kubelet.
+// It expects this endpoint to return stats for the current node, as well as for the aforementioned pod and each of its two containers.
+func (ts *EndToEndTestSuite) TestGetMetricsResource(t *testing.T) {
+	ctx := context.Background()
+
+	// Create a pod with prefix "nginx-" having three containers.
+	pod, err := f.CreatePod(ctx, f.CreateDummyPodObjectWithPrefix(t.Name(), "nginx", "foo", "bar", "baz"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Delete the "nginx-0-X" pod after the test finishes.
+	defer func() {
+		if err := f.DeletePodImmediately(ctx, pod.Namespace, pod.Name); err != nil && !apierrors.IsNotFound(err) {
+			t.Error(err)
+		}
+	}()
+
+	// Wait for the "nginx-" pod to be reported as running and ready.
+	if _, err := f.WaitUntilPodReady(pod.Namespace, pod.Name); err != nil {
+		t.Fatal(err)
+	}
+
+	// Grab the stats from the provider.
+	metricsResourceResponse, err := f.GetMetricsResource(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// decode metrics response bytes to metric family
+	reader := bytes.NewReader(metricsResourceResponse)
+	parser := expfmt.TextParser{}
+	metricsFamilyMap, err := parser.TextToMetricFamilies(reader)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Make sure the "nginx-" pod exists in the metrics returned.
+	currentContainerStatsCount := 0
+	found := false
+	for metricName, metricFamily := range metricsFamilyMap {
+		if metricName == "pod_cpu_usage_seconds_total" {
+			for _, metric := range metricFamily.Metric {
+				if *metric.Label[1].Value == pod.Name {
+					found = true
+				}
+			}
+		}
+		if metricName == "container_cpu_usage_seconds_total" {
+			for _, metric := range metricFamily.Metric {
+				if *metric.Label[1].Value == pod.Name {
+					currentContainerStatsCount += 1
+				}
+			}
+		}
+	}
+	if !found {
+		t.Fatalf("Pod %s not found in metrics", pod.Name)
+	}
+
+	// Make sure that we've got stats for all the containers in the "nginx-" pod.
+	desiredContainerStatsCount := len(pod.Spec.Containers)
+	if currentContainerStatsCount != desiredContainerStatsCount {
+		t.Fatalf("expected stats for %d containers, got stats for %d containers", desiredContainerStatsCount, currentContainerStatsCount)
+	}
+}
+
 // TestPodLifecycleGracefulDelete creates a pod and verifies that the provider has been asked to create it.
 // Then, it deletes the pods and verifies that the provider has been asked to delete it.
 // These verifications are made using the /stats/summary endpoint of the virtual-kubelet, by checking for the presence or absence of the pods.
@@ -183,8 +251,8 @@ func (ts *EndToEndTestSuite) TestPodLifecycleGracefulDelete(t *testing.T) {
 
 	// Make sure we saw the delete event, and the delete event was graceful
 	assert.Assert(t, podLast != nil)
-	assert.Assert(t, podLast.ObjectMeta.GetDeletionGracePeriodSeconds() != nil)
-	assert.Assert(t, *podLast.ObjectMeta.GetDeletionGracePeriodSeconds() > 0)
+	assert.Assert(t, podLast.GetDeletionGracePeriodSeconds() != nil)
+	assert.Assert(t, *podLast.GetDeletionGracePeriodSeconds() > 0)
 }
 
 // TestPodLifecycleForceDelete creates one podsand verifies that the provider has created them

trace/opencensus/opencensus.go

@@ -79,8 +79,12 @@ func (s *span) SetStatus(err error) {
 		status.Code = octrace.StatusCodeNotFound
 	case errdefs.IsInvalidInput(err):
 		status.Code = octrace.StatusCodeInvalidArgument
-		// TODO: other error types
+	case errdefs.IsForbidden(err):
+		status.Code = octrace.StatusCodePermissionDenied
+	case errdefs.IsUnauthorized(err):
+		status.Code = octrace.StatusCodeUnauthenticated
 	default:
+		// TODO: other error types
 		status.Code = octrace.StatusCodeUnknown
 	}
 

trace/opentelemetry/opentelemetry.go

@@ -257,7 +257,7 @@ func makeAttribute(key string, val interface{}) (attr attribute.KeyValue) {
 	// 	return attribute.BoolSlice(key, v)
 	case error:
 		if v == nil {
-			attribute.String(key, "")
+			return attribute.String(key, "")
 		}
 		return attribute.String(key, v.Error())
 	default:

trace/opentelemetry/opentelemetry_test.go

@@ -30,8 +30,7 @@ import (
 	"go.opentelemetry.io/otel/codes"
 	"go.opentelemetry.io/otel/sdk/resource"
 	sdktrace "go.opentelemetry.io/otel/sdk/trace"
-	semconv "go.opentelemetry.io/otel/semconv"
-	"go.opentelemetry.io/otel/trace"
+	semconv "go.opentelemetry.io/otel/semconv/v1.17.0"
 	"gotest.tools/assert"
 	"gotest.tools/assert/cmp"
 )
@@ -92,7 +91,6 @@ func TestSetStatus(t *testing.T) {
 
 			assert.Assert(t, !s.s.IsRecording())
 			assert.Assert(t, e.status == tt.expectedCode)
-			assert.Assert(t, e.statusMessage == tt.expectedDescription)
 			s.SetStatus(tt.inputStatus) // should not be panic even if span is ended.
 		})
 	}
@@ -203,7 +201,7 @@ func TestLog(t *testing.T) {
 		logLevel           logLevel
 		fields             log.Fields
 		msg                string
-		expectedEvents     []trace.Event
+		expectedEvents     []sdktrace.Event
 		expectedAttributes []attribute.KeyValue
 	}{
 		{
@@ -212,7 +210,7 @@ func TestLog(t *testing.T) {
 			logLevel:           lDebug,
 			fields:             log.Fields{"testKey1": "value1"},
 			msg:                "message",
-			expectedEvents:     []trace.Event{{Name: "message"}},
+			expectedEvents:     []sdktrace.Event{{Name: "message"}},
 			expectedAttributes: []attribute.KeyValue{{Key: "testKey1", Value: attribute.StringValue("value1")}},
 		}, {
 			description:        "info",
@@ -220,7 +218,7 @@ func TestLog(t *testing.T) {
 			logLevel:           lInfo,
 			fields:             log.Fields{"testKey1": "value1"},
 			msg:                "message",
-			expectedEvents:     []trace.Event{{Name: "message"}},
+			expectedEvents:     []sdktrace.Event{{Name: "message"}},
 			expectedAttributes: []attribute.KeyValue{{Key: "testKey1", Value: attribute.StringValue("value1")}},
 		}, {
 			description:        "warn",
@@ -228,7 +226,7 @@ func TestLog(t *testing.T) {
 			logLevel:           lWarn,
 			fields:             log.Fields{"testKey1": "value1"},
 			msg:                "message",
-			expectedEvents:     []trace.Event{{Name: "message"}},
+			expectedEvents:     []sdktrace.Event{{Name: "message"}},
 			expectedAttributes: []attribute.KeyValue{{Key: "testKey1", Value: attribute.StringValue("value1")}},
 		}, {
 			description:        "error",
@@ -236,7 +234,7 @@ func TestLog(t *testing.T) {
 			logLevel:           lErr,
 			fields:             log.Fields{"testKey1": "value1"},
 			msg:                "message",
-			expectedEvents:     []trace.Event{{Name: "message"}},
+			expectedEvents:     []sdktrace.Event{{Name: "message"}},
 			expectedAttributes: []attribute.KeyValue{{Key: "testKey1", Value: attribute.StringValue("value1")}},
 		}, {
 			description:        "fatal",
@@ -244,7 +242,7 @@ func TestLog(t *testing.T) {
 			logLevel:           lFatal,
 			fields:             log.Fields{"testKey1": "value1"},
 			msg:                "message",
-			expectedEvents:     []trace.Event{{Name: "message"}},
+			expectedEvents:     []sdktrace.Event{{Name: "message"}},
 			expectedAttributes: []attribute.KeyValue{{Key: "testKey1", Value: attribute.StringValue("value1")}},
 		},
 	}
@@ -296,7 +294,7 @@ func TestLogf(t *testing.T) {
 		msg                string
 		fields             log.Fields
 		args               []interface{}
-		expectedEvents     []trace.Event
+		expectedEvents     []sdktrace.Event
 		expectedAttributes []attribute.KeyValue
 	}{
 		{
@@ -306,7 +304,7 @@ func TestLogf(t *testing.T) {
 			msg:            "k1: %s, k2: %v, k3: %d, k4: %v",
 			fields:         map[string]interface{}{"k1": "test", "k2": []string{"test"}, "k3": 1, "k4": []int{1}},
 			args:           []interface{}{"test", []string{"test"}, int(1), []int{1}},
-			expectedEvents: []trace.Event{{Name: "k1: test, k2: [test], k3: 1, k4: [1]"}},
+			expectedEvents: []sdktrace.Event{{Name: "k1: test, k2: [test], k3: 1, k4: [1]"}},
 			expectedAttributes: []attribute.KeyValue{
 				attribute.String("k1", "test"),
 				attribute.String("k2", fmt.Sprintf("%+v", []string{"test"})),
@@ -320,7 +318,7 @@ func TestLogf(t *testing.T) {
 			msg:            "k1: %d, k2: %v, k3: %f, k4: %v",
 			fields:         map[string]interface{}{"k1": int64(3), "k2": []int64{4}, "k3": float64(2), "k4": []float64{4}},
 			args:           []interface{}{int64(3), []int64{4}, float64(2), []float64{4}},
-			expectedEvents: []trace.Event{{Name: "k1: 3, k2: [4], k3: 2.000000, k4: [4]"}},
+			expectedEvents: []sdktrace.Event{{Name: "k1: 3, k2: [4], k3: 2.000000, k4: [4]"}},
 			expectedAttributes: []attribute.KeyValue{
 				attribute.Int64("k1", 1),
 				attribute.String("k2", fmt.Sprintf("%+v", []int64{2})),
@@ -334,7 +332,7 @@ func TestLogf(t *testing.T) {
 			msg:            "k1: %v, k2: %v",
 			fields:         map[string]interface{}{"k1": map[int]int{1: 1}, "k2": num(1)},
 			args:           []interface{}{map[int]int{1: 1}, num(1)},
-			expectedEvents: []trace.Event{{Name: "k1: map[1:1], k2: 1"}},
+			expectedEvents: []sdktrace.Event{{Name: "k1: map[1:1], k2: 1"}},
 			expectedAttributes: []attribute.KeyValue{
 				attribute.String("k1", "{1:1}"),
 				attribute.Stringer("k2", num(1)),
@@ -346,7 +344,7 @@ func TestLogf(t *testing.T) {
 			msg:            "k1: %t, k2: %v, k3: %s",
 			fields:         map[string]interface{}{"k1": true, "k2": []bool{true}, "k3": errors.New("fake")},
 			args:           []interface{}{true, []bool{true}, errors.New("fake")},
-			expectedEvents: []trace.Event{{Name: "k1: true, k2: [true], k3: fake"}},
+			expectedEvents: []sdktrace.Event{{Name: "k1: true, k2: [true], k3: fake"}},
 			expectedAttributes: []attribute.KeyValue{
 				attribute.Bool("k1", true),
 				attribute.String("k2", fmt.Sprintf("%+v", []bool{true})),
@@ -356,13 +354,12 @@ func TestLogf(t *testing.T) {
 			description:        "fatal",
 			spanName:           "test",
 			logLevel:           lFatal,
-			expectedEvents:     []trace.Event{{Name: ""}},
+			expectedEvents:     []sdktrace.Event{{Name: ""}},
 			expectedAttributes: []attribute.KeyValue{},
 		},
 	}
 
 	for _, tt := range testCases {
-		tt := tt
 		t.Run(tt.description, func(t *testing.T) {
 			tearDown, p, e := setupSuite()
 			defer tearDown(p)
@@ -389,8 +386,6 @@ func TestLogf(t *testing.T) {
 
 			assert.Assert(t, len(e.events) == len(tt.expectedEvents))
 			for i, event := range tt.expectedEvents {
-				event := event
-				i := i
 				t.Run(fmt.Sprintf("event %s", event.Name), func(t *testing.T) {
 					assert.Check(t, cmp.Equal(e.events[i].Name, event.Name))
 					assert.Check(t, !e.events[i].Time.IsZero())
@@ -405,7 +400,6 @@ func TestLogf(t *testing.T) {
 				return fl.a[i].Key < fl.a[j].Key
 			})
 			for i, a := range tt.expectedAttributes {
-				a := a
 				t.Run(fmt.Sprintf("attribute %s", a.Key), func(t *testing.T) {
 					assert.Assert(t, fl.a[i].Key == a.Key)
 					assert.Assert(t, cmp.Equal(fl.a[i].Value.Type(), a.Value.Type()))
@@ -579,8 +573,7 @@ func setupSuite() (func(provider *sdktrace.TracerProvider), *sdktrace.TracerProv
 }
 
 func NewResource(name, version string) *resource.Resource {
-	return resource.NewWithAttributes(
-		semconv.ServiceNameKey.String(name),
+	return resource.NewWithAttributes(name,
 		semconv.ServiceVersionKey.String(version),
 	)
 }
@@ -590,36 +583,34 @@ type fakeExporter struct {
 	// attributes describe the aspects of the spans.
 	attributes []attribute.KeyValue
 	// Links returns all the links the span has to other spans.
-	links []trace.Link
+	links []sdktrace.Link
 	// Events returns all the events that occurred within in the spans
 	// lifetime.
-	events []trace.Event
+	events []sdktrace.Event
 	// Status returns the spans status.
-	status        codes.Code
-	statusMessage string
+	status codes.Code
 }
 
-func (f *fakeExporter) ExportSpans(_ context.Context, spans []*sdktrace.SpanSnapshot) error {
+func (f *fakeExporter) ExportSpans(_ context.Context, spans []sdktrace.ReadOnlySpan) error {
 	f.Lock()
 	defer f.Unlock()
 
 	f.attributes = make([]attribute.KeyValue, 0)
-	f.links = make([]trace.Link, 0)
-	f.events = make([]trace.Event, 0)
+	f.links = make([]sdktrace.Link, 0)
+	f.events = make([]sdktrace.Event, 0)
 	for _, s := range spans {
-		f.attributes = append(f.attributes, s.Attributes...)
-		f.links = append(f.links, s.Links...)
-		f.events = append(f.events, s.MessageEvents...)
-		f.status = s.StatusCode
-		f.statusMessage = s.StatusMessage
+		f.attributes = append(f.attributes, s.Attributes()...)
+		f.links = append(f.links, s.Links()...)
+		f.events = append(f.events, s.Events()...)
+		f.status = s.Status().Code
 	}
 	return nil
 }
 
 func (f *fakeExporter) Shutdown(_ context.Context) (err error) {
 	f.attributes = make([]attribute.KeyValue, 0)
-	f.links = make([]trace.Link, 0)
-	f.events = make([]trace.Event, 0)
+	f.links = make([]sdktrace.Link, 0)
+	f.events = make([]sdktrace.Event, 0)
 	return
 }
 

website/data/providers.yaml

@@ -19,6 +19,9 @@
   tag: huawei-cci
 - name: HashiCorp Nomad
   tag: nomad
+- name: InterLink
+  tag: interlink
+  org: intertwin-eu
 - name: Liqo
   tag: liqo
   org: liqotech
@@ -26,3 +29,6 @@
   tag: openstack-zun
 - name: Tencent Games Tensile Kube
   tag: tensile-kube
+- name: StackPath Edge Compute
+  tag: virtual-kubelet-stackpath
+  org: stackpath