Add webhook and anon auth support

Auth is not automatically enabled because this requires some
bootstrapping to work.
I'll leave this for some future work.
In the meantime people can use the current code similar to how they used
the node-cli code to inject their own auth.

cmd/virtual-kubelet/internal/commands/root/http.go

@@ -15,45 +15,15 @@
 package root
 
 import (
-	"crypto/tls"
 	"fmt"
 	"os"
 	"time"
-
-	"github.com/pkg/errors"
 )
 
-// AcceptedCiphers is the list of accepted TLS ciphers, with known weak ciphers elided
-// Note this list should be a moving target.
-var AcceptedCiphers = []uint16{
-	tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
-	tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
-	tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
-	tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
-
-	tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
-	tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
-	tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
-	tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
-}
-
-func loadTLSConfig(certPath, keyPath string) (*tls.Config, error) {
-	cert, err := tls.LoadX509KeyPair(certPath, keyPath)
-	if err != nil {
-		return nil, errors.Wrap(err, "error loading tls certs")
-	}
-
-	return &tls.Config{
-		Certificates:             []tls.Certificate{cert},
-		MinVersion:               tls.VersionTLS12,
-		PreferServerCipherSuites: true,
-		CipherSuites:             AcceptedCiphers,
-	}, nil
-}
-
 type apiServerConfig struct {
 	CertPath              string
 	KeyPath               string
+	CACertPath            string
 	Addr                  string
 	MetricsAddr           string
 	StreamIdleTimeout     time.Duration
@@ -62,8 +32,9 @@ type apiServerConfig struct {
 
 func getAPIConfig(c Opts) (*apiServerConfig, error) {
 	config := apiServerConfig{
-		CertPath: os.Getenv("APISERVER_CERT_LOCATION"),
+		CertPath:   os.Getenv("APISERVER_CERT_LOCATION"),
-		KeyPath:  os.Getenv("APISERVER_KEY_LOCATION"),
+		KeyPath:    os.Getenv("APISERVER_KEY_LOCATION"),
+		CACertPath: os.Getenv("APISERVER_CA_CERT_LOCATION"),
 	}
 
 	config.Addr = fmt.Sprintf(":%d", c.ListenPort)

cmd/virtual-kubelet/internal/commands/root/root.go

@@ -16,6 +16,8 @@ package root
 
 import (
 	"context"
+	"crypto/tls"
+	"net/http"
 	"os"
 	"runtime"
 
@@ -26,8 +28,11 @@ import (
 	"github.com/virtual-kubelet/virtual-kubelet/internal/manager"
 	"github.com/virtual-kubelet/virtual-kubelet/log"
 	"github.com/virtual-kubelet/virtual-kubelet/node"
+	"github.com/virtual-kubelet/virtual-kubelet/node/api"
 	"github.com/virtual-kubelet/virtual-kubelet/node/nodeutil"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apiserver/pkg/server/dynamiccertificates"
+	"k8s.io/client-go/kubernetes"
 )
 
 // NewCommand creates a new top-level command.
@@ -74,8 +79,8 @@ func runRootCommand(ctx context.Context, s *provider.Store, c Opts) error {
 		return err
 	}
 
+	mux := http.NewServeMux()
 	newProvider := func(cfg nodeutil.ProviderConfig) (nodeutil.Provider, node.NodeProvider, error) {
-		var err error
 		rm, err := manager.NewResourceManager(cfg.Pods, cfg.Secrets, cfg.ConfigMaps, cfg.Services)
 		if err != nil {
 			return nil, nil, errors.Wrap(err, "could not create resource manager")
@@ -99,11 +104,17 @@ func runRootCommand(ctx context.Context, s *provider.Store, c Opts) error {
 			return nil, nil, errors.Wrapf(err, "error initializing provider %s", c.Provider)
 		}
 		p.ConfigureNode(ctx, cfg.Node)
-
+		cfg.Node.Status.NodeInfo.KubeletVersion = c.Version
 		return p, nil, nil
 	}
 
-	cm, err := nodeutil.NewNodeFromClient(client, c.NodeName, newProvider, func(cfg *nodeutil.NodeConfig) error {
+	apiConfig, err := getAPIConfig(c)
+	if err != nil {
+		return err
+	}
+
+	cm, err := nodeutil.NewNodeFromClient(c.NodeName, newProvider, func(cfg *nodeutil.NodeConfig) error {
+		cfg.Handler = mux
 		cfg.InformerResyncPeriod = c.InformerResyncPeriod
 
 		if taint != nil {
@@ -112,23 +123,23 @@ func runRootCommand(ctx context.Context, s *provider.Store, c Opts) error {
 		cfg.NodeSpec.Status.NodeInfo.Architecture = runtime.GOARCH
 		cfg.NodeSpec.Status.NodeInfo.OperatingSystem = c.OperatingSystem
 
-		apiConfig, err := getAPIConfig(c)
-		if err != nil {
-			return err
-		}
-
 		cfg.HTTPListenAddr = apiConfig.Addr
 		cfg.StreamCreationTimeout = apiConfig.StreamCreationTimeout
 		cfg.StreamIdleTimeout = apiConfig.StreamIdleTimeout
-
-		cfg.TLSConfig, err = loadTLSConfig(apiConfig.CertPath, apiConfig.KeyPath)
-		if err != nil {
-			return errors.Wrap(err, "error loading tls config")
-		}
-
 		cfg.DebugHTTP = true
+
+		cfg.NumWorkers = c.PodSyncWorkers
+
 		return nil
-	})
+	},
+		nodeutil.WithClient(client),
+		setAuth(client, c.NodeName, apiConfig),
+		nodeutil.WithTLSConfig(
+			nodeutil.WithKeyPairFromPath(apiConfig.CertPath, apiConfig.KeyPath),
+			maybeCA(apiConfig.CACertPath),
+		),
+		nodeutil.AttachProviderRoutes(mux),
+	)
 	if err != nil {
 		return err
 	}
@@ -144,7 +155,7 @@ func runRootCommand(ctx context.Context, s *provider.Store, c Opts) error {
 		"watchedNamespace": c.KubeNamespace,
 	}))
 
-	go cm.Run(ctx, c.PodSyncWorkers) // nolint:errcheck
+	go cm.Run(ctx) //nolint:errcheck
 
 	defer func() {
 		log.G(ctx).Debug("Waiting for controllers to be done")
@@ -166,3 +177,32 @@ func runRootCommand(ctx context.Context, s *provider.Store, c Opts) error {
 	}
 	return nil
 }
+
+func setAuth(client kubernetes.Interface, node string, apiCfg *apiServerConfig) nodeutil.NodeOpt {
+	if apiCfg.CACertPath == "" {
+		return func(cfg *nodeutil.NodeConfig) error {
+			cfg.Handler = api.InstrumentHandler(nodeutil.WithAuth(nodeutil.NoAuth(), cfg.Handler))
+			return nil
+		}
+	}
+
+	return func(cfg *nodeutil.NodeConfig) error {
+		auth, err := nodeutil.WebhookAuth(client, node, func(cfg *nodeutil.WebhookAuthConfig) error {
+			var err error
+			cfg.AuthnConfig.ClientCertificateCAContentProvider, err = dynamiccertificates.NewDynamicCAContentFromFile("ca-cert-bundle", apiCfg.CACertPath)
+			return err
+		})
+		if err != nil {
+			return err
+		}
+		cfg.Handler = api.InstrumentHandler(nodeutil.WithAuth(auth, cfg.Handler))
+		return nil
+	}
+}
+
+func maybeCA(p string) func(*tls.Config) error {
+	if p == "" {
+		return func(*tls.Config) error { return nil }
+	}
+	return nodeutil.WithCAFromPath(p)
+}

go.sum

@@ -34,8 +34,10 @@ github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb0
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
 github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
+github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI=
 github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
 github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
+github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M=
 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
 github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
 github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
@@ -55,6 +57,7 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
+github.com/blang/semver v3.5.0+incompatible h1:CGxCgetQ64DKk7rdZ++Vfnb1+ogGNnB17OJKJXD2Cfs=
 github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
 github.com/bombsimon/logrusr v1.0.0 h1:CTCkURYAt5nhCCnKH9eLShYayj2/8Kn/4Qg3QfiU+Ro=
 github.com/bombsimon/logrusr v1.0.0/go.mod h1:Jq0nHtvxabKE5EMwAAdgTaz7dfWE8C4i11NOltxGQpc=
@@ -140,11 +143,13 @@ github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+
 github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M=
 github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M=
 github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg=
+github.com/go-openapi/jsonpointer v0.19.3 h1:gihV7YNZK1iK6Tgwwsxo2rJbD1GTbdm72325Bq8FI3w=
 github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
 github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg=
 github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I=
 github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I=
 github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc=
+github.com/go-openapi/jsonreference v0.19.3 h1:5cxNfTy0UVC3X8JL5ymxzyoUZmo8iZb+jeTWn7tUa8o=
 github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8=
 github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU=
 github.com/go-openapi/loads v0.18.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU=
@@ -158,6 +163,7 @@ github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nA
 github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI=
 github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI=
 github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY=
+github.com/go-openapi/spec v0.19.3 h1:0XRyw8kguri6Yw4SxhsQA/atC88yqrk0+G4YhI2wabc=
 github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo=
 github.com/go-openapi/strfmt v0.17.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU=
 github.com/go-openapi/strfmt v0.18.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU=
@@ -167,6 +173,7 @@ github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dp
 github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg=
 github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg=
 github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
+github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY=
 github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
 github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4=
 github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA=
@@ -281,6 +288,7 @@ github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN
 github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM=
 github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
@@ -371,6 +379,7 @@ github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrf
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
+github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
@@ -704,6 +713,7 @@ k8s.io/utils v0.0.0-20200912215256-4140de9c8800 h1:9ZNvfPvVIEsp/T1ez4GQuzCcCTEQW
 k8s.io/utils v0.0.0-20200912215256-4140de9c8800/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.9/go.mod h1:dzAXnQbTRyDlZPJX2SUPEqvnB+j7AJjtlox7PEwigU0=
+sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15 h1:4uqm9Mv+w2MmBYD+F4qf/v6tDFUdPOk29C095RbU5mY=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg=
 sigs.k8s.io/controller-runtime v0.7.1 h1:nqVwzVzdenfd9xIbB35pC7JJH2IXVL4hDo3MNzkyCh4=
 sigs.k8s.io/controller-runtime v0.7.1/go.mod h1:pJ3YBrJiAqMAZKi6UVGuE98ZrroV1p+pIhoHsMm9wdU=

hack/skaffold/virtual-kubelet/base.yml

@@ -56,6 +56,14 @@ rules:
   verbs:
   - create
   - patch
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - get
+  - create
+  - update
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding

hack/skaffold/virtual-kubelet/pod.yml

@@ -4,6 +4,8 @@ metadata:
   name: vkubelet-mock-0
 spec:
   containers:
+  - name: jaeger-tracing
+    image: jaegertracing/all-in-one:1.22
   - name: vkubelet-mock-0
     image: virtual-kubelet
     # "IfNotPresent" is used to prevent Minikube from trying to pull from the registry (and failing) in the first place.
@@ -23,7 +25,12 @@ spec:
     - --klog.logtostderr
     - --log-level
     - debug
+    - --trace-exporter
+    - jaeger
+    - --trace-sample-rate=always
     env:
+    - name: JAEGER_AGENT_ENDPOINT
+      value: localhost:6831
     - name: KUBELET_PORT
       value: "10250"
     - name: VKUBELET_POD_IP

node/nodeutil/auth.go

@@ -0,0 +1,220 @@
+package nodeutil
+
+import (
+	"context"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/virtual-kubelet/virtual-kubelet/log"
+	"github.com/virtual-kubelet/virtual-kubelet/trace"
+	"k8s.io/apiserver/pkg/authentication/authenticator"
+	"k8s.io/apiserver/pkg/authentication/authenticatorfactory"
+	"k8s.io/apiserver/pkg/authentication/request/anonymous"
+	"k8s.io/apiserver/pkg/authentication/user"
+	"k8s.io/apiserver/pkg/authorization/authorizer"
+	"k8s.io/apiserver/pkg/authorization/authorizerfactory"
+	"k8s.io/client-go/kubernetes"
+)
+
+// Auth is the interface used to implement authn/authz for http requests
+type Auth interface {
+	authenticator.Request
+	authorizer.RequestAttributesGetter
+	authorizer.Authorizer
+}
+
+type authWrapper struct {
+	authenticator.Request
+	authorizer.RequestAttributesGetter
+	authorizer.Authorizer
+}
+
+// InstrumentAuth wraps the provided Auth in a new instrumented Auth
+//
+// Note: You would only need this if you rolled your own auth.
+// 	The Auth implementations defined in this package are already instrumented.
+func InstrumentAuth(auth Auth) Auth {
+	if _, ok := auth.(*authWrapper); ok {
+		// This is already instrumented
+		return auth
+	}
+	return &authWrapper{
+		Request:                 auth,
+		RequestAttributesGetter: auth,
+		Authorizer:              auth,
+	}
+}
+
+// NoAuth creates an Auth which allows anonymous access to all resouorces
+func NoAuth() Auth {
+	return &authWrapper{
+		Request:                 anonymous.NewAuthenticator(),
+		RequestAttributesGetter: &NodeRequestAttr{},
+		Authorizer:              authorizerfactory.NewAlwaysAllowAuthorizer(),
+	}
+}
+
+// WithAuth makes a new http handler which wraps the provided handler with authn/authz.
+func WithAuth(auth Auth, h http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		handleAuth(auth, w, r, h)
+	})
+}
+
+func handleAuth(auth Auth, w http.ResponseWriter, r *http.Request, next http.Handler) {
+	ctx := r.Context()
+	ctx, span := trace.StartSpan(ctx, "vk.handleAuth")
+	defer span.End()
+	r = r.WithContext(ctx)
+
+	info, ok, err := auth.AuthenticateRequest(r)
+	if err != nil || !ok {
+		log.G(r.Context()).WithError(err).Error("Authorization error")
+		http.Error(w, "Unauthorized", http.StatusUnauthorized)
+		return
+	}
+
+	logger := log.G(ctx).WithFields(log.Fields{
+		"user-name": info.User.GetName(),
+		"user-id":   info.User.GetUID(),
+	})
+
+	ctx = log.WithLogger(ctx, logger)
+	r = r.WithContext(ctx)
+
+	attrs := auth.GetRequestAttributes(info.User, r)
+
+	decision, _, err := auth.Authorize(ctx, attrs)
+	if err != nil {
+		log.G(r.Context()).WithError(err).Error("Authorization error")
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	if decision != authorizer.DecisionAllow {
+		http.Error(w, "Forbidden", http.StatusForbidden)
+		return
+	}
+
+	next.ServeHTTP(w, r)
+}
+
+// WebhookAuthOption is used as a functional argument to configure webhook auth.
+type WebhookAuthOption func(*WebhookAuthConfig) error
+
+// WebhookAuthConfig stores the configurations for authn/authz and is used by WebhookAuthOption to expose to callers.
+type WebhookAuthConfig struct {
+	AuthnConfig authenticatorfactory.DelegatingAuthenticatorConfig
+	AuthzConfig authorizerfactory.DelegatingAuthorizerConfig
+}
+
+// WebhookAuth creates an Auth suitable to use with kubelet webhook auth.
+// You must provide a CA provider to the authentication config, otherwise mTLS is disabled.
+func WebhookAuth(client kubernetes.Interface, nodeName string, opts ...WebhookAuthOption) (Auth, error) {
+	cfg := WebhookAuthConfig{
+		AuthnConfig: authenticatorfactory.DelegatingAuthenticatorConfig{
+			CacheTTL: 2 * time.Minute, // default taken from k8s.io/kubernetes/pkg/kubelet/apis/config/v1beta1
+			// TODO: After upgrading k8s libs, we need to add the retry backoff option
+		},
+		AuthzConfig: authorizerfactory.DelegatingAuthorizerConfig{
+			AllowCacheTTL: 5 * time.Minute,  // default taken from k8s.io/kubernetes/pkg/kubelet/apis/config/v1beta1
+			DenyCacheTTL:  30 * time.Second, // default taken from k8s.io/kubernetes/pkg/kubelet/apis/config/v1beta1
+			// TODO: After upgrading k8s libs, we need to add the retry backoff option
+		},
+	}
+
+	for _, o := range opts {
+		if err := o(&cfg); err != nil {
+			return nil, err
+		}
+	}
+
+	cfg.AuthnConfig.TokenAccessReviewClient = client.AuthenticationV1().TokenReviews()
+	cfg.AuthzConfig.SubjectAccessReviewClient = client.AuthorizationV1().SubjectAccessReviews()
+
+	authn, _, err := cfg.AuthnConfig.New()
+	if err != nil {
+		return nil, err
+	}
+
+	authz, err := cfg.AuthzConfig.New()
+	if err != nil {
+		return nil, err
+	}
+	return &authWrapper{
+		Request:                 authn,
+		RequestAttributesGetter: NodeRequestAttr{nodeName},
+		Authorizer:              authz,
+	}, nil
+}
+
+func (w *authWrapper) AuthenticateRequest(r *http.Request) (*authenticator.Response, bool, error) {
+	ctx, span := trace.StartSpan(r.Context(), "AuthenticateRequest")
+	defer span.End()
+	return w.Request.AuthenticateRequest(r.WithContext(ctx))
+}
+
+func (w *authWrapper) Authorize(ctx context.Context, a authorizer.Attributes) (authorizer.Decision, string, error) {
+	ctx, span := trace.StartSpan(ctx, "Authorize")
+	defer span.End()
+	return w.Authorizer.Authorize(ctx, a)
+}
+
+// NodeRequestAttr is a authorizor.RequeestAttributesGetter which can be used in the Auth interface.
+type NodeRequestAttr struct {
+	NodeName string
+}
+
+// GetRequestAttributes satisfies the authorizer.RequestAttributesGetter interface for use with an `Auth`.
+func (a NodeRequestAttr) GetRequestAttributes(u user.Info, r *http.Request) authorizer.Attributes {
+	return authorizer.AttributesRecord{
+		User:            u,
+		Verb:            getAPIVerb(r),
+		Namespace:       "",
+		APIGroup:        "",
+		APIVersion:      "v1",
+		Resource:        "nodes",
+		Name:            a.NodeName,
+		ResourceRequest: true,
+		Path:            r.URL.Path,
+		Subresource:     getSubresource(r),
+	}
+}
+
+func getAPIVerb(r *http.Request) string {
+	switch r.Method {
+	case http.MethodPost:
+		return "create"
+	case http.MethodGet:
+		return "get"
+	case http.MethodPut:
+		return "update"
+	case http.MethodPatch:
+		return "patch"
+	case http.MethodDelete:
+		return "delete"
+	}
+	return ""
+}
+
+func isSubpath(subpath, path string) bool {
+	// Taken from k8s.io/kubernetes/pkg/kubelet/server/auth.go
+	return subpath == path || (strings.HasPrefix(subpath, path) && subpath[len(path)] == '/')
+}
+
+func getSubresource(r *http.Request) string {
+	if isSubpath(r.URL.Path, "/stats") {
+		return "stats"
+	}
+	if isSubpath(r.URL.Path, "/metrics") {
+		return "metrics"
+	}
+	if isSubpath(r.URL.Path, "/logs") {
+		// yes, "log", not "logs"
+		// per kubelet code: "log" to match other log subresources (pods/log, etc)
+		return "log"
+	}
+
+	return "proxy"
+}

node/nodeutil/controller.go

@@ -8,19 +8,19 @@ import (
 	"net/http"
 	"os"
 	"path"
+	"runtime"
 	"time"
 
 	"github.com/pkg/errors"
 	"github.com/virtual-kubelet/virtual-kubelet/log"
 	"github.com/virtual-kubelet/virtual-kubelet/node"
-	"github.com/virtual-kubelet/virtual-kubelet/node/api"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/client-go/informers"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/kubernetes/scheme"
 	corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
+	corev1listers "k8s.io/client-go/listers/core/v1"
 	"k8s.io/client-go/tools/record"
 )
 
@@ -42,9 +42,11 @@ type Node struct {
 	scmInformerFactory informers.SharedInformerFactory
 	client             kubernetes.Interface
 
-	listenAddr  string
+	listenAddr string
-	httpHandler HTTPHandler
+	h          http.Handler
-	tlsConfig   *tls.Config
+	tlsConfig  *tls.Config
+
+	workers int
 
 	eb record.EventBroadcaster
 }
@@ -59,8 +61,35 @@ func (n *Node) PodController() *node.PodController {
 	return n.pc
 }
 
+func (n *Node) runHTTP(ctx context.Context) (func(), error) {
+	if n.tlsConfig == nil {
+		log.G(ctx).Warn("TLS config not provided, not starting up http service")
+		return func() {}, nil
+	}
+	if n.h == nil {
+		log.G(ctx).Debug("No http handler, not starting up http service")
+		return func() {}, nil
+	}
+
+	l, err := tls.Listen("tcp", n.listenAddr, n.tlsConfig)
+	if err != nil {
+		return nil, errors.Wrap(err, "error starting http listener")
+	}
+
+	log.G(ctx).Debug("Started TLS listener")
+
+	srv := &http.Server{Handler: n.h, TLSConfig: n.tlsConfig}
+	go srv.Serve(l) //nolint:errcheck
+	log.G(ctx).Debug("HTTP server running")
+
+	return func() {
+		srv.Close()
+		l.Close()
+	}, nil
+}
+
 // Run starts all the underlying controllers
-func (n *Node) Run(ctx context.Context, workers int) (retErr error) {
+func (n *Node) Run(ctx context.Context) (retErr error) {
 	ctx, cancel := context.WithCancel(ctx)
 	defer func() {
 		cancel()
@@ -69,31 +98,22 @@ func (n *Node) Run(ctx context.Context, workers int) (retErr error) {
 		close(n.done)
 	}()
 
-	if n.podInformerFactory != nil {
-		go n.podInformerFactory.Start(ctx.Done())
-	}
-	if n.scmInformerFactory != nil {
-		go n.scmInformerFactory.Start(ctx.Done())
-	}
-
 	if n.eb != nil {
 		n.eb.StartLogging(log.G(ctx).Infof)
 		n.eb.StartRecordingToSink(&corev1client.EventSinkImpl{Interface: n.client.CoreV1().Events(v1.NamespaceAll)})
+		defer n.eb.Shutdown()
+		log.G(ctx).Debug("Started event broadcaster")
 	}
 
-	l, err := tls.Listen("tcp", n.listenAddr, n.tlsConfig)
+	cancelHTTP, err := n.runHTTP(ctx)
 	if err != nil {
-		return errors.Wrap(err, "error starting http listener")
+		return err
 	}
-	log.G(ctx).Debug("Started TLS listener")
+	defer cancelHTTP()
-	defer l.Close()
 
-	srv := &http.Server{Handler: n.httpHandler, TLSConfig: n.tlsConfig}
+	go n.podInformerFactory.Start(ctx.Done())
-	go srv.Serve(l)
+	go n.scmInformerFactory.Start(ctx.Done())
-	defer srv.Close()
+	go n.pc.Run(ctx, n.workers) //nolint:errcheck
-
-	go n.pc.Run(ctx, workers) //nolint:errcheck
-	log.G(ctx).Debug("HTTP server running")
 
 	defer func() {
 		cancel()
@@ -110,7 +130,7 @@ func (n *Node) Run(ctx context.Context, workers int) (retErr error) {
 
 	log.G(ctx).Debug("pod controller ready")
 
-	go n.nc.Run(ctx) // nolint:errcheck
+	go n.nc.Run(ctx) //nolint:errcheck
 
 	defer func() {
 		cancel()
@@ -193,6 +213,9 @@ type NodeOpt func(c *NodeConfig) error
 // NodeConfig is used to hold configuration items for a Node.
 // It gets used in conjection with NodeOpt in NewNodeFromClient
 type NodeConfig struct {
+	// Set the client to use, otherwise a client will be created from ClientsetFromEnv
+	Client kubernetes.Interface
+
 	// Set the node spec to register with Kubernetes
 	NodeSpec v1.Node
 	// Set the path to read a kubeconfig from for creating a client.
@@ -206,8 +229,9 @@ type NodeConfig struct {
 	// Set a custom API handler to use.
 	// You can use this to setup, for example, authentication middleware.
 	// If one is not provided a default one will be created.
-	// Pod routes will be attached to this handler when creating the node
+	//
-	HTTPHandler HTTPHandler
+	// Note: If you provide your own handler, you'll need to handle all auth, routes, etc.
+	Handler http.Handler
 	// Set the timeout for idle http streams
 	StreamIdleTimeout time.Duration
 	// Set the timeout for creating http streams
@@ -220,11 +244,12 @@ type NodeConfig struct {
 	// Specify the event recorder to use
 	// If this is not provided, a default one will be used.
 	EventRecorder record.EventRecorder
-}
 
-type HTTPHandler interface {
+	// Set the number of workers to reconcile pods
-	api.ServeMux
+	// The default value is derived from the number of cores available.
-	http.Handler
+	NumWorkers int
+
+	routeAttacher func(Provider, NodeConfig, corev1listers.PodLister)
 }
 
 // WithNodeConfig returns a NodeOpt which replaces the NodeConfig with the passed in value.
@@ -235,9 +260,12 @@ func WithNodeConfig(c NodeConfig) NodeOpt {
 	}
 }
 
-// NewNode calls NewNodeFromClient with a nil client
+// WithClient return a NodeOpt that sets the client that will be used to create/manage the node.
-func NewNode(name string, newProvider NewProviderFunc, opts ...NodeOpt) (*Node, error) {
+func WithClient(c kubernetes.Interface) NodeOpt {
-	return NewNodeFromClient(nil, name, newProvider, opts...)
+	return func(cfg *NodeConfig) error {
+		cfg.Client = c
+		return nil
+	}
 }
 
 // NewNodeFromClient creates a new node using the provided client and name.
@@ -247,9 +275,11 @@ func NewNode(name string, newProvider NewProviderFunc, opts ...NodeOpt) (*Node,
 // Some basic values are set for node status, you'll almost certainly want to modify it.
 //
 // If client is nil, this will construct a client using ClientsetFromEnv
-func NewNodeFromClient(client kubernetes.Interface, name string, newProvider NewProviderFunc, opts ...NodeOpt) (*Node, error) {
+//
+// It is up to the caller to configure auth on the HTTP handler.
+func NewNodeFromClient(name string, newProvider NewProviderFunc, opts ...NodeOpt) (*Node, error) {
 	cfg := NodeConfig{
-		// TODO: this is what was set in the cli code... its not clear what a good value actually is.
+		NumWorkers:           runtime.NumCPU(),
 		InformerResyncPeriod: time.Minute,
 		KubeconfigPath:       os.Getenv("KUBECONFIG"),
 		HTTPListenAddr:       ":10250",
@@ -285,10 +315,7 @@ func NewNodeFromClient(client kubernetes.Interface, name string, newProvider New
 		return nil, errors.Wrap(err, "error parsing http listen address")
 	}
 
-	if cfg.HTTPHandler == nil {
+	client := cfg.Client
-		cfg.HTTPHandler = http.NewServeMux()
-	}
-
 	if client == nil {
 		var err error
 		client, err = ClientsetFromEnv(cfg.KubeconfigPath)
@@ -324,17 +351,9 @@ func NewNodeFromClient(client kubernetes.Interface, name string, newProvider New
 		return nil, errors.Wrap(err, "error creating provider")
 	}
 
-	api.AttachPodRoutes(api.PodHandlerConfig{
+	if cfg.routeAttacher != nil {
-		RunInContainer:   p.RunInContainer,
+		cfg.routeAttacher(p, cfg, podInformer.Lister())
-		GetContainerLogs: p.GetContainerLogs,
+	}
-		GetPods:          p.GetPods,
-		GetPodsFromKubernetes: func(context.Context) ([]*v1.Pod, error) {
-			return podInformer.Lister().List(labels.Everything())
-		},
-		GetStatsSummary:       p.GetStatsSummary,
-		StreamIdleTimeout:     cfg.StreamIdleTimeout,
-		StreamCreationTimeout: cfg.StreamCreationTimeout,
-	}, cfg.HTTPHandler, cfg.DebugHTTP)
 
 	var readyCb func(context.Context) error
 	if np == nil {
@@ -360,7 +379,7 @@ func NewNodeFromClient(client kubernetes.Interface, name string, newProvider New
 
 	var eb record.EventBroadcaster
 	if cfg.EventRecorder == nil {
-		eb := record.NewBroadcaster()
+		eb = record.NewBroadcaster()
 		cfg.EventRecorder = eb.NewRecorder(scheme.Scheme, v1.EventSource{Component: path.Join(name, "pod-controller")})
 	}
 
@@ -388,8 +407,9 @@ func NewNodeFromClient(client kubernetes.Interface, name string, newProvider New
 		scmInformerFactory: scmInformerFactory,
 		client:             client,
 		tlsConfig:          cfg.TLSConfig,
-		httpHandler:        cfg.HTTPHandler,
+		h:                  cfg.Handler,
 		listenAddr:         cfg.HTTPListenAddr,
+		workers:            cfg.NumWorkers,
 	}, nil
 }
 

node/nodeutil/provider.go

@@ -8,6 +8,7 @@ import (
 	"github.com/virtual-kubelet/virtual-kubelet/node/api"
 	"github.com/virtual-kubelet/virtual-kubelet/node/api/statsv1alpha1"
 	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/labels"
 	corev1listers "k8s.io/client-go/listers/core/v1"
 )
 
@@ -45,3 +46,25 @@ type ProviderConfig struct {
 // NewProviderFunc is used from NewNodeFromClient to bootstrap a provider using the client/listers/etc created there.
 // If a nil node provider is returned a default one will be used.
 type NewProviderFunc func(ProviderConfig) (Provider, node.NodeProvider, error)
+
+// AttachProviderRoutes returns a NodeOpt which uses api.PodHandler to attach the routes to the provider functions.
+//
+// Note this only attaches routes, you'll need to ensure to set the handler in the node config.
+func AttachProviderRoutes(mux api.ServeMux) NodeOpt {
+	return func(cfg *NodeConfig) error {
+		cfg.routeAttacher = func(p Provider, cfg NodeConfig, pods corev1listers.PodLister) {
+			mux.Handle("/", api.PodHandler(api.PodHandlerConfig{
+				RunInContainer:   p.RunInContainer,
+				GetContainerLogs: p.GetContainerLogs,
+				GetPods:          p.GetPods,
+				GetPodsFromKubernetes: func(context.Context) ([]*v1.Pod, error) {
+					return pods.List(labels.Everything())
+				},
+				GetStatsSummary:       p.GetStatsSummary,
+				StreamIdleTimeout:     cfg.StreamIdleTimeout,
+				StreamCreationTimeout: cfg.StreamCreationTimeout,
+			}, true))
+		}
+		return nil
+	}
+}

node/nodeutil/tls.go

@@ -0,0 +1,83 @@
+package nodeutil
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"io/ioutil"
+)
+
+// WithTLSConfig returns a NodeOpt which creates a base TLSConfig with the default cipher suites and tls min verions.
+// The tls config can be modified through functional options.
+func WithTLSConfig(opts ...func(*tls.Config) error) NodeOpt {
+	return func(cfg *NodeConfig) error {
+		tlsCfg := &tls.Config{
+			MinVersion:               tls.VersionTLS12,
+			PreferServerCipherSuites: true,
+			CipherSuites:             DefaultServerCiphers(),
+			ClientAuth:               tls.RequestClientCert,
+		}
+		for _, o := range opts {
+			if err := o(tlsCfg); err != nil {
+				return err
+			}
+		}
+
+		cfg.TLSConfig = tlsCfg
+		return nil
+	}
+}
+
+// WithCAFromPath makes a TLS config option to set up client auth using the path to a PEM encoded CA cert.
+func WithCAFromPath(p string) func(*tls.Config) error {
+	return func(cfg *tls.Config) error {
+		pem, err := ioutil.ReadFile(p)
+		if err != nil {
+			return fmt.Errorf("error reading ca cert pem: %w", err)
+		}
+		cfg.ClientAuth = tls.RequireAndVerifyClientCert
+		return WithCACert(pem)(cfg)
+	}
+}
+
+// WithKeyPairFromPath make sa TLS config option which loads the key pair paths from disk and appends them to the tls config.
+func WithKeyPairFromPath(cert, key string) func(*tls.Config) error {
+	return func(cfg *tls.Config) error {
+		cert, err := tls.LoadX509KeyPair(cert, key)
+		if err != nil {
+			return err
+		}
+		cfg.Certificates = append(cfg.Certificates, cert)
+		return nil
+	}
+}
+
+// WithCACert makes a TLS config opotion which appends the provided PEM encoded bytes the tls config's cert pool.
+// If a cert pool is not defined on the tls config an empty one will be created.
+func WithCACert(pem []byte) func(*tls.Config) error {
+	return func(cfg *tls.Config) error {
+		if cfg.ClientCAs == nil {
+			cfg.ClientCAs = x509.NewCertPool()
+		}
+		if !cfg.ClientCAs.AppendCertsFromPEM(pem) {
+			return fmt.Errorf("could not parse ca cert pem")
+		}
+		return nil
+	}
+}
+
+// DefaultServerCiphers is the list of accepted TLS ciphers, with known weak ciphers elided
+// Note this list should be a moving target.
+func DefaultServerCiphers() []uint16 {
+	return []uint16{
+		tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
+		tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
+		tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
+		tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
+
+		tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
+		tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
+		tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
+		tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
+	}
+}