Add nodeutil opt to bootstrapping from rest.Config

This uses a rest.Config to bootstrap TLS for the http server, webhook
auth, and the client.

This can be expanded later to do other kinds of TLS bootstrapping. For
now this seems to get the job done in terms of what VK expects for
permissions on the cluster.

Signed-off-by: Brian Goff <cpuguy83@gmail.com>

errdefs/auth.go

@@ -0,0 +1,33 @@
+package errdefs
+
+import (
+	"errors"
+	"fmt"
+)
+
+var (
+	// ErrForbidden is returned when the user is not authorized to perform the operation.
+	ErrForbidden = errors.New("forbidden")
+	// ErrUnauthorized is returned when the user is not authenticated.
+	ErrUnauthorized = errors.New("unauthorized")
+)
+
+// Unauthorized wraps ErrUnauthorized with a message.
+func Unauthorized(msg string) error {
+	return fmt.Errorf("%w: %s", ErrUnauthorized, msg)
+}
+
+// Forbidden wraps ErrForbidden with a message.
+func Forbidden(msg string) error {
+	return fmt.Errorf("%w: %s", ErrForbidden, msg)
+}
+
+// IsForbidden returns true if the error has ErrForbidden in the error chain.
+func IsForbidden(err error) bool {
+	return errors.Is(err, ErrForbidden)
+}
+
+// IsUnauthorized returns true if the error has ErrUnauthorized in the error chain.
+func IsUnauthorized(err error) bool {
+	return errors.Is(err, ErrUnauthorized)
+}

node/nodeutil/auth.go

@@ -6,6 +6,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/virtual-kubelet/virtual-kubelet/errdefs"
 	"github.com/virtual-kubelet/virtual-kubelet/log"
 	"github.com/virtual-kubelet/virtual-kubelet/trace"
 	"k8s.io/apiserver/pkg/authentication/authenticator"
@@ -69,14 +70,24 @@ func handleAuth(auth Auth, w http.ResponseWriter, r *http.Request, next http.Han
 	defer span.End()
 	r = r.WithContext(ctx)
 
+	logger := log.G(r.Context())
 	info, ok, err := auth.AuthenticateRequest(r)
-	if err != nil || !ok {
-		log.G(r.Context()).WithError(err).Error("Authorization error")
+	if err != nil {
+		logger.WithError(err).Error("Error authenticating request")
 		http.Error(w, "Unauthorized", http.StatusUnauthorized)
+		span.SetStatus(err)
 		return
 	}
 
-	logger := log.G(ctx).WithFields(log.Fields{
+	if !ok {
+		logger.Error("Request not authenticated")
+		log.G(r.Context()).Infof("Unauthorized: RequestURI: %s", r.RequestURI)
+		http.Error(w, "Unauthorized", http.StatusUnauthorized)
+		span.SetStatus(errdefs.ErrUnauthorized)
+		return
+	}
+
+	logger = logger.WithFields(log.Fields{
 		"user-name": info.User.GetName(),
 		"user-id":   info.User.GetUID(),
 	})
@@ -90,11 +101,13 @@ func handleAuth(auth Auth, w http.ResponseWriter, r *http.Request, next http.Han
 	if err != nil {
 		log.G(r.Context()).WithError(err).Error("Authorization error")
 		http.Error(w, err.Error(), http.StatusInternalServerError)
+		span.SetStatus(err)
 		return
 	}
 
 	if decision != authorizer.DecisionAllow {
 		http.Error(w, "Forbidden", http.StatusForbidden)
+		span.SetStatus(errdefs.ErrForbidden)
 		return
 	}
 

node/nodeutil/client.go

@@ -24,7 +24,7 @@ func ClientsetFromEnv(kubeConfigPath string) (*kubernetes.Clientset, error) {
 	)
 
 	if kubeConfigPath != "" {
-		config, err = clientsetFromEnvKubeConfigPath(kubeConfigPath)
+		config, err = RestConfigFromEnv(kubeConfigPath)
 	} else {
 		config, err = rest.InClusterConfig()
 	}
@@ -36,7 +36,8 @@ func ClientsetFromEnv(kubeConfigPath string) (*kubernetes.Clientset, error) {
 	return kubernetes.NewForConfig(config)
 }
 
-func clientsetFromEnvKubeConfigPath(kubeConfigPath string) (*rest.Config, error) {
+// RestConfigFromEnv is like ClientsetFromEnv except it returns a rest config instead of a full client.
+func RestConfigFromEnv(kubeConfigPath string) (*rest.Config, error) {
 	_, err := os.Stat(kubeConfigPath)
 	if os.IsNotExist(err) {
 		return rest.InClusterConfig()

node/nodeutil/controller.go

@@ -48,7 +48,8 @@ type Node struct {
 
 	workers int
 
-	eb record.EventBroadcaster
+	eb           record.EventBroadcaster
+	caController caController
 }
 
 // NodeController returns the configured node controller.
@@ -107,6 +108,10 @@ func (n *Node) Run(ctx context.Context) (retErr error) {
 		log.G(ctx).Debug("Started event broadcaster")
 	}
 
+	if n.caController != nil {
+		go n.caController.Run(1, ctx.Done())
+	}
+
 	cancelHTTP, err := n.runHTTP(ctx)
 	if err != nil {
 		return err
@@ -212,6 +217,8 @@ func (n *Node) Err() error {
 // NodeOpt is used as functional options when configuring a new node in NewNodeFromClient
 type NodeOpt func(c *NodeConfig) error
 
+type caController interface{ Run(int, <-chan struct{}) }
+
 // NodeConfig is used to hold configuration items for a Node.
 // It gets used in conjection with NodeOpt in NewNodeFromClient
 type NodeConfig struct {
@@ -260,22 +267,8 @@ type NodeConfig struct {
 	SkipDownwardAPIResolution bool
 
 	routeAttacher func(Provider, NodeConfig, corev1listers.PodLister)
-}
 
-// WithNodeConfig returns a NodeOpt which replaces the NodeConfig with the passed in value.
-func WithNodeConfig(c NodeConfig) NodeOpt {
-	return func(orig *NodeConfig) error {
-		*orig = c
-		return nil
-	}
-}
-
-// WithClient return a NodeOpt that sets the client that will be used to create/manage the node.
-func WithClient(c kubernetes.Interface) NodeOpt {
-	return func(cfg *NodeConfig) error {
-		cfg.Client = c
-		return nil
-	}
+	caController caController
 }
 
 // NewNode creates a new node using the provided client and name.
@@ -326,10 +319,6 @@ func NewNode(name string, newProvider NewProviderFunc, opts ...NodeOpt) (*Node,
 		return nil, errors.Wrap(err, "error parsing http listen address")
 	}
 
-	if cfg.Client == nil {
-		return nil, errors.New("no client provided")
-	}
-
 	podInformerFactory := informers.NewSharedInformerFactoryWithOptions(
 		cfg.Client,
 		cfg.InformerResyncPeriod,
@@ -425,6 +414,7 @@ func NewNode(name string, newProvider NewProviderFunc, opts ...NodeOpt) (*Node,
 		h:                  cfg.Handler,
 		listenAddr:         cfg.HTTPListenAddr,
 		workers:            cfg.NumWorkers,
+		caController:       cfg.caController,
 	}, nil
 }
 

node/nodeutil/opts.go

@@ -0,0 +1,150 @@
+package nodeutil
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/virtual-kubelet/virtual-kubelet/errdefs"
+	"github.com/virtual-kubelet/virtual-kubelet/node/api"
+	"k8s.io/apiserver/pkg/server/dynamiccertificates"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+)
+
+// WithNodeConfig returns a NodeOpt which replaces the NodeConfig with the passed in value.
+func WithNodeConfig(c NodeConfig) NodeOpt {
+	return func(orig *NodeConfig) error {
+		*orig = c
+		return nil
+	}
+}
+
+// WithClient return a NodeOpt that sets the client that will be used to create/manage the node.
+func WithClient(c kubernetes.Interface) NodeOpt {
+	return func(cfg *NodeConfig) error {
+		cfg.Client = c
+		return nil
+	}
+}
+
+// BootstrapConfig is the configuration for bootstrapping a node.
+type BootstrapConfig struct {
+	// ClientCAConfigMapSpec is the config map information for getting the CA for authenticating clients
+	// If not set, the CA in the rest config will be used.
+	ClientCAConfigMapSpec *ConfigMapCASpec
+	// A set of options to pass when creating the webhook auth.
+	WebhookAuthOpts []WebhookAuthOption
+	// RestConfig is the rest config to use for the client
+	// If it is not provided a default one from the environment will be created.
+	RestConfig *rest.Config
+}
+
+// ConfigMapCASpec is the spec for a config map that contains a CA.
+type ConfigMapCASpec struct {
+	Namespace string
+	Name      string
+	Key       string
+}
+
+// BootstrapOpt is a functional option used to configure node bootstrapping
+type BootstrapOpt func(*BootstrapConfig) error
+
+// WithBootstrapFromRestConfig takes a reset config (or a default one from the environment) and returns a NodeOpt that will:
+// 1. Create a client from the rest config (if no client is set)
+// 2. Create webhook authn/authz from the rest config
+// 3. Configure the TLS config for the HTTP server from the certs in the rest config.
+func WithBootstrapFromRestConfig(opts ...BootstrapOpt) NodeOpt {
+	return func(cfg *NodeConfig) error {
+		var bOpts BootstrapConfig
+		if rCfg, err := RestConfigFromEnv(cfg.KubeconfigPath); err == nil {
+			bOpts.RestConfig = rCfg
+		}
+
+		for _, o := range opts {
+			if err := o(&bOpts); err != nil {
+				return err
+			}
+		}
+
+		if bOpts.RestConfig == nil {
+			return fmt.Errorf("no rest config provided")
+		}
+
+		if cfg.Client == nil {
+			client, err := kubernetes.NewForConfig(bOpts.RestConfig)
+			if err != nil {
+				return err
+			}
+			cfg.Client = client
+		}
+
+		if err := WithTLSConfig(tlsFromRestConfig(bOpts.RestConfig))(cfg); err != nil {
+			return err
+		}
+
+		if err := configureWebhookCA(cfg, &bOpts); err != nil {
+			return fmt.Errorf("error configure webhook auth: %w", err)
+		}
+
+		var mux *http.ServeMux
+		if cfg.Handler == nil {
+			mux = http.NewServeMux()
+			cfg.Handler = mux
+		}
+		if cfg.routeAttacher == nil && mux != nil {
+			if err := AttachProviderRoutes(mux)(cfg); err != nil {
+				return err
+			}
+		}
+
+		auth, err := WebhookAuth(cfg.Client, cfg.NodeSpec.Name, bOpts.WebhookAuthOpts...)
+		if err != nil {
+			return err
+		}
+
+		cfg.Handler = api.InstrumentHandler(WithAuth(auth, cfg.Handler))
+		return nil
+	}
+}
+
+func configureWebhookCA(cfg *NodeConfig, bCfg *BootstrapConfig) error {
+	if bCfg.ClientCAConfigMapSpec != nil {
+		bCfg.WebhookAuthOpts = append(bCfg.WebhookAuthOpts, func(auth *WebhookAuthConfig) error {
+			cmCA, err := dynamiccertificates.NewDynamicCAFromConfigMapController("client-ca", bCfg.ClientCAConfigMapSpec.Namespace, bCfg.ClientCAConfigMapSpec.Name, bCfg.ClientCAConfigMapSpec.Key, cfg.Client)
+			if err != nil {
+				return fmt.Errorf("error loading dynamic CA from config map: %w", err)
+			}
+			auth.AuthnConfig.ClientCertificateCAContentProvider = cmCA
+			cfg.caController = cmCA
+			return nil
+		})
+		return nil
+	}
+
+	if bCfg.RestConfig.CAFile != "" {
+		bCfg.WebhookAuthOpts = append(bCfg.WebhookAuthOpts, func(auth *WebhookAuthConfig) error {
+			caFile, err := dynamiccertificates.NewDynamicCAContentFromFile("ca-file", bCfg.RestConfig.CAFile)
+			if err != nil {
+				return fmt.Errorf("error loading dynamic CA file from rest config: %w", err)
+			}
+			auth.AuthnConfig.ClientCertificateCAContentProvider = caFile
+			cfg.caController = caFile
+			return nil
+		})
+		return nil
+	}
+
+	if bCfg.RestConfig.CAData != nil {
+		bCfg.WebhookAuthOpts = append(bCfg.WebhookAuthOpts, func(auth *WebhookAuthConfig) error {
+			caData, err := dynamiccertificates.NewStaticCAContent("ca-data", bCfg.RestConfig.CAData)
+			if err != nil {
+				return fmt.Errorf("error loading static ca from rest config: %w", err)
+			}
+			auth.AuthnConfig.ClientCertificateCAContentProvider = caData
+			return nil
+		})
+		return nil
+	}
+
+	return errdefs.InvalidInput("no client CA found")
+}

node/nodeutil/tls.go

@@ -5,25 +5,28 @@ import (
 	"crypto/x509"
 	"fmt"
 	"os"
+
+	"k8s.io/client-go/rest"
 )
 
 // WithTLSConfig returns a NodeOpt which creates a base TLSConfig with the default cipher suites and tls min verions.
 // The tls config can be modified through functional options.
 func WithTLSConfig(opts ...func(*tls.Config) error) NodeOpt {
 	return func(cfg *NodeConfig) error {
-		tlsCfg := &tls.Config{
-			MinVersion:               tls.VersionTLS12,
-			PreferServerCipherSuites: true,
-			CipherSuites:             DefaultServerCiphers(),
-			ClientAuth:               tls.RequestClientCert,
+		if cfg.TLSConfig == nil {
+			cfg.TLSConfig = &tls.Config{
+				MinVersion:               tls.VersionTLS12,
+				PreferServerCipherSuites: true,
+				CipherSuites:             DefaultServerCiphers(),
+				ClientAuth:               tls.RequestClientCert,
+			}
 		}
 		for _, o := range opts {
-			if err := o(tlsCfg); err != nil {
+			if err := o(cfg.TLSConfig); err != nil {
 				return err
 			}
 		}
 
-		cfg.TLSConfig = tlsCfg
 		return nil
 	}
 }
@@ -45,7 +48,7 @@ func WithKeyPairFromPath(cert, key string) func(*tls.Config) error {
 	return func(cfg *tls.Config) error {
 		cert, err := tls.LoadX509KeyPair(cert, key)
 		if err != nil {
-			return err
+			return fmt.Errorf("error loading x509 key pair: %w", err)
 		}
 		cfg.Certificates = append(cfg.Certificates, cert)
 		return nil
@@ -56,7 +59,7 @@ func WithKeyPairFromPath(cert, key string) func(*tls.Config) error {
 // If a cert pool is not defined on the tls config an empty one will be created.
 func WithCACert(pem []byte) func(*tls.Config) error {
 	return func(cfg *tls.Config) error {
-		if cfg.ClientCAs == nil {
+		if cfg.RootCAs == nil {
 			cfg.ClientCAs = x509.NewCertPool()
 		}
 		if !cfg.ClientCAs.AppendCertsFromPEM(pem) {
@@ -81,3 +84,48 @@ func DefaultServerCiphers() []uint16 {
 		tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
 	}
 }
+
+func tlsFromRestConfig(r *rest.Config) func(*tls.Config) error {
+	return func(cfg *tls.Config) error {
+		var err error
+
+		cfg.ClientAuth = tls.RequestClientCert
+
+		certData := r.CertData
+		if certData == nil && r.CertFile != "" {
+			certData, err = os.ReadFile(r.CertFile)
+			if err != nil {
+				return fmt.Errorf("error reading cert file from clientset: %w", err)
+			}
+		}
+
+		keyData := r.KeyData
+		if keyData == nil && r.KeyFile != "" {
+			keyData, err = os.ReadFile(r.KeyFile)
+			if err != nil {
+				return fmt.Errorf("error reading key file from clientset: %w", err)
+			}
+		}
+		if keyData != nil && certData != nil {
+			pem, err := tls.X509KeyPair(certData, keyData)
+			if err != nil {
+				return fmt.Errorf("error creating key pair from clientset: %w", err)
+			}
+			cfg.Certificates = append(cfg.Certificates, pem)
+			cfg.ClientAuth = tls.RequestClientCert
+		}
+
+		caData := r.CAData
+		if certData == nil && r.CAFile != "" {
+			caData, err = os.ReadFile(r.CAFile)
+			if err != nil {
+				return fmt.Errorf("error reading ca file from clientset: %w", err)
+			}
+		}
+		if caData != nil {
+			return WithCACert(caData)(cfg)
+		}
+
+		return nil
+	}
+}

trace/opencensus/opencensus.go

@@ -79,8 +79,12 @@ func (s *span) SetStatus(err error) {
 		status.Code = octrace.StatusCodeNotFound
 	case errdefs.IsInvalidInput(err):
 		status.Code = octrace.StatusCodeInvalidArgument
-		// TODO: other error types
+	case errdefs.IsForbidden(err):
+		status.Code = octrace.StatusCodePermissionDenied
+	case errdefs.IsUnauthorized(err):
+		status.Code = octrace.StatusCodeUnauthenticated
 	default:
+		// TODO: other error types
 		status.Code = octrace.StatusCodeUnknown
 	}