diff --git a/errdefs/auth.go b/errdefs/auth.go new file mode 100644 index 000000000..833e7cf63 --- /dev/null +++ b/errdefs/auth.go @@ -0,0 +1,33 @@ +package errdefs + +import ( + "errors" + "fmt" +) + +var ( + // ErrForbidden is returned when the user is not authorized to perform the operation. + ErrForbidden = errors.New("forbidden") + // ErrUnauthorized is returned when the user is not authenticated. + ErrUnauthorized = errors.New("unauthorized") +) + +// Unauthorized wraps ErrUnauthorized with a message. +func Unauthorized(msg string) error { + return fmt.Errorf("%w: %s", ErrUnauthorized, msg) +} + +// Forbidden wraps ErrForbidden with a message. +func Forbidden(msg string) error { + return fmt.Errorf("%w: %s", ErrForbidden, msg) +} + +// IsForbidden returns true if the error has ErrForbidden in the error chain. +func IsForbidden(err error) bool { + return errors.Is(err, ErrForbidden) +} + +// IsUnauthorized returns true if the error has ErrUnauthorized in the error chain. +func IsUnauthorized(err error) bool { + return errors.Is(err, ErrUnauthorized) +} diff --git a/node/nodeutil/auth.go b/node/nodeutil/auth.go index 552046fac..78ef12409 100644 --- a/node/nodeutil/auth.go +++ b/node/nodeutil/auth.go @@ -6,6 +6,7 @@ import ( "strings" "time" + "github.com/virtual-kubelet/virtual-kubelet/errdefs" "github.com/virtual-kubelet/virtual-kubelet/log" "github.com/virtual-kubelet/virtual-kubelet/trace" "k8s.io/apiserver/pkg/authentication/authenticator" @@ -69,14 +70,24 @@ func handleAuth(auth Auth, w http.ResponseWriter, r *http.Request, next http.Han defer span.End() r = r.WithContext(ctx) + logger := log.G(r.Context()) info, ok, err := auth.AuthenticateRequest(r) - if err != nil || !ok { - log.G(r.Context()).WithError(err).Error("Authorization error") + if err != nil { + logger.WithError(err).Error("Error authenticating request") http.Error(w, "Unauthorized", http.StatusUnauthorized) + span.SetStatus(err) return } - logger := log.G(ctx).WithFields(log.Fields{ + if !ok { + logger.Error("Request not authenticated") + log.G(r.Context()).Infof("Unauthorized: RequestURI: %s", r.RequestURI) + http.Error(w, "Unauthorized", http.StatusUnauthorized) + span.SetStatus(errdefs.ErrUnauthorized) + return + } + + logger = logger.WithFields(log.Fields{ "user-name": info.User.GetName(), "user-id": info.User.GetUID(), }) @@ -90,11 +101,13 @@ func handleAuth(auth Auth, w http.ResponseWriter, r *http.Request, next http.Han if err != nil { log.G(r.Context()).WithError(err).Error("Authorization error") http.Error(w, err.Error(), http.StatusInternalServerError) + span.SetStatus(err) return } if decision != authorizer.DecisionAllow { http.Error(w, "Forbidden", http.StatusForbidden) + span.SetStatus(errdefs.ErrForbidden) return } diff --git a/node/nodeutil/client.go b/node/nodeutil/client.go index 4274a4a4b..7aa17e1f5 100644 --- a/node/nodeutil/client.go +++ b/node/nodeutil/client.go @@ -24,7 +24,7 @@ func ClientsetFromEnv(kubeConfigPath string) (*kubernetes.Clientset, error) { ) if kubeConfigPath != "" { - config, err = clientsetFromEnvKubeConfigPath(kubeConfigPath) + config, err = RestConfigFromEnv(kubeConfigPath) } else { config, err = rest.InClusterConfig() } @@ -36,7 +36,8 @@ func ClientsetFromEnv(kubeConfigPath string) (*kubernetes.Clientset, error) { return kubernetes.NewForConfig(config) } -func clientsetFromEnvKubeConfigPath(kubeConfigPath string) (*rest.Config, error) { +// RestConfigFromEnv is like ClientsetFromEnv except it returns a rest config instead of a full client. +func RestConfigFromEnv(kubeConfigPath string) (*rest.Config, error) { _, err := os.Stat(kubeConfigPath) if os.IsNotExist(err) { return rest.InClusterConfig() diff --git a/node/nodeutil/controller.go b/node/nodeutil/controller.go index 96d1a8aa5..6dcd67a20 100644 --- a/node/nodeutil/controller.go +++ b/node/nodeutil/controller.go @@ -48,7 +48,8 @@ type Node struct { workers int - eb record.EventBroadcaster + eb record.EventBroadcaster + caController caController } // NodeController returns the configured node controller. @@ -107,6 +108,10 @@ func (n *Node) Run(ctx context.Context) (retErr error) { log.G(ctx).Debug("Started event broadcaster") } + if n.caController != nil { + go n.caController.Run(1, ctx.Done()) + } + cancelHTTP, err := n.runHTTP(ctx) if err != nil { return err @@ -212,6 +217,8 @@ func (n *Node) Err() error { // NodeOpt is used as functional options when configuring a new node in NewNodeFromClient type NodeOpt func(c *NodeConfig) error +type caController interface{ Run(int, <-chan struct{}) } + // NodeConfig is used to hold configuration items for a Node. // It gets used in conjection with NodeOpt in NewNodeFromClient type NodeConfig struct { @@ -260,22 +267,8 @@ type NodeConfig struct { SkipDownwardAPIResolution bool routeAttacher func(Provider, NodeConfig, corev1listers.PodLister) -} -// WithNodeConfig returns a NodeOpt which replaces the NodeConfig with the passed in value. -func WithNodeConfig(c NodeConfig) NodeOpt { - return func(orig *NodeConfig) error { - *orig = c - return nil - } -} - -// WithClient return a NodeOpt that sets the client that will be used to create/manage the node. -func WithClient(c kubernetes.Interface) NodeOpt { - return func(cfg *NodeConfig) error { - cfg.Client = c - return nil - } + caController caController } // NewNode creates a new node using the provided client and name. @@ -326,10 +319,6 @@ func NewNode(name string, newProvider NewProviderFunc, opts ...NodeOpt) (*Node, return nil, errors.Wrap(err, "error parsing http listen address") } - if cfg.Client == nil { - return nil, errors.New("no client provided") - } - podInformerFactory := informers.NewSharedInformerFactoryWithOptions( cfg.Client, cfg.InformerResyncPeriod, @@ -425,6 +414,7 @@ func NewNode(name string, newProvider NewProviderFunc, opts ...NodeOpt) (*Node, h: cfg.Handler, listenAddr: cfg.HTTPListenAddr, workers: cfg.NumWorkers, + caController: cfg.caController, }, nil } diff --git a/node/nodeutil/opts.go b/node/nodeutil/opts.go new file mode 100644 index 000000000..e9732b851 --- /dev/null +++ b/node/nodeutil/opts.go @@ -0,0 +1,150 @@ +package nodeutil + +import ( + "fmt" + "net/http" + + "github.com/virtual-kubelet/virtual-kubelet/errdefs" + "github.com/virtual-kubelet/virtual-kubelet/node/api" + "k8s.io/apiserver/pkg/server/dynamiccertificates" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" +) + +// WithNodeConfig returns a NodeOpt which replaces the NodeConfig with the passed in value. +func WithNodeConfig(c NodeConfig) NodeOpt { + return func(orig *NodeConfig) error { + *orig = c + return nil + } +} + +// WithClient return a NodeOpt that sets the client that will be used to create/manage the node. +func WithClient(c kubernetes.Interface) NodeOpt { + return func(cfg *NodeConfig) error { + cfg.Client = c + return nil + } +} + +// BootstrapConfig is the configuration for bootstrapping a node. +type BootstrapConfig struct { + // ClientCAConfigMapSpec is the config map information for getting the CA for authenticating clients + // If not set, the CA in the rest config will be used. + ClientCAConfigMapSpec *ConfigMapCASpec + // A set of options to pass when creating the webhook auth. + WebhookAuthOpts []WebhookAuthOption + // RestConfig is the rest config to use for the client + // If it is not provided a default one from the environment will be created. + RestConfig *rest.Config +} + +// ConfigMapCASpec is the spec for a config map that contains a CA. +type ConfigMapCASpec struct { + Namespace string + Name string + Key string +} + +// BootstrapOpt is a functional option used to configure node bootstrapping +type BootstrapOpt func(*BootstrapConfig) error + +// WithBootstrapFromRestConfig takes a reset config (or a default one from the environment) and returns a NodeOpt that will: +// 1. Create a client from the rest config (if no client is set) +// 2. Create webhook authn/authz from the rest config +// 3. Configure the TLS config for the HTTP server from the certs in the rest config. +func WithBootstrapFromRestConfig(opts ...BootstrapOpt) NodeOpt { + return func(cfg *NodeConfig) error { + var bOpts BootstrapConfig + if rCfg, err := RestConfigFromEnv(cfg.KubeconfigPath); err == nil { + bOpts.RestConfig = rCfg + } + + for _, o := range opts { + if err := o(&bOpts); err != nil { + return err + } + } + + if bOpts.RestConfig == nil { + return fmt.Errorf("no rest config provided") + } + + if cfg.Client == nil { + client, err := kubernetes.NewForConfig(bOpts.RestConfig) + if err != nil { + return err + } + cfg.Client = client + } + + if err := WithTLSConfig(tlsFromRestConfig(bOpts.RestConfig))(cfg); err != nil { + return err + } + + if err := configureWebhookCA(cfg, &bOpts); err != nil { + return fmt.Errorf("error configure webhook auth: %w", err) + } + + var mux *http.ServeMux + if cfg.Handler == nil { + mux = http.NewServeMux() + cfg.Handler = mux + } + if cfg.routeAttacher == nil && mux != nil { + if err := AttachProviderRoutes(mux)(cfg); err != nil { + return err + } + } + + auth, err := WebhookAuth(cfg.Client, cfg.NodeSpec.Name, bOpts.WebhookAuthOpts...) + if err != nil { + return err + } + + cfg.Handler = api.InstrumentHandler(WithAuth(auth, cfg.Handler)) + return nil + } +} + +func configureWebhookCA(cfg *NodeConfig, bCfg *BootstrapConfig) error { + if bCfg.ClientCAConfigMapSpec != nil { + bCfg.WebhookAuthOpts = append(bCfg.WebhookAuthOpts, func(auth *WebhookAuthConfig) error { + cmCA, err := dynamiccertificates.NewDynamicCAFromConfigMapController("client-ca", bCfg.ClientCAConfigMapSpec.Namespace, bCfg.ClientCAConfigMapSpec.Name, bCfg.ClientCAConfigMapSpec.Key, cfg.Client) + if err != nil { + return fmt.Errorf("error loading dynamic CA from config map: %w", err) + } + auth.AuthnConfig.ClientCertificateCAContentProvider = cmCA + cfg.caController = cmCA + return nil + }) + return nil + } + + if bCfg.RestConfig.CAFile != "" { + bCfg.WebhookAuthOpts = append(bCfg.WebhookAuthOpts, func(auth *WebhookAuthConfig) error { + caFile, err := dynamiccertificates.NewDynamicCAContentFromFile("ca-file", bCfg.RestConfig.CAFile) + if err != nil { + return fmt.Errorf("error loading dynamic CA file from rest config: %w", err) + } + auth.AuthnConfig.ClientCertificateCAContentProvider = caFile + cfg.caController = caFile + return nil + }) + return nil + } + + if bCfg.RestConfig.CAData != nil { + bCfg.WebhookAuthOpts = append(bCfg.WebhookAuthOpts, func(auth *WebhookAuthConfig) error { + caData, err := dynamiccertificates.NewStaticCAContent("ca-data", bCfg.RestConfig.CAData) + if err != nil { + return fmt.Errorf("error loading static ca from rest config: %w", err) + } + auth.AuthnConfig.ClientCertificateCAContentProvider = caData + return nil + }) + return nil + } + + return errdefs.InvalidInput("no client CA found") +} diff --git a/node/nodeutil/tls.go b/node/nodeutil/tls.go index 7c6f3ed45..3a72f5d10 100644 --- a/node/nodeutil/tls.go +++ b/node/nodeutil/tls.go @@ -5,25 +5,28 @@ import ( "crypto/x509" "fmt" "os" + + "k8s.io/client-go/rest" ) // WithTLSConfig returns a NodeOpt which creates a base TLSConfig with the default cipher suites and tls min verions. // The tls config can be modified through functional options. func WithTLSConfig(opts ...func(*tls.Config) error) NodeOpt { return func(cfg *NodeConfig) error { - tlsCfg := &tls.Config{ - MinVersion: tls.VersionTLS12, - PreferServerCipherSuites: true, - CipherSuites: DefaultServerCiphers(), - ClientAuth: tls.RequestClientCert, + if cfg.TLSConfig == nil { + cfg.TLSConfig = &tls.Config{ + MinVersion: tls.VersionTLS12, + PreferServerCipherSuites: true, + CipherSuites: DefaultServerCiphers(), + ClientAuth: tls.RequestClientCert, + } } for _, o := range opts { - if err := o(tlsCfg); err != nil { + if err := o(cfg.TLSConfig); err != nil { return err } } - cfg.TLSConfig = tlsCfg return nil } } @@ -45,7 +48,7 @@ func WithKeyPairFromPath(cert, key string) func(*tls.Config) error { return func(cfg *tls.Config) error { cert, err := tls.LoadX509KeyPair(cert, key) if err != nil { - return err + return fmt.Errorf("error loading x509 key pair: %w", err) } cfg.Certificates = append(cfg.Certificates, cert) return nil @@ -56,7 +59,7 @@ func WithKeyPairFromPath(cert, key string) func(*tls.Config) error { // If a cert pool is not defined on the tls config an empty one will be created. func WithCACert(pem []byte) func(*tls.Config) error { return func(cfg *tls.Config) error { - if cfg.ClientCAs == nil { + if cfg.RootCAs == nil { cfg.ClientCAs = x509.NewCertPool() } if !cfg.ClientCAs.AppendCertsFromPEM(pem) { @@ -81,3 +84,48 @@ func DefaultServerCiphers() []uint16 { tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, } } + +func tlsFromRestConfig(r *rest.Config) func(*tls.Config) error { + return func(cfg *tls.Config) error { + var err error + + cfg.ClientAuth = tls.RequestClientCert + + certData := r.CertData + if certData == nil && r.CertFile != "" { + certData, err = os.ReadFile(r.CertFile) + if err != nil { + return fmt.Errorf("error reading cert file from clientset: %w", err) + } + } + + keyData := r.KeyData + if keyData == nil && r.KeyFile != "" { + keyData, err = os.ReadFile(r.KeyFile) + if err != nil { + return fmt.Errorf("error reading key file from clientset: %w", err) + } + } + if keyData != nil && certData != nil { + pem, err := tls.X509KeyPair(certData, keyData) + if err != nil { + return fmt.Errorf("error creating key pair from clientset: %w", err) + } + cfg.Certificates = append(cfg.Certificates, pem) + cfg.ClientAuth = tls.RequestClientCert + } + + caData := r.CAData + if certData == nil && r.CAFile != "" { + caData, err = os.ReadFile(r.CAFile) + if err != nil { + return fmt.Errorf("error reading ca file from clientset: %w", err) + } + } + if caData != nil { + return WithCACert(caData)(cfg) + } + + return nil + } +} diff --git a/trace/opencensus/opencensus.go b/trace/opencensus/opencensus.go index 4145b5233..256d0c4b7 100644 --- a/trace/opencensus/opencensus.go +++ b/trace/opencensus/opencensus.go @@ -79,8 +79,12 @@ func (s *span) SetStatus(err error) { status.Code = octrace.StatusCodeNotFound case errdefs.IsInvalidInput(err): status.Code = octrace.StatusCodeInvalidArgument - // TODO: other error types + case errdefs.IsForbidden(err): + status.Code = octrace.StatusCodePermissionDenied + case errdefs.IsUnauthorized(err): + status.Code = octrace.StatusCodeUnauthenticated default: + // TODO: other error types status.Code = octrace.StatusCodeUnknown }