d8dfun/virtual-kubelet
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Add webhook and anon auth support

Browse Source 
Auth is not automatically enabled because this requires some
bootstrapping to work.
I'll leave this for some future work.
In the meantime people can use the current code similar to how they used
the node-cli code to inject their own auth.
This commit is contained in:
Brian Goff
2021-06-08 16:37:29 +00:00
parent e1342777d6
commit 4974e062d0
9 changed files with 483 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files

220
node/nodeutil/auth.go Normal file
View File

@@ -0,0 +1,220 @@
package nodeutil
import (
"context"
"net/http"
"strings"
"time"
"github.com/virtual-kubelet/virtual-kubelet/log"
"github.com/virtual-kubelet/virtual-kubelet/trace"
"k8s.io/apiserver/pkg/authentication/authenticator"
"k8s.io/apiserver/pkg/authentication/authenticatorfactory"
"k8s.io/apiserver/pkg/authentication/request/anonymous"
"k8s.io/apiserver/pkg/authentication/user"
"k8s.io/apiserver/pkg/authorization/authorizer"
"k8s.io/apiserver/pkg/authorization/authorizerfactory"
"k8s.io/client-go/kubernetes"
)
// Auth is the interface used to implement authn/authz for http requests
type Auth interface {
authenticator.Request
authorizer.RequestAttributesGetter
authorizer.Authorizer
}
type authWrapper struct {
authenticator.Request
authorizer.RequestAttributesGetter
authorizer.Authorizer
}
// InstrumentAuth wraps the provided Auth in a new instrumented Auth
//
// Note: You would only need this if you rolled your own auth.
// The Auth implementations defined in this package are already instrumented.
func InstrumentAuth(auth Auth) Auth {
if _, ok := auth.(*authWrapper); ok {
// This is already instrumented
return auth
}
return &authWrapper{
Request: auth,
RequestAttributesGetter: auth,
Authorizer: auth,
}
}
// NoAuth creates an Auth which allows anonymous access to all resouorces
func NoAuth() Auth {
return &authWrapper{
Request: anonymous.NewAuthenticator(),
RequestAttributesGetter: &NodeRequestAttr{},
Authorizer: authorizerfactory.NewAlwaysAllowAuthorizer(),
}
}
// WithAuth makes a new http handler which wraps the provided handler with authn/authz.
func WithAuth(auth Auth, h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
handleAuth(auth, w, r, h)
})
}
func handleAuth(auth Auth, w http.ResponseWriter, r *http.Request, next http.Handler) {
ctx := r.Context()
ctx, span := trace.StartSpan(ctx, "vk.handleAuth")
defer span.End()
r = r.WithContext(ctx)
info, ok, err := auth.AuthenticateRequest(r)
if err != nil || !ok {
log.G(r.Context()).WithError(err).Error("Authorization error")
http.Error(w, "Unauthorized", http.StatusUnauthorized)
return
}
logger := log.G(ctx).WithFields(log.Fields{
"user-name": info.User.GetName(),
"user-id": info.User.GetUID(),
})
ctx = log.WithLogger(ctx, logger)
r = r.WithContext(ctx)
attrs := auth.GetRequestAttributes(info.User, r)
decision, _, err := auth.Authorize(ctx, attrs)
if err != nil {
log.G(r.Context()).WithError(err).Error("Authorization error")
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
if decision != authorizer.DecisionAllow {
http.Error(w, "Forbidden", http.StatusForbidden)
return
}
next.ServeHTTP(w, r)
}
// WebhookAuthOption is used as a functional argument to configure webhook auth.
type WebhookAuthOption func(*WebhookAuthConfig) error
// WebhookAuthConfig stores the configurations for authn/authz and is used by WebhookAuthOption to expose to callers.
type WebhookAuthConfig struct {
AuthnConfig authenticatorfactory.DelegatingAuthenticatorConfig
AuthzConfig authorizerfactory.DelegatingAuthorizerConfig
}
// WebhookAuth creates an Auth suitable to use with kubelet webhook auth.
// You must provide a CA provider to the authentication config, otherwise mTLS is disabled.
func WebhookAuth(client kubernetes.Interface, nodeName string, opts ...WebhookAuthOption) (Auth, error) {
cfg := WebhookAuthConfig{
AuthnConfig: authenticatorfactory.DelegatingAuthenticatorConfig{
CacheTTL: 2 * time.Minute, // default taken from k8s.io/kubernetes/pkg/kubelet/apis/config/v1beta1
// TODO: After upgrading k8s libs, we need to add the retry backoff option
},
AuthzConfig: authorizerfactory.DelegatingAuthorizerConfig{
AllowCacheTTL: 5 * time.Minute, // default taken from k8s.io/kubernetes/pkg/kubelet/apis/config/v1beta1
DenyCacheTTL: 30 * time.Second, // default taken from k8s.io/kubernetes/pkg/kubelet/apis/config/v1beta1
// TODO: After upgrading k8s libs, we need to add the retry backoff option
},
}
for _, o := range opts {
if err := o(&cfg); err != nil {
return nil, err
}
}
cfg.AuthnConfig.TokenAccessReviewClient = client.AuthenticationV1().TokenReviews()
cfg.AuthzConfig.SubjectAccessReviewClient = client.AuthorizationV1().SubjectAccessReviews()
authn, _, err := cfg.AuthnConfig.New()
if err != nil {
return nil, err
}
authz, err := cfg.AuthzConfig.New()
if err != nil {
return nil, err
}
return &authWrapper{
Request: authn,
RequestAttributesGetter: NodeRequestAttr{nodeName},
Authorizer: authz,
}, nil
}
func (w *authWrapper) AuthenticateRequest(r *http.Request) (*authenticator.Response, bool, error) {
ctx, span := trace.StartSpan(r.Context(), "AuthenticateRequest")
defer span.End()
return w.Request.AuthenticateRequest(r.WithContext(ctx))
}
func (w *authWrapper) Authorize(ctx context.Context, a authorizer.Attributes) (authorizer.Decision, string, error) {
ctx, span := trace.StartSpan(ctx, "Authorize")
defer span.End()
return w.Authorizer.Authorize(ctx, a)
}
// NodeRequestAttr is a authorizor.RequeestAttributesGetter which can be used in the Auth interface.
type NodeRequestAttr struct {
NodeName string
}
// GetRequestAttributes satisfies the authorizer.RequestAttributesGetter interface for use with an `Auth`.
func (a NodeRequestAttr) GetRequestAttributes(u user.Info, r *http.Request) authorizer.Attributes {
return authorizer.AttributesRecord{
User: u,
Verb: getAPIVerb(r),
Namespace: "",
APIGroup: "",
APIVersion: "v1",
Resource: "nodes",
Name: a.NodeName,
ResourceRequest: true,
Path: r.URL.Path,
Subresource: getSubresource(r),
}
}
func getAPIVerb(r *http.Request) string {
switch r.Method {
case http.MethodPost:
return "create"
case http.MethodGet:
return "get"
case http.MethodPut:
return "update"
case http.MethodPatch:
return "patch"
case http.MethodDelete:
return "delete"
}
return ""
}
func isSubpath(subpath, path string) bool {
// Taken from k8s.io/kubernetes/pkg/kubelet/server/auth.go
return subpath == path || (strings.HasPrefix(subpath, path) && subpath[len(path)] == '/')
}
func getSubresource(r *http.Request) string {
if isSubpath(r.URL.Path, "/stats") {
return "stats"
}
if isSubpath(r.URL.Path, "/metrics") {
return "metrics"
}
if isSubpath(r.URL.Path, "/logs") {
// yes, "log", not "logs"
// per kubelet code: "log" to match other log subresources (pods/log, etc)
return "log"
}
return "proxy"
}

124
node/nodeutil/controller.go
View File

@@ -8,19 +8,19 @@ import (
"net/http"
"os"
"path"
"runtime"
"time"
"github.com/pkg/errors"
"github.com/virtual-kubelet/virtual-kubelet/log"
"github.com/virtual-kubelet/virtual-kubelet/node"
"github.com/virtual-kubelet/virtual-kubelet/node/api"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/kubernetes/scheme"
corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
corev1listers "k8s.io/client-go/listers/core/v1"
"k8s.io/client-go/tools/record"
)
@@ -42,9 +42,11 @@ type Node struct {
scmInformerFactory informers.SharedInformerFactory
client kubernetes.Interface
listenAddr string
httpHandler HTTPHandler
tlsConfig *tls.Config
listenAddr string
h http.Handler
tlsConfig *tls.Config
workers int
eb record.EventBroadcaster
}
@@ -59,8 +61,35 @@ func (n *Node) PodController() *node.PodController {
return n.pc
}
func (n *Node) runHTTP(ctx context.Context) (func(), error) {
if n.tlsConfig == nil {
log.G(ctx).Warn("TLS config not provided, not starting up http service")
return func() {}, nil
}
if n.h == nil {
log.G(ctx).Debug("No http handler, not starting up http service")
return func() {}, nil
}
l, err := tls.Listen("tcp", n.listenAddr, n.tlsConfig)
if err != nil {
return nil, errors.Wrap(err, "error starting http listener")
}
log.G(ctx).Debug("Started TLS listener")
srv := &http.Server{Handler: n.h, TLSConfig: n.tlsConfig}
go srv.Serve(l) //nolint:errcheck
log.G(ctx).Debug("HTTP server running")
return func() {
srv.Close()
l.Close()
}, nil
}
// Run starts all the underlying controllers
func (n *Node) Run(ctx context.Context, workers int) (retErr error) {
func (n *Node) Run(ctx context.Context) (retErr error) {
ctx, cancel := context.WithCancel(ctx)
defer func() {
cancel()
@@ -69,31 +98,22 @@ func (n *Node) Run(ctx context.Context, workers int) (retErr error) {
close(n.done)
}()
if n.podInformerFactory != nil {
go n.podInformerFactory.Start(ctx.Done())
}
if n.scmInformerFactory != nil {
go n.scmInformerFactory.Start(ctx.Done())
}
if n.eb != nil {
n.eb.StartLogging(log.G(ctx).Infof)
n.eb.StartRecordingToSink(&corev1client.EventSinkImpl{Interface: n.client.CoreV1().Events(v1.NamespaceAll)})
defer n.eb.Shutdown()
log.G(ctx).Debug("Started event broadcaster")
}
l, err := tls.Listen("tcp", n.listenAddr, n.tlsConfig)
cancelHTTP, err := n.runHTTP(ctx)
if err != nil {
return errors.Wrap(err, "error starting http listener")
return err
}
log.G(ctx).Debug("Started TLS listener")
defer l.Close()
defer cancelHTTP()
srv := &http.Server{Handler: n.httpHandler, TLSConfig: n.tlsConfig}
go srv.Serve(l)
defer srv.Close()
go n.pc.Run(ctx, workers) //nolint:errcheck
log.G(ctx).Debug("HTTP server running")
go n.podInformerFactory.Start(ctx.Done())
go n.scmInformerFactory.Start(ctx.Done())
go n.pc.Run(ctx, n.workers) //nolint:errcheck
defer func() {
cancel()
@@ -110,7 +130,7 @@ func (n *Node) Run(ctx context.Context, workers int) (retErr error) {
log.G(ctx).Debug("pod controller ready")
go n.nc.Run(ctx) // nolint:errcheck
go n.nc.Run(ctx) //nolint:errcheck
defer func() {
cancel()
@@ -193,6 +213,9 @@ type NodeOpt func(c *NodeConfig) error
// NodeConfig is used to hold configuration items for a Node.
// It gets used in conjection with NodeOpt in NewNodeFromClient
type NodeConfig struct {
// Set the client to use, otherwise a client will be created from ClientsetFromEnv
Client kubernetes.Interface
// Set the node spec to register with Kubernetes
NodeSpec v1.Node
// Set the path to read a kubeconfig from for creating a client.
@@ -206,8 +229,9 @@ type NodeConfig struct {
// Set a custom API handler to use.
// You can use this to setup, for example, authentication middleware.
// If one is not provided a default one will be created.
// Pod routes will be attached to this handler when creating the node
HTTPHandler HTTPHandler
//
// Note: If you provide your own handler, you'll need to handle all auth, routes, etc.
Handler http.Handler
// Set the timeout for idle http streams
StreamIdleTimeout time.Duration
// Set the timeout for creating http streams
@@ -220,11 +244,12 @@ type NodeConfig struct {
// Specify the event recorder to use
// If this is not provided, a default one will be used.
EventRecorder record.EventRecorder
}
type HTTPHandler interface {
api.ServeMux
http.Handler
// Set the number of workers to reconcile pods
// The default value is derived from the number of cores available.
NumWorkers int
routeAttacher func(Provider, NodeConfig, corev1listers.PodLister)
}
// WithNodeConfig returns a NodeOpt which replaces the NodeConfig with the passed in value.
@@ -235,9 +260,12 @@ func WithNodeConfig(c NodeConfig) NodeOpt {
}
}
// NewNode calls NewNodeFromClient with a nil client
func NewNode(name string, newProvider NewProviderFunc, opts ...NodeOpt) (*Node, error) {
return NewNodeFromClient(nil, name, newProvider, opts...)
// WithClient return a NodeOpt that sets the client that will be used to create/manage the node.
func WithClient(c kubernetes.Interface) NodeOpt {
return func(cfg *NodeConfig) error {
cfg.Client = c
return nil
}
}
// NewNodeFromClient creates a new node using the provided client and name.
@@ -247,9 +275,11 @@ func NewNode(name string, newProvider NewProviderFunc, opts ...NodeOpt) (*Node,
// Some basic values are set for node status, you'll almost certainly want to modify it.
//
// If client is nil, this will construct a client using ClientsetFromEnv
func NewNodeFromClient(client kubernetes.Interface, name string, newProvider NewProviderFunc, opts ...NodeOpt) (*Node, error) {
//
// It is up to the caller to configure auth on the HTTP handler.
func NewNodeFromClient(name string, newProvider NewProviderFunc, opts ...NodeOpt) (*Node, error) {
cfg := NodeConfig{
// TODO: this is what was set in the cli code... its not clear what a good value actually is.
NumWorkers: runtime.NumCPU(),
InformerResyncPeriod: time.Minute,
KubeconfigPath: os.Getenv("KUBECONFIG"),
HTTPListenAddr: ":10250",
@@ -285,10 +315,7 @@ func NewNodeFromClient(client kubernetes.Interface, name string, newProvider New
return nil, errors.Wrap(err, "error parsing http listen address")
}
if cfg.HTTPHandler == nil {
cfg.HTTPHandler = http.NewServeMux()
}
client := cfg.Client
if client == nil {
var err error
client, err = ClientsetFromEnv(cfg.KubeconfigPath)
@@ -324,17 +351,9 @@ func NewNodeFromClient(client kubernetes.Interface, name string, newProvider New
return nil, errors.Wrap(err, "error creating provider")
}
api.AttachPodRoutes(api.PodHandlerConfig{
RunInContainer: p.RunInContainer,
GetContainerLogs: p.GetContainerLogs,
GetPods: p.GetPods,
GetPodsFromKubernetes: func(context.Context) ([]*v1.Pod, error) {
return podInformer.Lister().List(labels.Everything())
},
GetStatsSummary: p.GetStatsSummary,
StreamIdleTimeout: cfg.StreamIdleTimeout,
StreamCreationTimeout: cfg.StreamCreationTimeout,
}, cfg.HTTPHandler, cfg.DebugHTTP)
if cfg.routeAttacher != nil {
cfg.routeAttacher(p, cfg, podInformer.Lister())
}
var readyCb func(context.Context) error
if np == nil {
@@ -360,7 +379,7 @@ func NewNodeFromClient(client kubernetes.Interface, name string, newProvider New
var eb record.EventBroadcaster
if cfg.EventRecorder == nil {
eb := record.NewBroadcaster()
eb = record.NewBroadcaster()
cfg.EventRecorder = eb.NewRecorder(scheme.Scheme, v1.EventSource{Component: path.Join(name, "pod-controller")})
}
@@ -388,8 +407,9 @@ func NewNodeFromClient(client kubernetes.Interface, name string, newProvider New
scmInformerFactory: scmInformerFactory,
client: client,
tlsConfig: cfg.TLSConfig,
httpHandler: cfg.HTTPHandler,
h: cfg.Handler,
listenAddr: cfg.HTTPListenAddr,
workers: cfg.NumWorkers,
}, nil
}

23
node/nodeutil/provider.go
View File

@@ -8,6 +8,7 @@ import (
"github.com/virtual-kubelet/virtual-kubelet/node/api"
"github.com/virtual-kubelet/virtual-kubelet/node/api/statsv1alpha1"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/labels"
corev1listers "k8s.io/client-go/listers/core/v1"
)
@@ -45,3 +46,25 @@ type ProviderConfig struct {
// NewProviderFunc is used from NewNodeFromClient to bootstrap a provider using the client/listers/etc created there.
// If a nil node provider is returned a default one will be used.
type NewProviderFunc func(ProviderConfig) (Provider, node.NodeProvider, error)
// AttachProviderRoutes returns a NodeOpt which uses api.PodHandler to attach the routes to the provider functions.
//
// Note this only attaches routes, you'll need to ensure to set the handler in the node config.
func AttachProviderRoutes(mux api.ServeMux) NodeOpt {
return func(cfg *NodeConfig) error {
cfg.routeAttacher = func(p Provider, cfg NodeConfig, pods corev1listers.PodLister) {
mux.Handle("/", api.PodHandler(api.PodHandlerConfig{
RunInContainer: p.RunInContainer,
GetContainerLogs: p.GetContainerLogs,
GetPods: p.GetPods,
GetPodsFromKubernetes: func(context.Context) ([]*v1.Pod, error) {
return pods.List(labels.Everything())
},
GetStatsSummary: p.GetStatsSummary,
StreamIdleTimeout: cfg.StreamIdleTimeout,
StreamCreationTimeout: cfg.StreamCreationTimeout,
}, true))
}
return nil
}
}

83
node/nodeutil/tls.go Normal file
View File

@@ -0,0 +1,83 @@
package nodeutil
import (
"crypto/tls"
"crypto/x509"
"fmt"
"io/ioutil"
)
// WithTLSConfig returns a NodeOpt which creates a base TLSConfig with the default cipher suites and tls min verions.
// The tls config can be modified through functional options.
func WithTLSConfig(opts ...func(*tls.Config) error) NodeOpt {
return func(cfg *NodeConfig) error {
tlsCfg := &tls.Config{
MinVersion: tls.VersionTLS12,
PreferServerCipherSuites: true,
CipherSuites: DefaultServerCiphers(),
ClientAuth: tls.RequestClientCert,
}
for _, o := range opts {
if err := o(tlsCfg); err != nil {
return err
}
}
cfg.TLSConfig = tlsCfg
return nil
}
}
// WithCAFromPath makes a TLS config option to set up client auth using the path to a PEM encoded CA cert.
func WithCAFromPath(p string) func(*tls.Config) error {
return func(cfg *tls.Config) error {
pem, err := ioutil.ReadFile(p)
if err != nil {
return fmt.Errorf("error reading ca cert pem: %w", err)
}
cfg.ClientAuth = tls.RequireAndVerifyClientCert
return WithCACert(pem)(cfg)
}
}
// WithKeyPairFromPath make sa TLS config option which loads the key pair paths from disk and appends them to the tls config.
func WithKeyPairFromPath(cert, key string) func(*tls.Config) error {
return func(cfg *tls.Config) error {
cert, err := tls.LoadX509KeyPair(cert, key)
if err != nil {
return err
}
cfg.Certificates = append(cfg.Certificates, cert)
return nil
}
}
// WithCACert makes a TLS config opotion which appends the provided PEM encoded bytes the tls config's cert pool.
// If a cert pool is not defined on the tls config an empty one will be created.
func WithCACert(pem []byte) func(*tls.Config) error {
return func(cfg *tls.Config) error {
if cfg.ClientCAs == nil {
cfg.ClientCAs = x509.NewCertPool()
}
if !cfg.ClientCAs.AppendCertsFromPEM(pem) {
return fmt.Errorf("could not parse ca cert pem")
}
return nil
}
}
// DefaultServerCiphers is the list of accepted TLS ciphers, with known weak ciphers elided
// Note this list should be a moving target.
func DefaultServerCiphers() []uint16 {
return []uint16{
tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
}
}