diff --git a/cmd/virtual-kubelet/internal/commands/root/http.go b/cmd/virtual-kubelet/internal/commands/root/http.go index 2b9633edf..a14b2c9df 100644 --- a/cmd/virtual-kubelet/internal/commands/root/http.go +++ b/cmd/virtual-kubelet/internal/commands/root/http.go @@ -15,45 +15,15 @@ package root import ( - "crypto/tls" "fmt" "os" "time" - - "github.com/pkg/errors" ) -// AcceptedCiphers is the list of accepted TLS ciphers, with known weak ciphers elided -// Note this list should be a moving target. -var AcceptedCiphers = []uint16{ - tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, - tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, - tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, - tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, - - tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, - tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, - tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, - tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, -} - -func loadTLSConfig(certPath, keyPath string) (*tls.Config, error) { - cert, err := tls.LoadX509KeyPair(certPath, keyPath) - if err != nil { - return nil, errors.Wrap(err, "error loading tls certs") - } - - return &tls.Config{ - Certificates: []tls.Certificate{cert}, - MinVersion: tls.VersionTLS12, - PreferServerCipherSuites: true, - CipherSuites: AcceptedCiphers, - }, nil -} - type apiServerConfig struct { CertPath string KeyPath string + CACertPath string Addr string MetricsAddr string StreamIdleTimeout time.Duration @@ -62,8 +32,9 @@ type apiServerConfig struct { func getAPIConfig(c Opts) (*apiServerConfig, error) { config := apiServerConfig{ - CertPath: os.Getenv("APISERVER_CERT_LOCATION"), - KeyPath: os.Getenv("APISERVER_KEY_LOCATION"), + CertPath: os.Getenv("APISERVER_CERT_LOCATION"), + KeyPath: os.Getenv("APISERVER_KEY_LOCATION"), + CACertPath: os.Getenv("APISERVER_CA_CERT_LOCATION"), } config.Addr = fmt.Sprintf(":%d", c.ListenPort) diff --git a/cmd/virtual-kubelet/internal/commands/root/root.go b/cmd/virtual-kubelet/internal/commands/root/root.go index c4777023d..ade4daa61 100644 --- a/cmd/virtual-kubelet/internal/commands/root/root.go +++ b/cmd/virtual-kubelet/internal/commands/root/root.go @@ -16,6 +16,8 @@ package root import ( "context" + "crypto/tls" + "net/http" "os" "runtime" @@ -26,8 +28,11 @@ import ( "github.com/virtual-kubelet/virtual-kubelet/internal/manager" "github.com/virtual-kubelet/virtual-kubelet/log" "github.com/virtual-kubelet/virtual-kubelet/node" + "github.com/virtual-kubelet/virtual-kubelet/node/api" "github.com/virtual-kubelet/virtual-kubelet/node/nodeutil" corev1 "k8s.io/api/core/v1" + "k8s.io/apiserver/pkg/server/dynamiccertificates" + "k8s.io/client-go/kubernetes" ) // NewCommand creates a new top-level command. @@ -74,8 +79,8 @@ func runRootCommand(ctx context.Context, s *provider.Store, c Opts) error { return err } + mux := http.NewServeMux() newProvider := func(cfg nodeutil.ProviderConfig) (nodeutil.Provider, node.NodeProvider, error) { - var err error rm, err := manager.NewResourceManager(cfg.Pods, cfg.Secrets, cfg.ConfigMaps, cfg.Services) if err != nil { return nil, nil, errors.Wrap(err, "could not create resource manager") @@ -99,11 +104,17 @@ func runRootCommand(ctx context.Context, s *provider.Store, c Opts) error { return nil, nil, errors.Wrapf(err, "error initializing provider %s", c.Provider) } p.ConfigureNode(ctx, cfg.Node) - + cfg.Node.Status.NodeInfo.KubeletVersion = c.Version return p, nil, nil } - cm, err := nodeutil.NewNodeFromClient(client, c.NodeName, newProvider, func(cfg *nodeutil.NodeConfig) error { + apiConfig, err := getAPIConfig(c) + if err != nil { + return err + } + + cm, err := nodeutil.NewNodeFromClient(c.NodeName, newProvider, func(cfg *nodeutil.NodeConfig) error { + cfg.Handler = mux cfg.InformerResyncPeriod = c.InformerResyncPeriod if taint != nil { @@ -112,23 +123,23 @@ func runRootCommand(ctx context.Context, s *provider.Store, c Opts) error { cfg.NodeSpec.Status.NodeInfo.Architecture = runtime.GOARCH cfg.NodeSpec.Status.NodeInfo.OperatingSystem = c.OperatingSystem - apiConfig, err := getAPIConfig(c) - if err != nil { - return err - } - cfg.HTTPListenAddr = apiConfig.Addr cfg.StreamCreationTimeout = apiConfig.StreamCreationTimeout cfg.StreamIdleTimeout = apiConfig.StreamIdleTimeout - - cfg.TLSConfig, err = loadTLSConfig(apiConfig.CertPath, apiConfig.KeyPath) - if err != nil { - return errors.Wrap(err, "error loading tls config") - } - cfg.DebugHTTP = true + + cfg.NumWorkers = c.PodSyncWorkers + return nil - }) + }, + nodeutil.WithClient(client), + setAuth(client, c.NodeName, apiConfig), + nodeutil.WithTLSConfig( + nodeutil.WithKeyPairFromPath(apiConfig.CertPath, apiConfig.KeyPath), + maybeCA(apiConfig.CACertPath), + ), + nodeutil.AttachProviderRoutes(mux), + ) if err != nil { return err } @@ -144,7 +155,7 @@ func runRootCommand(ctx context.Context, s *provider.Store, c Opts) error { "watchedNamespace": c.KubeNamespace, })) - go cm.Run(ctx, c.PodSyncWorkers) // nolint:errcheck + go cm.Run(ctx) //nolint:errcheck defer func() { log.G(ctx).Debug("Waiting for controllers to be done") @@ -166,3 +177,32 @@ func runRootCommand(ctx context.Context, s *provider.Store, c Opts) error { } return nil } + +func setAuth(client kubernetes.Interface, node string, apiCfg *apiServerConfig) nodeutil.NodeOpt { + if apiCfg.CACertPath == "" { + return func(cfg *nodeutil.NodeConfig) error { + cfg.Handler = api.InstrumentHandler(nodeutil.WithAuth(nodeutil.NoAuth(), cfg.Handler)) + return nil + } + } + + return func(cfg *nodeutil.NodeConfig) error { + auth, err := nodeutil.WebhookAuth(client, node, func(cfg *nodeutil.WebhookAuthConfig) error { + var err error + cfg.AuthnConfig.ClientCertificateCAContentProvider, err = dynamiccertificates.NewDynamicCAContentFromFile("ca-cert-bundle", apiCfg.CACertPath) + return err + }) + if err != nil { + return err + } + cfg.Handler = api.InstrumentHandler(nodeutil.WithAuth(auth, cfg.Handler)) + return nil + } +} + +func maybeCA(p string) func(*tls.Config) error { + if p == "" { + return func(*tls.Config) error { return nil } + } + return nodeutil.WithCAFromPath(p) +} diff --git a/go.sum b/go.sum index 0c24d7c00..5910cae94 100644 --- a/go.sum +++ b/go.sum @@ -34,8 +34,10 @@ github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb0 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= @@ -55,6 +57,7 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/blang/semver v3.5.0+incompatible h1:CGxCgetQ64DKk7rdZ++Vfnb1+ogGNnB17OJKJXD2Cfs= github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/bombsimon/logrusr v1.0.0 h1:CTCkURYAt5nhCCnKH9eLShYayj2/8Kn/4Qg3QfiU+Ro= github.com/bombsimon/logrusr v1.0.0/go.mod h1:Jq0nHtvxabKE5EMwAAdgTaz7dfWE8C4i11NOltxGQpc= @@ -140,11 +143,13 @@ github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+ github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= +github.com/go-openapi/jsonpointer v0.19.3 h1:gihV7YNZK1iK6Tgwwsxo2rJbD1GTbdm72325Bq8FI3w= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= +github.com/go-openapi/jsonreference v0.19.3 h1:5cxNfTy0UVC3X8JL5ymxzyoUZmo8iZb+jeTWn7tUa8o= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= github.com/go-openapi/loads v0.18.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= @@ -158,6 +163,7 @@ github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nA github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY= +github.com/go-openapi/spec v0.19.3 h1:0XRyw8kguri6Yw4SxhsQA/atC88yqrk0+G4YhI2wabc= github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= github.com/go-openapi/strfmt v0.17.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= github.com/go-openapi/strfmt v0.18.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= @@ -167,6 +173,7 @@ github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dp github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA= @@ -281,6 +288,7 @@ github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= @@ -371,6 +379,7 @@ github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrf github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= @@ -704,6 +713,7 @@ k8s.io/utils v0.0.0-20200912215256-4140de9c8800 h1:9ZNvfPvVIEsp/T1ez4GQuzCcCTEQW k8s.io/utils v0.0.0-20200912215256-4140de9c8800/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.9/go.mod h1:dzAXnQbTRyDlZPJX2SUPEqvnB+j7AJjtlox7PEwigU0= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15 h1:4uqm9Mv+w2MmBYD+F4qf/v6tDFUdPOk29C095RbU5mY= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/controller-runtime v0.7.1 h1:nqVwzVzdenfd9xIbB35pC7JJH2IXVL4hDo3MNzkyCh4= sigs.k8s.io/controller-runtime v0.7.1/go.mod h1:pJ3YBrJiAqMAZKi6UVGuE98ZrroV1p+pIhoHsMm9wdU= diff --git a/hack/skaffold/virtual-kubelet/base.yml b/hack/skaffold/virtual-kubelet/base.yml index 0cbfc14c2..f13ad0426 100644 --- a/hack/skaffold/virtual-kubelet/base.yml +++ b/hack/skaffold/virtual-kubelet/base.yml @@ -56,6 +56,14 @@ rules: verbs: - create - patch +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/hack/skaffold/virtual-kubelet/pod.yml b/hack/skaffold/virtual-kubelet/pod.yml index e95455b00..bdc4ff5f5 100644 --- a/hack/skaffold/virtual-kubelet/pod.yml +++ b/hack/skaffold/virtual-kubelet/pod.yml @@ -4,6 +4,8 @@ metadata: name: vkubelet-mock-0 spec: containers: + - name: jaeger-tracing + image: jaegertracing/all-in-one:1.22 - name: vkubelet-mock-0 image: virtual-kubelet # "IfNotPresent" is used to prevent Minikube from trying to pull from the registry (and failing) in the first place. @@ -23,7 +25,12 @@ spec: - --klog.logtostderr - --log-level - debug + - --trace-exporter + - jaeger + - --trace-sample-rate=always env: + - name: JAEGER_AGENT_ENDPOINT + value: localhost:6831 - name: KUBELET_PORT value: "10250" - name: VKUBELET_POD_IP diff --git a/node/nodeutil/auth.go b/node/nodeutil/auth.go new file mode 100644 index 000000000..ef5e52cac --- /dev/null +++ b/node/nodeutil/auth.go @@ -0,0 +1,220 @@ +package nodeutil + +import ( + "context" + "net/http" + "strings" + "time" + + "github.com/virtual-kubelet/virtual-kubelet/log" + "github.com/virtual-kubelet/virtual-kubelet/trace" + "k8s.io/apiserver/pkg/authentication/authenticator" + "k8s.io/apiserver/pkg/authentication/authenticatorfactory" + "k8s.io/apiserver/pkg/authentication/request/anonymous" + "k8s.io/apiserver/pkg/authentication/user" + "k8s.io/apiserver/pkg/authorization/authorizer" + "k8s.io/apiserver/pkg/authorization/authorizerfactory" + "k8s.io/client-go/kubernetes" +) + +// Auth is the interface used to implement authn/authz for http requests +type Auth interface { + authenticator.Request + authorizer.RequestAttributesGetter + authorizer.Authorizer +} + +type authWrapper struct { + authenticator.Request + authorizer.RequestAttributesGetter + authorizer.Authorizer +} + +// InstrumentAuth wraps the provided Auth in a new instrumented Auth +// +// Note: You would only need this if you rolled your own auth. +// The Auth implementations defined in this package are already instrumented. +func InstrumentAuth(auth Auth) Auth { + if _, ok := auth.(*authWrapper); ok { + // This is already instrumented + return auth + } + return &authWrapper{ + Request: auth, + RequestAttributesGetter: auth, + Authorizer: auth, + } +} + +// NoAuth creates an Auth which allows anonymous access to all resouorces +func NoAuth() Auth { + return &authWrapper{ + Request: anonymous.NewAuthenticator(), + RequestAttributesGetter: &NodeRequestAttr{}, + Authorizer: authorizerfactory.NewAlwaysAllowAuthorizer(), + } +} + +// WithAuth makes a new http handler which wraps the provided handler with authn/authz. +func WithAuth(auth Auth, h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + handleAuth(auth, w, r, h) + }) +} + +func handleAuth(auth Auth, w http.ResponseWriter, r *http.Request, next http.Handler) { + ctx := r.Context() + ctx, span := trace.StartSpan(ctx, "vk.handleAuth") + defer span.End() + r = r.WithContext(ctx) + + info, ok, err := auth.AuthenticateRequest(r) + if err != nil || !ok { + log.G(r.Context()).WithError(err).Error("Authorization error") + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return + } + + logger := log.G(ctx).WithFields(log.Fields{ + "user-name": info.User.GetName(), + "user-id": info.User.GetUID(), + }) + + ctx = log.WithLogger(ctx, logger) + r = r.WithContext(ctx) + + attrs := auth.GetRequestAttributes(info.User, r) + + decision, _, err := auth.Authorize(ctx, attrs) + if err != nil { + log.G(r.Context()).WithError(err).Error("Authorization error") + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + if decision != authorizer.DecisionAllow { + http.Error(w, "Forbidden", http.StatusForbidden) + return + } + + next.ServeHTTP(w, r) +} + +// WebhookAuthOption is used as a functional argument to configure webhook auth. +type WebhookAuthOption func(*WebhookAuthConfig) error + +// WebhookAuthConfig stores the configurations for authn/authz and is used by WebhookAuthOption to expose to callers. +type WebhookAuthConfig struct { + AuthnConfig authenticatorfactory.DelegatingAuthenticatorConfig + AuthzConfig authorizerfactory.DelegatingAuthorizerConfig +} + +// WebhookAuth creates an Auth suitable to use with kubelet webhook auth. +// You must provide a CA provider to the authentication config, otherwise mTLS is disabled. +func WebhookAuth(client kubernetes.Interface, nodeName string, opts ...WebhookAuthOption) (Auth, error) { + cfg := WebhookAuthConfig{ + AuthnConfig: authenticatorfactory.DelegatingAuthenticatorConfig{ + CacheTTL: 2 * time.Minute, // default taken from k8s.io/kubernetes/pkg/kubelet/apis/config/v1beta1 + // TODO: After upgrading k8s libs, we need to add the retry backoff option + }, + AuthzConfig: authorizerfactory.DelegatingAuthorizerConfig{ + AllowCacheTTL: 5 * time.Minute, // default taken from k8s.io/kubernetes/pkg/kubelet/apis/config/v1beta1 + DenyCacheTTL: 30 * time.Second, // default taken from k8s.io/kubernetes/pkg/kubelet/apis/config/v1beta1 + // TODO: After upgrading k8s libs, we need to add the retry backoff option + }, + } + + for _, o := range opts { + if err := o(&cfg); err != nil { + return nil, err + } + } + + cfg.AuthnConfig.TokenAccessReviewClient = client.AuthenticationV1().TokenReviews() + cfg.AuthzConfig.SubjectAccessReviewClient = client.AuthorizationV1().SubjectAccessReviews() + + authn, _, err := cfg.AuthnConfig.New() + if err != nil { + return nil, err + } + + authz, err := cfg.AuthzConfig.New() + if err != nil { + return nil, err + } + return &authWrapper{ + Request: authn, + RequestAttributesGetter: NodeRequestAttr{nodeName}, + Authorizer: authz, + }, nil +} + +func (w *authWrapper) AuthenticateRequest(r *http.Request) (*authenticator.Response, bool, error) { + ctx, span := trace.StartSpan(r.Context(), "AuthenticateRequest") + defer span.End() + return w.Request.AuthenticateRequest(r.WithContext(ctx)) +} + +func (w *authWrapper) Authorize(ctx context.Context, a authorizer.Attributes) (authorizer.Decision, string, error) { + ctx, span := trace.StartSpan(ctx, "Authorize") + defer span.End() + return w.Authorizer.Authorize(ctx, a) +} + +// NodeRequestAttr is a authorizor.RequeestAttributesGetter which can be used in the Auth interface. +type NodeRequestAttr struct { + NodeName string +} + +// GetRequestAttributes satisfies the authorizer.RequestAttributesGetter interface for use with an `Auth`. +func (a NodeRequestAttr) GetRequestAttributes(u user.Info, r *http.Request) authorizer.Attributes { + return authorizer.AttributesRecord{ + User: u, + Verb: getAPIVerb(r), + Namespace: "", + APIGroup: "", + APIVersion: "v1", + Resource: "nodes", + Name: a.NodeName, + ResourceRequest: true, + Path: r.URL.Path, + Subresource: getSubresource(r), + } +} + +func getAPIVerb(r *http.Request) string { + switch r.Method { + case http.MethodPost: + return "create" + case http.MethodGet: + return "get" + case http.MethodPut: + return "update" + case http.MethodPatch: + return "patch" + case http.MethodDelete: + return "delete" + } + return "" +} + +func isSubpath(subpath, path string) bool { + // Taken from k8s.io/kubernetes/pkg/kubelet/server/auth.go + return subpath == path || (strings.HasPrefix(subpath, path) && subpath[len(path)] == '/') +} + +func getSubresource(r *http.Request) string { + if isSubpath(r.URL.Path, "/stats") { + return "stats" + } + if isSubpath(r.URL.Path, "/metrics") { + return "metrics" + } + if isSubpath(r.URL.Path, "/logs") { + // yes, "log", not "logs" + // per kubelet code: "log" to match other log subresources (pods/log, etc) + return "log" + } + + return "proxy" +} diff --git a/node/nodeutil/controller.go b/node/nodeutil/controller.go index 0500f9a6f..1edae926b 100644 --- a/node/nodeutil/controller.go +++ b/node/nodeutil/controller.go @@ -8,19 +8,19 @@ import ( "net/http" "os" "path" + "runtime" "time" "github.com/pkg/errors" "github.com/virtual-kubelet/virtual-kubelet/log" "github.com/virtual-kubelet/virtual-kubelet/node" - "github.com/virtual-kubelet/virtual-kubelet/node/api" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" corev1client "k8s.io/client-go/kubernetes/typed/core/v1" + corev1listers "k8s.io/client-go/listers/core/v1" "k8s.io/client-go/tools/record" ) @@ -42,9 +42,11 @@ type Node struct { scmInformerFactory informers.SharedInformerFactory client kubernetes.Interface - listenAddr string - httpHandler HTTPHandler - tlsConfig *tls.Config + listenAddr string + h http.Handler + tlsConfig *tls.Config + + workers int eb record.EventBroadcaster } @@ -59,8 +61,35 @@ func (n *Node) PodController() *node.PodController { return n.pc } +func (n *Node) runHTTP(ctx context.Context) (func(), error) { + if n.tlsConfig == nil { + log.G(ctx).Warn("TLS config not provided, not starting up http service") + return func() {}, nil + } + if n.h == nil { + log.G(ctx).Debug("No http handler, not starting up http service") + return func() {}, nil + } + + l, err := tls.Listen("tcp", n.listenAddr, n.tlsConfig) + if err != nil { + return nil, errors.Wrap(err, "error starting http listener") + } + + log.G(ctx).Debug("Started TLS listener") + + srv := &http.Server{Handler: n.h, TLSConfig: n.tlsConfig} + go srv.Serve(l) //nolint:errcheck + log.G(ctx).Debug("HTTP server running") + + return func() { + srv.Close() + l.Close() + }, nil +} + // Run starts all the underlying controllers -func (n *Node) Run(ctx context.Context, workers int) (retErr error) { +func (n *Node) Run(ctx context.Context) (retErr error) { ctx, cancel := context.WithCancel(ctx) defer func() { cancel() @@ -69,31 +98,22 @@ func (n *Node) Run(ctx context.Context, workers int) (retErr error) { close(n.done) }() - if n.podInformerFactory != nil { - go n.podInformerFactory.Start(ctx.Done()) - } - if n.scmInformerFactory != nil { - go n.scmInformerFactory.Start(ctx.Done()) - } - if n.eb != nil { n.eb.StartLogging(log.G(ctx).Infof) n.eb.StartRecordingToSink(&corev1client.EventSinkImpl{Interface: n.client.CoreV1().Events(v1.NamespaceAll)}) + defer n.eb.Shutdown() + log.G(ctx).Debug("Started event broadcaster") } - l, err := tls.Listen("tcp", n.listenAddr, n.tlsConfig) + cancelHTTP, err := n.runHTTP(ctx) if err != nil { - return errors.Wrap(err, "error starting http listener") + return err } - log.G(ctx).Debug("Started TLS listener") - defer l.Close() + defer cancelHTTP() - srv := &http.Server{Handler: n.httpHandler, TLSConfig: n.tlsConfig} - go srv.Serve(l) - defer srv.Close() - - go n.pc.Run(ctx, workers) //nolint:errcheck - log.G(ctx).Debug("HTTP server running") + go n.podInformerFactory.Start(ctx.Done()) + go n.scmInformerFactory.Start(ctx.Done()) + go n.pc.Run(ctx, n.workers) //nolint:errcheck defer func() { cancel() @@ -110,7 +130,7 @@ func (n *Node) Run(ctx context.Context, workers int) (retErr error) { log.G(ctx).Debug("pod controller ready") - go n.nc.Run(ctx) // nolint:errcheck + go n.nc.Run(ctx) //nolint:errcheck defer func() { cancel() @@ -193,6 +213,9 @@ type NodeOpt func(c *NodeConfig) error // NodeConfig is used to hold configuration items for a Node. // It gets used in conjection with NodeOpt in NewNodeFromClient type NodeConfig struct { + // Set the client to use, otherwise a client will be created from ClientsetFromEnv + Client kubernetes.Interface + // Set the node spec to register with Kubernetes NodeSpec v1.Node // Set the path to read a kubeconfig from for creating a client. @@ -206,8 +229,9 @@ type NodeConfig struct { // Set a custom API handler to use. // You can use this to setup, for example, authentication middleware. // If one is not provided a default one will be created. - // Pod routes will be attached to this handler when creating the node - HTTPHandler HTTPHandler + // + // Note: If you provide your own handler, you'll need to handle all auth, routes, etc. + Handler http.Handler // Set the timeout for idle http streams StreamIdleTimeout time.Duration // Set the timeout for creating http streams @@ -220,11 +244,12 @@ type NodeConfig struct { // Specify the event recorder to use // If this is not provided, a default one will be used. EventRecorder record.EventRecorder -} -type HTTPHandler interface { - api.ServeMux - http.Handler + // Set the number of workers to reconcile pods + // The default value is derived from the number of cores available. + NumWorkers int + + routeAttacher func(Provider, NodeConfig, corev1listers.PodLister) } // WithNodeConfig returns a NodeOpt which replaces the NodeConfig with the passed in value. @@ -235,9 +260,12 @@ func WithNodeConfig(c NodeConfig) NodeOpt { } } -// NewNode calls NewNodeFromClient with a nil client -func NewNode(name string, newProvider NewProviderFunc, opts ...NodeOpt) (*Node, error) { - return NewNodeFromClient(nil, name, newProvider, opts...) +// WithClient return a NodeOpt that sets the client that will be used to create/manage the node. +func WithClient(c kubernetes.Interface) NodeOpt { + return func(cfg *NodeConfig) error { + cfg.Client = c + return nil + } } // NewNodeFromClient creates a new node using the provided client and name. @@ -247,9 +275,11 @@ func NewNode(name string, newProvider NewProviderFunc, opts ...NodeOpt) (*Node, // Some basic values are set for node status, you'll almost certainly want to modify it. // // If client is nil, this will construct a client using ClientsetFromEnv -func NewNodeFromClient(client kubernetes.Interface, name string, newProvider NewProviderFunc, opts ...NodeOpt) (*Node, error) { +// +// It is up to the caller to configure auth on the HTTP handler. +func NewNodeFromClient(name string, newProvider NewProviderFunc, opts ...NodeOpt) (*Node, error) { cfg := NodeConfig{ - // TODO: this is what was set in the cli code... its not clear what a good value actually is. + NumWorkers: runtime.NumCPU(), InformerResyncPeriod: time.Minute, KubeconfigPath: os.Getenv("KUBECONFIG"), HTTPListenAddr: ":10250", @@ -285,10 +315,7 @@ func NewNodeFromClient(client kubernetes.Interface, name string, newProvider New return nil, errors.Wrap(err, "error parsing http listen address") } - if cfg.HTTPHandler == nil { - cfg.HTTPHandler = http.NewServeMux() - } - + client := cfg.Client if client == nil { var err error client, err = ClientsetFromEnv(cfg.KubeconfigPath) @@ -324,17 +351,9 @@ func NewNodeFromClient(client kubernetes.Interface, name string, newProvider New return nil, errors.Wrap(err, "error creating provider") } - api.AttachPodRoutes(api.PodHandlerConfig{ - RunInContainer: p.RunInContainer, - GetContainerLogs: p.GetContainerLogs, - GetPods: p.GetPods, - GetPodsFromKubernetes: func(context.Context) ([]*v1.Pod, error) { - return podInformer.Lister().List(labels.Everything()) - }, - GetStatsSummary: p.GetStatsSummary, - StreamIdleTimeout: cfg.StreamIdleTimeout, - StreamCreationTimeout: cfg.StreamCreationTimeout, - }, cfg.HTTPHandler, cfg.DebugHTTP) + if cfg.routeAttacher != nil { + cfg.routeAttacher(p, cfg, podInformer.Lister()) + } var readyCb func(context.Context) error if np == nil { @@ -360,7 +379,7 @@ func NewNodeFromClient(client kubernetes.Interface, name string, newProvider New var eb record.EventBroadcaster if cfg.EventRecorder == nil { - eb := record.NewBroadcaster() + eb = record.NewBroadcaster() cfg.EventRecorder = eb.NewRecorder(scheme.Scheme, v1.EventSource{Component: path.Join(name, "pod-controller")}) } @@ -388,8 +407,9 @@ func NewNodeFromClient(client kubernetes.Interface, name string, newProvider New scmInformerFactory: scmInformerFactory, client: client, tlsConfig: cfg.TLSConfig, - httpHandler: cfg.HTTPHandler, + h: cfg.Handler, listenAddr: cfg.HTTPListenAddr, + workers: cfg.NumWorkers, }, nil } diff --git a/node/nodeutil/provider.go b/node/nodeutil/provider.go index b85b454cb..0f6a72f03 100644 --- a/node/nodeutil/provider.go +++ b/node/nodeutil/provider.go @@ -8,6 +8,7 @@ import ( "github.com/virtual-kubelet/virtual-kubelet/node/api" "github.com/virtual-kubelet/virtual-kubelet/node/api/statsv1alpha1" v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/labels" corev1listers "k8s.io/client-go/listers/core/v1" ) @@ -45,3 +46,25 @@ type ProviderConfig struct { // NewProviderFunc is used from NewNodeFromClient to bootstrap a provider using the client/listers/etc created there. // If a nil node provider is returned a default one will be used. type NewProviderFunc func(ProviderConfig) (Provider, node.NodeProvider, error) + +// AttachProviderRoutes returns a NodeOpt which uses api.PodHandler to attach the routes to the provider functions. +// +// Note this only attaches routes, you'll need to ensure to set the handler in the node config. +func AttachProviderRoutes(mux api.ServeMux) NodeOpt { + return func(cfg *NodeConfig) error { + cfg.routeAttacher = func(p Provider, cfg NodeConfig, pods corev1listers.PodLister) { + mux.Handle("/", api.PodHandler(api.PodHandlerConfig{ + RunInContainer: p.RunInContainer, + GetContainerLogs: p.GetContainerLogs, + GetPods: p.GetPods, + GetPodsFromKubernetes: func(context.Context) ([]*v1.Pod, error) { + return pods.List(labels.Everything()) + }, + GetStatsSummary: p.GetStatsSummary, + StreamIdleTimeout: cfg.StreamIdleTimeout, + StreamCreationTimeout: cfg.StreamCreationTimeout, + }, true)) + } + return nil + } +} diff --git a/node/nodeutil/tls.go b/node/nodeutil/tls.go new file mode 100644 index 000000000..ac29e3f1b --- /dev/null +++ b/node/nodeutil/tls.go @@ -0,0 +1,83 @@ +package nodeutil + +import ( + "crypto/tls" + "crypto/x509" + "fmt" + "io/ioutil" +) + +// WithTLSConfig returns a NodeOpt which creates a base TLSConfig with the default cipher suites and tls min verions. +// The tls config can be modified through functional options. +func WithTLSConfig(opts ...func(*tls.Config) error) NodeOpt { + return func(cfg *NodeConfig) error { + tlsCfg := &tls.Config{ + MinVersion: tls.VersionTLS12, + PreferServerCipherSuites: true, + CipherSuites: DefaultServerCiphers(), + ClientAuth: tls.RequestClientCert, + } + for _, o := range opts { + if err := o(tlsCfg); err != nil { + return err + } + } + + cfg.TLSConfig = tlsCfg + return nil + } +} + +// WithCAFromPath makes a TLS config option to set up client auth using the path to a PEM encoded CA cert. +func WithCAFromPath(p string) func(*tls.Config) error { + return func(cfg *tls.Config) error { + pem, err := ioutil.ReadFile(p) + if err != nil { + return fmt.Errorf("error reading ca cert pem: %w", err) + } + cfg.ClientAuth = tls.RequireAndVerifyClientCert + return WithCACert(pem)(cfg) + } +} + +// WithKeyPairFromPath make sa TLS config option which loads the key pair paths from disk and appends them to the tls config. +func WithKeyPairFromPath(cert, key string) func(*tls.Config) error { + return func(cfg *tls.Config) error { + cert, err := tls.LoadX509KeyPair(cert, key) + if err != nil { + return err + } + cfg.Certificates = append(cfg.Certificates, cert) + return nil + } +} + +// WithCACert makes a TLS config opotion which appends the provided PEM encoded bytes the tls config's cert pool. +// If a cert pool is not defined on the tls config an empty one will be created. +func WithCACert(pem []byte) func(*tls.Config) error { + return func(cfg *tls.Config) error { + if cfg.ClientCAs == nil { + cfg.ClientCAs = x509.NewCertPool() + } + if !cfg.ClientCAs.AppendCertsFromPEM(pem) { + return fmt.Errorf("could not parse ca cert pem") + } + return nil + } +} + +// DefaultServerCiphers is the list of accepted TLS ciphers, with known weak ciphers elided +// Note this list should be a moving target. +func DefaultServerCiphers() []uint16 { + return []uint16{ + tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, + tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, + tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, + tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, + + tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + } +}