Add webhook and anon auth support

Auth is not automatically enabled because this requires some
bootstrapping to work.
I'll leave this for some future work.
In the meantime people can use the current code similar to how they used
the node-cli code to inject their own auth.
This commit is contained in:
Brian Goff
2021-06-08 16:37:29 +00:00
parent e1342777d6
commit 4974e062d0
9 changed files with 483 additions and 101 deletions

View File

@@ -15,45 +15,15 @@
package root
import (
"crypto/tls"
"fmt"
"os"
"time"
"github.com/pkg/errors"
)
// AcceptedCiphers is the list of accepted TLS ciphers, with known weak ciphers elided
// Note this list should be a moving target.
var AcceptedCiphers = []uint16{
tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
}
func loadTLSConfig(certPath, keyPath string) (*tls.Config, error) {
cert, err := tls.LoadX509KeyPair(certPath, keyPath)
if err != nil {
return nil, errors.Wrap(err, "error loading tls certs")
}
return &tls.Config{
Certificates: []tls.Certificate{cert},
MinVersion: tls.VersionTLS12,
PreferServerCipherSuites: true,
CipherSuites: AcceptedCiphers,
}, nil
}
type apiServerConfig struct {
CertPath string
KeyPath string
CACertPath string
Addr string
MetricsAddr string
StreamIdleTimeout time.Duration
@@ -64,6 +34,7 @@ func getAPIConfig(c Opts) (*apiServerConfig, error) {
config := apiServerConfig{
CertPath: os.Getenv("APISERVER_CERT_LOCATION"),
KeyPath: os.Getenv("APISERVER_KEY_LOCATION"),
CACertPath: os.Getenv("APISERVER_CA_CERT_LOCATION"),
}
config.Addr = fmt.Sprintf(":%d", c.ListenPort)

View File

@@ -16,6 +16,8 @@ package root
import (
"context"
"crypto/tls"
"net/http"
"os"
"runtime"
@@ -26,8 +28,11 @@ import (
"github.com/virtual-kubelet/virtual-kubelet/internal/manager"
"github.com/virtual-kubelet/virtual-kubelet/log"
"github.com/virtual-kubelet/virtual-kubelet/node"
"github.com/virtual-kubelet/virtual-kubelet/node/api"
"github.com/virtual-kubelet/virtual-kubelet/node/nodeutil"
corev1 "k8s.io/api/core/v1"
"k8s.io/apiserver/pkg/server/dynamiccertificates"
"k8s.io/client-go/kubernetes"
)
// NewCommand creates a new top-level command.
@@ -74,8 +79,8 @@ func runRootCommand(ctx context.Context, s *provider.Store, c Opts) error {
return err
}
mux := http.NewServeMux()
newProvider := func(cfg nodeutil.ProviderConfig) (nodeutil.Provider, node.NodeProvider, error) {
var err error
rm, err := manager.NewResourceManager(cfg.Pods, cfg.Secrets, cfg.ConfigMaps, cfg.Services)
if err != nil {
return nil, nil, errors.Wrap(err, "could not create resource manager")
@@ -99,11 +104,17 @@ func runRootCommand(ctx context.Context, s *provider.Store, c Opts) error {
return nil, nil, errors.Wrapf(err, "error initializing provider %s", c.Provider)
}
p.ConfigureNode(ctx, cfg.Node)
cfg.Node.Status.NodeInfo.KubeletVersion = c.Version
return p, nil, nil
}
cm, err := nodeutil.NewNodeFromClient(client, c.NodeName, newProvider, func(cfg *nodeutil.NodeConfig) error {
apiConfig, err := getAPIConfig(c)
if err != nil {
return err
}
cm, err := nodeutil.NewNodeFromClient(c.NodeName, newProvider, func(cfg *nodeutil.NodeConfig) error {
cfg.Handler = mux
cfg.InformerResyncPeriod = c.InformerResyncPeriod
if taint != nil {
@@ -112,23 +123,23 @@ func runRootCommand(ctx context.Context, s *provider.Store, c Opts) error {
cfg.NodeSpec.Status.NodeInfo.Architecture = runtime.GOARCH
cfg.NodeSpec.Status.NodeInfo.OperatingSystem = c.OperatingSystem
apiConfig, err := getAPIConfig(c)
if err != nil {
return err
}
cfg.HTTPListenAddr = apiConfig.Addr
cfg.StreamCreationTimeout = apiConfig.StreamCreationTimeout
cfg.StreamIdleTimeout = apiConfig.StreamIdleTimeout
cfg.TLSConfig, err = loadTLSConfig(apiConfig.CertPath, apiConfig.KeyPath)
if err != nil {
return errors.Wrap(err, "error loading tls config")
}
cfg.DebugHTTP = true
cfg.NumWorkers = c.PodSyncWorkers
return nil
})
},
nodeutil.WithClient(client),
setAuth(client, c.NodeName, apiConfig),
nodeutil.WithTLSConfig(
nodeutil.WithKeyPairFromPath(apiConfig.CertPath, apiConfig.KeyPath),
maybeCA(apiConfig.CACertPath),
),
nodeutil.AttachProviderRoutes(mux),
)
if err != nil {
return err
}
@@ -144,7 +155,7 @@ func runRootCommand(ctx context.Context, s *provider.Store, c Opts) error {
"watchedNamespace": c.KubeNamespace,
}))
go cm.Run(ctx, c.PodSyncWorkers) // nolint:errcheck
go cm.Run(ctx) //nolint:errcheck
defer func() {
log.G(ctx).Debug("Waiting for controllers to be done")
@@ -166,3 +177,32 @@ func runRootCommand(ctx context.Context, s *provider.Store, c Opts) error {
}
return nil
}
func setAuth(client kubernetes.Interface, node string, apiCfg *apiServerConfig) nodeutil.NodeOpt {
if apiCfg.CACertPath == "" {
return func(cfg *nodeutil.NodeConfig) error {
cfg.Handler = api.InstrumentHandler(nodeutil.WithAuth(nodeutil.NoAuth(), cfg.Handler))
return nil
}
}
return func(cfg *nodeutil.NodeConfig) error {
auth, err := nodeutil.WebhookAuth(client, node, func(cfg *nodeutil.WebhookAuthConfig) error {
var err error
cfg.AuthnConfig.ClientCertificateCAContentProvider, err = dynamiccertificates.NewDynamicCAContentFromFile("ca-cert-bundle", apiCfg.CACertPath)
return err
})
if err != nil {
return err
}
cfg.Handler = api.InstrumentHandler(nodeutil.WithAuth(auth, cfg.Handler))
return nil
}
}
func maybeCA(p string) func(*tls.Config) error {
if p == "" {
return func(*tls.Config) error { return nil }
}
return nodeutil.WithCAFromPath(p)
}

10
go.sum
View File

@@ -34,8 +34,10 @@ github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb0
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI=
github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M=
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
@@ -55,6 +57,7 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
github.com/blang/semver v3.5.0+incompatible h1:CGxCgetQ64DKk7rdZ++Vfnb1+ogGNnB17OJKJXD2Cfs=
github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
github.com/bombsimon/logrusr v1.0.0 h1:CTCkURYAt5nhCCnKH9eLShYayj2/8Kn/4Qg3QfiU+Ro=
github.com/bombsimon/logrusr v1.0.0/go.mod h1:Jq0nHtvxabKE5EMwAAdgTaz7dfWE8C4i11NOltxGQpc=
@@ -140,11 +143,13 @@ github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+
github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M=
github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M=
github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg=
github.com/go-openapi/jsonpointer v0.19.3 h1:gihV7YNZK1iK6Tgwwsxo2rJbD1GTbdm72325Bq8FI3w=
github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg=
github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I=
github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I=
github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc=
github.com/go-openapi/jsonreference v0.19.3 h1:5cxNfTy0UVC3X8JL5ymxzyoUZmo8iZb+jeTWn7tUa8o=
github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8=
github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU=
github.com/go-openapi/loads v0.18.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU=
@@ -158,6 +163,7 @@ github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nA
github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI=
github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI=
github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY=
github.com/go-openapi/spec v0.19.3 h1:0XRyw8kguri6Yw4SxhsQA/atC88yqrk0+G4YhI2wabc=
github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo=
github.com/go-openapi/strfmt v0.17.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU=
github.com/go-openapi/strfmt v0.18.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU=
@@ -167,6 +173,7 @@ github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dp
github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg=
github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg=
github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY=
github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4=
github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA=
@@ -281,6 +288,7 @@ github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN
github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM=
github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
@@ -371,6 +379,7 @@ github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrf
github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
@@ -704,6 +713,7 @@ k8s.io/utils v0.0.0-20200912215256-4140de9c8800 h1:9ZNvfPvVIEsp/T1ez4GQuzCcCTEQW
k8s.io/utils v0.0.0-20200912215256-4140de9c8800/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.9/go.mod h1:dzAXnQbTRyDlZPJX2SUPEqvnB+j7AJjtlox7PEwigU0=
sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15 h1:4uqm9Mv+w2MmBYD+F4qf/v6tDFUdPOk29C095RbU5mY=
sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg=
sigs.k8s.io/controller-runtime v0.7.1 h1:nqVwzVzdenfd9xIbB35pC7JJH2IXVL4hDo3MNzkyCh4=
sigs.k8s.io/controller-runtime v0.7.1/go.mod h1:pJ3YBrJiAqMAZKi6UVGuE98ZrroV1p+pIhoHsMm9wdU=

View File

@@ -56,6 +56,14 @@ rules:
verbs:
- create
- patch
- apiGroups:
- coordination.k8s.io
resources:
- leases
verbs:
- get
- create
- update
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding

View File

@@ -4,6 +4,8 @@ metadata:
name: vkubelet-mock-0
spec:
containers:
- name: jaeger-tracing
image: jaegertracing/all-in-one:1.22
- name: vkubelet-mock-0
image: virtual-kubelet
# "IfNotPresent" is used to prevent Minikube from trying to pull from the registry (and failing) in the first place.
@@ -23,7 +25,12 @@ spec:
- --klog.logtostderr
- --log-level
- debug
- --trace-exporter
- jaeger
- --trace-sample-rate=always
env:
- name: JAEGER_AGENT_ENDPOINT
value: localhost:6831
- name: KUBELET_PORT
value: "10250"
- name: VKUBELET_POD_IP

220
node/nodeutil/auth.go Normal file
View File

@@ -0,0 +1,220 @@
package nodeutil
import (
"context"
"net/http"
"strings"
"time"
"github.com/virtual-kubelet/virtual-kubelet/log"
"github.com/virtual-kubelet/virtual-kubelet/trace"
"k8s.io/apiserver/pkg/authentication/authenticator"
"k8s.io/apiserver/pkg/authentication/authenticatorfactory"
"k8s.io/apiserver/pkg/authentication/request/anonymous"
"k8s.io/apiserver/pkg/authentication/user"
"k8s.io/apiserver/pkg/authorization/authorizer"
"k8s.io/apiserver/pkg/authorization/authorizerfactory"
"k8s.io/client-go/kubernetes"
)
// Auth is the interface used to implement authn/authz for http requests
type Auth interface {
authenticator.Request
authorizer.RequestAttributesGetter
authorizer.Authorizer
}
type authWrapper struct {
authenticator.Request
authorizer.RequestAttributesGetter
authorizer.Authorizer
}
// InstrumentAuth wraps the provided Auth in a new instrumented Auth
//
// Note: You would only need this if you rolled your own auth.
// The Auth implementations defined in this package are already instrumented.
func InstrumentAuth(auth Auth) Auth {
if _, ok := auth.(*authWrapper); ok {
// This is already instrumented
return auth
}
return &authWrapper{
Request: auth,
RequestAttributesGetter: auth,
Authorizer: auth,
}
}
// NoAuth creates an Auth which allows anonymous access to all resouorces
func NoAuth() Auth {
return &authWrapper{
Request: anonymous.NewAuthenticator(),
RequestAttributesGetter: &NodeRequestAttr{},
Authorizer: authorizerfactory.NewAlwaysAllowAuthorizer(),
}
}
// WithAuth makes a new http handler which wraps the provided handler with authn/authz.
func WithAuth(auth Auth, h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
handleAuth(auth, w, r, h)
})
}
func handleAuth(auth Auth, w http.ResponseWriter, r *http.Request, next http.Handler) {
ctx := r.Context()
ctx, span := trace.StartSpan(ctx, "vk.handleAuth")
defer span.End()
r = r.WithContext(ctx)
info, ok, err := auth.AuthenticateRequest(r)
if err != nil || !ok {
log.G(r.Context()).WithError(err).Error("Authorization error")
http.Error(w, "Unauthorized", http.StatusUnauthorized)
return
}
logger := log.G(ctx).WithFields(log.Fields{
"user-name": info.User.GetName(),
"user-id": info.User.GetUID(),
})
ctx = log.WithLogger(ctx, logger)
r = r.WithContext(ctx)
attrs := auth.GetRequestAttributes(info.User, r)
decision, _, err := auth.Authorize(ctx, attrs)
if err != nil {
log.G(r.Context()).WithError(err).Error("Authorization error")
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
if decision != authorizer.DecisionAllow {
http.Error(w, "Forbidden", http.StatusForbidden)
return
}
next.ServeHTTP(w, r)
}
// WebhookAuthOption is used as a functional argument to configure webhook auth.
type WebhookAuthOption func(*WebhookAuthConfig) error
// WebhookAuthConfig stores the configurations for authn/authz and is used by WebhookAuthOption to expose to callers.
type WebhookAuthConfig struct {
AuthnConfig authenticatorfactory.DelegatingAuthenticatorConfig
AuthzConfig authorizerfactory.DelegatingAuthorizerConfig
}
// WebhookAuth creates an Auth suitable to use with kubelet webhook auth.
// You must provide a CA provider to the authentication config, otherwise mTLS is disabled.
func WebhookAuth(client kubernetes.Interface, nodeName string, opts ...WebhookAuthOption) (Auth, error) {
cfg := WebhookAuthConfig{
AuthnConfig: authenticatorfactory.DelegatingAuthenticatorConfig{
CacheTTL: 2 * time.Minute, // default taken from k8s.io/kubernetes/pkg/kubelet/apis/config/v1beta1
// TODO: After upgrading k8s libs, we need to add the retry backoff option
},
AuthzConfig: authorizerfactory.DelegatingAuthorizerConfig{
AllowCacheTTL: 5 * time.Minute, // default taken from k8s.io/kubernetes/pkg/kubelet/apis/config/v1beta1
DenyCacheTTL: 30 * time.Second, // default taken from k8s.io/kubernetes/pkg/kubelet/apis/config/v1beta1
// TODO: After upgrading k8s libs, we need to add the retry backoff option
},
}
for _, o := range opts {
if err := o(&cfg); err != nil {
return nil, err
}
}
cfg.AuthnConfig.TokenAccessReviewClient = client.AuthenticationV1().TokenReviews()
cfg.AuthzConfig.SubjectAccessReviewClient = client.AuthorizationV1().SubjectAccessReviews()
authn, _, err := cfg.AuthnConfig.New()
if err != nil {
return nil, err
}
authz, err := cfg.AuthzConfig.New()
if err != nil {
return nil, err
}
return &authWrapper{
Request: authn,
RequestAttributesGetter: NodeRequestAttr{nodeName},
Authorizer: authz,
}, nil
}
func (w *authWrapper) AuthenticateRequest(r *http.Request) (*authenticator.Response, bool, error) {
ctx, span := trace.StartSpan(r.Context(), "AuthenticateRequest")
defer span.End()
return w.Request.AuthenticateRequest(r.WithContext(ctx))
}
func (w *authWrapper) Authorize(ctx context.Context, a authorizer.Attributes) (authorizer.Decision, string, error) {
ctx, span := trace.StartSpan(ctx, "Authorize")
defer span.End()
return w.Authorizer.Authorize(ctx, a)
}
// NodeRequestAttr is a authorizor.RequeestAttributesGetter which can be used in the Auth interface.
type NodeRequestAttr struct {
NodeName string
}
// GetRequestAttributes satisfies the authorizer.RequestAttributesGetter interface for use with an `Auth`.
func (a NodeRequestAttr) GetRequestAttributes(u user.Info, r *http.Request) authorizer.Attributes {
return authorizer.AttributesRecord{
User: u,
Verb: getAPIVerb(r),
Namespace: "",
APIGroup: "",
APIVersion: "v1",
Resource: "nodes",
Name: a.NodeName,
ResourceRequest: true,
Path: r.URL.Path,
Subresource: getSubresource(r),
}
}
func getAPIVerb(r *http.Request) string {
switch r.Method {
case http.MethodPost:
return "create"
case http.MethodGet:
return "get"
case http.MethodPut:
return "update"
case http.MethodPatch:
return "patch"
case http.MethodDelete:
return "delete"
}
return ""
}
func isSubpath(subpath, path string) bool {
// Taken from k8s.io/kubernetes/pkg/kubelet/server/auth.go
return subpath == path || (strings.HasPrefix(subpath, path) && subpath[len(path)] == '/')
}
func getSubresource(r *http.Request) string {
if isSubpath(r.URL.Path, "/stats") {
return "stats"
}
if isSubpath(r.URL.Path, "/metrics") {
return "metrics"
}
if isSubpath(r.URL.Path, "/logs") {
// yes, "log", not "logs"
// per kubelet code: "log" to match other log subresources (pods/log, etc)
return "log"
}
return "proxy"
}

View File

@@ -8,19 +8,19 @@ import (
"net/http"
"os"
"path"
"runtime"
"time"
"github.com/pkg/errors"
"github.com/virtual-kubelet/virtual-kubelet/log"
"github.com/virtual-kubelet/virtual-kubelet/node"
"github.com/virtual-kubelet/virtual-kubelet/node/api"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/kubernetes/scheme"
corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
corev1listers "k8s.io/client-go/listers/core/v1"
"k8s.io/client-go/tools/record"
)
@@ -43,9 +43,11 @@ type Node struct {
client kubernetes.Interface
listenAddr string
httpHandler HTTPHandler
h http.Handler
tlsConfig *tls.Config
workers int
eb record.EventBroadcaster
}
@@ -59,8 +61,35 @@ func (n *Node) PodController() *node.PodController {
return n.pc
}
func (n *Node) runHTTP(ctx context.Context) (func(), error) {
if n.tlsConfig == nil {
log.G(ctx).Warn("TLS config not provided, not starting up http service")
return func() {}, nil
}
if n.h == nil {
log.G(ctx).Debug("No http handler, not starting up http service")
return func() {}, nil
}
l, err := tls.Listen("tcp", n.listenAddr, n.tlsConfig)
if err != nil {
return nil, errors.Wrap(err, "error starting http listener")
}
log.G(ctx).Debug("Started TLS listener")
srv := &http.Server{Handler: n.h, TLSConfig: n.tlsConfig}
go srv.Serve(l) //nolint:errcheck
log.G(ctx).Debug("HTTP server running")
return func() {
srv.Close()
l.Close()
}, nil
}
// Run starts all the underlying controllers
func (n *Node) Run(ctx context.Context, workers int) (retErr error) {
func (n *Node) Run(ctx context.Context) (retErr error) {
ctx, cancel := context.WithCancel(ctx)
defer func() {
cancel()
@@ -69,31 +98,22 @@ func (n *Node) Run(ctx context.Context, workers int) (retErr error) {
close(n.done)
}()
if n.podInformerFactory != nil {
go n.podInformerFactory.Start(ctx.Done())
}
if n.scmInformerFactory != nil {
go n.scmInformerFactory.Start(ctx.Done())
}
if n.eb != nil {
n.eb.StartLogging(log.G(ctx).Infof)
n.eb.StartRecordingToSink(&corev1client.EventSinkImpl{Interface: n.client.CoreV1().Events(v1.NamespaceAll)})
defer n.eb.Shutdown()
log.G(ctx).Debug("Started event broadcaster")
}
l, err := tls.Listen("tcp", n.listenAddr, n.tlsConfig)
cancelHTTP, err := n.runHTTP(ctx)
if err != nil {
return errors.Wrap(err, "error starting http listener")
return err
}
log.G(ctx).Debug("Started TLS listener")
defer l.Close()
defer cancelHTTP()
srv := &http.Server{Handler: n.httpHandler, TLSConfig: n.tlsConfig}
go srv.Serve(l)
defer srv.Close()
go n.pc.Run(ctx, workers) //nolint:errcheck
log.G(ctx).Debug("HTTP server running")
go n.podInformerFactory.Start(ctx.Done())
go n.scmInformerFactory.Start(ctx.Done())
go n.pc.Run(ctx, n.workers) //nolint:errcheck
defer func() {
cancel()
@@ -193,6 +213,9 @@ type NodeOpt func(c *NodeConfig) error
// NodeConfig is used to hold configuration items for a Node.
// It gets used in conjection with NodeOpt in NewNodeFromClient
type NodeConfig struct {
// Set the client to use, otherwise a client will be created from ClientsetFromEnv
Client kubernetes.Interface
// Set the node spec to register with Kubernetes
NodeSpec v1.Node
// Set the path to read a kubeconfig from for creating a client.
@@ -206,8 +229,9 @@ type NodeConfig struct {
// Set a custom API handler to use.
// You can use this to setup, for example, authentication middleware.
// If one is not provided a default one will be created.
// Pod routes will be attached to this handler when creating the node
HTTPHandler HTTPHandler
//
// Note: If you provide your own handler, you'll need to handle all auth, routes, etc.
Handler http.Handler
// Set the timeout for idle http streams
StreamIdleTimeout time.Duration
// Set the timeout for creating http streams
@@ -220,11 +244,12 @@ type NodeConfig struct {
// Specify the event recorder to use
// If this is not provided, a default one will be used.
EventRecorder record.EventRecorder
}
type HTTPHandler interface {
api.ServeMux
http.Handler
// Set the number of workers to reconcile pods
// The default value is derived from the number of cores available.
NumWorkers int
routeAttacher func(Provider, NodeConfig, corev1listers.PodLister)
}
// WithNodeConfig returns a NodeOpt which replaces the NodeConfig with the passed in value.
@@ -235,9 +260,12 @@ func WithNodeConfig(c NodeConfig) NodeOpt {
}
}
// NewNode calls NewNodeFromClient with a nil client
func NewNode(name string, newProvider NewProviderFunc, opts ...NodeOpt) (*Node, error) {
return NewNodeFromClient(nil, name, newProvider, opts...)
// WithClient return a NodeOpt that sets the client that will be used to create/manage the node.
func WithClient(c kubernetes.Interface) NodeOpt {
return func(cfg *NodeConfig) error {
cfg.Client = c
return nil
}
}
// NewNodeFromClient creates a new node using the provided client and name.
@@ -247,9 +275,11 @@ func NewNode(name string, newProvider NewProviderFunc, opts ...NodeOpt) (*Node,
// Some basic values are set for node status, you'll almost certainly want to modify it.
//
// If client is nil, this will construct a client using ClientsetFromEnv
func NewNodeFromClient(client kubernetes.Interface, name string, newProvider NewProviderFunc, opts ...NodeOpt) (*Node, error) {
//
// It is up to the caller to configure auth on the HTTP handler.
func NewNodeFromClient(name string, newProvider NewProviderFunc, opts ...NodeOpt) (*Node, error) {
cfg := NodeConfig{
// TODO: this is what was set in the cli code... its not clear what a good value actually is.
NumWorkers: runtime.NumCPU(),
InformerResyncPeriod: time.Minute,
KubeconfigPath: os.Getenv("KUBECONFIG"),
HTTPListenAddr: ":10250",
@@ -285,10 +315,7 @@ func NewNodeFromClient(client kubernetes.Interface, name string, newProvider New
return nil, errors.Wrap(err, "error parsing http listen address")
}
if cfg.HTTPHandler == nil {
cfg.HTTPHandler = http.NewServeMux()
}
client := cfg.Client
if client == nil {
var err error
client, err = ClientsetFromEnv(cfg.KubeconfigPath)
@@ -324,17 +351,9 @@ func NewNodeFromClient(client kubernetes.Interface, name string, newProvider New
return nil, errors.Wrap(err, "error creating provider")
}
api.AttachPodRoutes(api.PodHandlerConfig{
RunInContainer: p.RunInContainer,
GetContainerLogs: p.GetContainerLogs,
GetPods: p.GetPods,
GetPodsFromKubernetes: func(context.Context) ([]*v1.Pod, error) {
return podInformer.Lister().List(labels.Everything())
},
GetStatsSummary: p.GetStatsSummary,
StreamIdleTimeout: cfg.StreamIdleTimeout,
StreamCreationTimeout: cfg.StreamCreationTimeout,
}, cfg.HTTPHandler, cfg.DebugHTTP)
if cfg.routeAttacher != nil {
cfg.routeAttacher(p, cfg, podInformer.Lister())
}
var readyCb func(context.Context) error
if np == nil {
@@ -360,7 +379,7 @@ func NewNodeFromClient(client kubernetes.Interface, name string, newProvider New
var eb record.EventBroadcaster
if cfg.EventRecorder == nil {
eb := record.NewBroadcaster()
eb = record.NewBroadcaster()
cfg.EventRecorder = eb.NewRecorder(scheme.Scheme, v1.EventSource{Component: path.Join(name, "pod-controller")})
}
@@ -388,8 +407,9 @@ func NewNodeFromClient(client kubernetes.Interface, name string, newProvider New
scmInformerFactory: scmInformerFactory,
client: client,
tlsConfig: cfg.TLSConfig,
httpHandler: cfg.HTTPHandler,
h: cfg.Handler,
listenAddr: cfg.HTTPListenAddr,
workers: cfg.NumWorkers,
}, nil
}

View File

@@ -8,6 +8,7 @@ import (
"github.com/virtual-kubelet/virtual-kubelet/node/api"
"github.com/virtual-kubelet/virtual-kubelet/node/api/statsv1alpha1"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/labels"
corev1listers "k8s.io/client-go/listers/core/v1"
)
@@ -45,3 +46,25 @@ type ProviderConfig struct {
// NewProviderFunc is used from NewNodeFromClient to bootstrap a provider using the client/listers/etc created there.
// If a nil node provider is returned a default one will be used.
type NewProviderFunc func(ProviderConfig) (Provider, node.NodeProvider, error)
// AttachProviderRoutes returns a NodeOpt which uses api.PodHandler to attach the routes to the provider functions.
//
// Note this only attaches routes, you'll need to ensure to set the handler in the node config.
func AttachProviderRoutes(mux api.ServeMux) NodeOpt {
return func(cfg *NodeConfig) error {
cfg.routeAttacher = func(p Provider, cfg NodeConfig, pods corev1listers.PodLister) {
mux.Handle("/", api.PodHandler(api.PodHandlerConfig{
RunInContainer: p.RunInContainer,
GetContainerLogs: p.GetContainerLogs,
GetPods: p.GetPods,
GetPodsFromKubernetes: func(context.Context) ([]*v1.Pod, error) {
return pods.List(labels.Everything())
},
GetStatsSummary: p.GetStatsSummary,
StreamIdleTimeout: cfg.StreamIdleTimeout,
StreamCreationTimeout: cfg.StreamCreationTimeout,
}, true))
}
return nil
}
}

83
node/nodeutil/tls.go Normal file
View File

@@ -0,0 +1,83 @@
package nodeutil
import (
"crypto/tls"
"crypto/x509"
"fmt"
"io/ioutil"
)
// WithTLSConfig returns a NodeOpt which creates a base TLSConfig with the default cipher suites and tls min verions.
// The tls config can be modified through functional options.
func WithTLSConfig(opts ...func(*tls.Config) error) NodeOpt {
return func(cfg *NodeConfig) error {
tlsCfg := &tls.Config{
MinVersion: tls.VersionTLS12,
PreferServerCipherSuites: true,
CipherSuites: DefaultServerCiphers(),
ClientAuth: tls.RequestClientCert,
}
for _, o := range opts {
if err := o(tlsCfg); err != nil {
return err
}
}
cfg.TLSConfig = tlsCfg
return nil
}
}
// WithCAFromPath makes a TLS config option to set up client auth using the path to a PEM encoded CA cert.
func WithCAFromPath(p string) func(*tls.Config) error {
return func(cfg *tls.Config) error {
pem, err := ioutil.ReadFile(p)
if err != nil {
return fmt.Errorf("error reading ca cert pem: %w", err)
}
cfg.ClientAuth = tls.RequireAndVerifyClientCert
return WithCACert(pem)(cfg)
}
}
// WithKeyPairFromPath make sa TLS config option which loads the key pair paths from disk and appends them to the tls config.
func WithKeyPairFromPath(cert, key string) func(*tls.Config) error {
return func(cfg *tls.Config) error {
cert, err := tls.LoadX509KeyPair(cert, key)
if err != nil {
return err
}
cfg.Certificates = append(cfg.Certificates, cert)
return nil
}
}
// WithCACert makes a TLS config opotion which appends the provided PEM encoded bytes the tls config's cert pool.
// If a cert pool is not defined on the tls config an empty one will be created.
func WithCACert(pem []byte) func(*tls.Config) error {
return func(cfg *tls.Config) error {
if cfg.ClientCAs == nil {
cfg.ClientCAs = x509.NewCertPool()
}
if !cfg.ClientCAs.AppendCertsFromPEM(pem) {
return fmt.Errorf("could not parse ca cert pem")
}
return nil
}
}
// DefaultServerCiphers is the list of accepted TLS ciphers, with known weak ciphers elided
// Note this list should be a moving target.
func DefaultServerCiphers() []uint16 {
return []uint16{
tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
}
}