d8dfun/virtual-kubelet
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Decouple vkubelet/* packages from providers (#626)

Browse Source 
This makes the concept of a `Provider` wholely implemented in the cli
implementation in cmd/virtual-kubelet.

It allows us to slim down the interfaces used in vkubelet (and
vkubelet/api) to what is actually used there rather than a huge
interface that is only there to serve the CLI's needs.
This commit is contained in:
Brian Goff
2019-05-17 17:01:05 -07:00
committed by GitHub
parent 87e72bf4df
commit 7dd49516d8
26 changed files with 217 additions and 194 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
vkubelet/api/exec.go
View File

@@ -10,21 +10,38 @@ import (
"github.com/cpuguy83/strongerrors"
"github.com/gorilla/mux"
"github.com/pkg/errors"
"github.com/virtual-kubelet/virtual-kubelet/providers"
"k8s.io/apimachinery/pkg/types"
remoteutils "k8s.io/client-go/tools/remotecommand"
api "k8s.io/kubernetes/pkg/apis/core"
"k8s.io/kubernetes/pkg/kubelet/server/remotecommand"
)
type ExecBackend interface {
RunInContainer(ctx context.Context, namespace, podName, containerName string, cmd []string, attach providers.AttachIO) error
// ContainerExecHandlerFunc defines the handler function used for "execing" into a
// container in a pod.
type ContainerExecHandlerFunc func(ctx context.Context, namespace, podName, containerName string, cmd []string, attach AttachIO) error
// AttachIO is used to pass in streams to attach to a container process
type AttachIO interface {
Stdin() io.Reader
Stdout() io.WriteCloser
Stderr() io.WriteCloser
TTY() bool
Resize() <-chan TermSize
}
// PodExecHandlerFunc makes an http handler func from a Provider which execs a command in a pod's container
// TermSize is used to set the terminal size from attached clients.
type TermSize struct {
Width uint16
Height uint16
}
// HandleContainerExec makes an http handler func from a Provider which execs a command in a pod's container
// Note that this handler currently depends on gorrilla/mux to get url parts as variables.
// TODO(@cpuguy83): don't force gorilla/mux on consumers of this function
func PodExecHandlerFunc(backend ExecBackend) http.HandlerFunc {
func HandleContainerExec(h ContainerExecHandlerFunc) http.HandlerFunc {
if h == nil {
return NotImplemented
}
return handleError(func(w http.ResponseWriter, req *http.Request) error {
vars := mux.Vars(req)
@@ -48,7 +65,7 @@ func PodExecHandlerFunc(backend ExecBackend) http.HandlerFunc {
ctx, cancel := context.WithCancel(context.TODO())
defer cancel()
exec := &containerExecContext{ctx: ctx, b: backend, pod: pod, namespace: namespace, container: container}
exec := &containerExecContext{ctx: ctx, h: h, pod: pod, namespace: namespace, container: container}
remotecommand.ServeExec(w, req, exec, "", "", container, command, streamOpts, idleTimeout, streamCreationTimeout, supportedStreamProtocols)
return nil
@@ -77,7 +94,7 @@ func getExecOptions(req *http.Request) (*remotecommand.Options, error) {
}
type containerExecContext struct {
b ExecBackend
h ContainerExecHandlerFunc
eio *execIO
namespace, pod, container string
ctx context.Context
@@ -95,7 +112,7 @@ func (c *containerExecContext) ExecInContainer(name string, uid types.UID, conta
}
if tty {
eio.chResize = make(chan providers.TermSize)
eio.chResize = make(chan TermSize)
}
ctx, cancel := context.WithCancel(c.ctx)
@@ -105,7 +122,7 @@ func (c *containerExecContext) ExecInContainer(name string, uid types.UID, conta
go func() {
send := func(s remoteutils.TerminalSize) bool {
select {
case eio.chResize <- providers.TermSize{Width: s.Width, Height: s.Height}:
case eio.chResize <- TermSize{Width: s.Width, Height: s.Height}:
return false
case <-ctx.Done():
return true
@@ -125,7 +142,7 @@ func (c *containerExecContext) ExecInContainer(name string, uid types.UID, conta
}()
}
return c.b.RunInContainer(c.ctx, c.namespace, c.pod, c.container, cmd, eio)
return c.h(c.ctx, c.namespace, c.pod, c.container, cmd, eio)
}
type execIO struct {
@@ -133,7 +150,7 @@ type execIO struct {
stdin io.Reader
stdout io.WriteCloser
stderr io.WriteCloser
chResize chan providers.TermSize
chResize chan TermSize
}
func (e *execIO) TTY() bool {
@@ -152,6 +169,6 @@ func (e *execIO) Stderr() io.WriteCloser {
return e.stderr
}
func (e *execIO) Resize() <-chan providers.TermSize {
func (e *execIO) Resize() <-chan TermSize {
return e.chResize
}

26
vkubelet/api/logs.go
View File

@@ -5,21 +5,31 @@ import (
"io"
"net/http"
"strconv"
"time"
"github.com/cpuguy83/strongerrors"
"github.com/gorilla/mux"
"github.com/pkg/errors"
"github.com/virtual-kubelet/virtual-kubelet/log"
"github.com/virtual-kubelet/virtual-kubelet/providers"
)
// ContainerLogsBackend is used in place of backend implementations for getting container logs
type ContainerLogsBackend interface {
GetContainerLogs(ctx context.Context, namespace, podName, containerName string, opts providers.ContainerLogOpts) (io.ReadCloser, error)
// ContainerLogsHandlerFunc is used in place of backend implementations for getting container logs
type ContainerLogsHandlerFunc func(ctx context.Context, namespace, podName, containerName string, opts ContainerLogOpts) (io.ReadCloser, error)
// ContainerLogOpts are used to pass along options to be set on the container
// log stream.
type ContainerLogOpts struct {
Tail int
Since time.Duration
LimitBytes int
Timestamps bool
}
// PodLogsHandlerFunc creates an http handler function from a provider to serve logs from a pod
func PodLogsHandlerFunc(p ContainerLogsBackend) http.HandlerFunc {
// HandleContainerLogs creates an http handler function from a provider to serve logs from a pod
func HandleContainerLogs(h ContainerLogsHandlerFunc) http.HandlerFunc {
if h == nil {
return NotImplemented
}
return handleError(func(w http.ResponseWriter, req *http.Request) error {
vars := mux.Vars(req)
if len(vars) != 3 {
@@ -45,11 +55,11 @@ func PodLogsHandlerFunc(p ContainerLogsBackend) http.HandlerFunc {
// TODO(@cpuguy83): support v1.PodLogOptions
// The kubelet decoding here is not straight forward, so this needs to be disected
opts := providers.ContainerLogOpts{
opts := ContainerLogOpts{
Tail: tail,
}
logs, err := p.GetContainerLogs(ctx, namespace, pod, container, opts)
logs, err := h(ctx, namespace, pod, container, opts)
if err != nil {
return errors.Wrap(err, "error getting container logs?)")
}

12
vkubelet/api/pods.go
View File

@@ -6,18 +6,14 @@ import (
"github.com/cpuguy83/strongerrors"
"github.com/virtual-kubelet/virtual-kubelet/log"
"k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/serializer"
)
// ContainerLogsBackend is used in place of backend implementations for the provider's pods
type RunningPodsBackend interface {
// GetPods retrieves a list of all pods running on the provider (can be cached).
GetPods(context.Context) ([]*v1.Pod, error)
}
type PodListerFunc func(context.Context) ([]*v1.Pod, error)
func RunningPodsHandlerFunc(p RunningPodsBackend) http.HandlerFunc {
func HandleRunningPods(getPods PodListerFunc) http.HandlerFunc {
scheme := runtime.NewScheme()
v1.SchemeBuilder.AddToScheme(scheme)
codecs := serializer.NewCodecFactory(scheme)
@@ -25,7 +21,7 @@ func RunningPodsHandlerFunc(p RunningPodsBackend) http.HandlerFunc {
return handleError(func(w http.ResponseWriter, req *http.Request) error {
ctx := req.Context()
ctx = log.WithLogger(ctx, log.L)
pods, err := p.GetPods(ctx)
pods, err := getPods(ctx)
if err != nil {
return err
}

56
vkubelet/apiserver.go → vkubelet/api/server.go
View File

@@ -1,12 +1,10 @@
package vkubelet
package api
import (
"net/http"
"github.com/gorilla/mux"
"github.com/virtual-kubelet/virtual-kubelet/log"
"github.com/virtual-kubelet/virtual-kubelet/providers"
"github.com/virtual-kubelet/virtual-kubelet/vkubelet/api"
"go.opencensus.io/plugin/ochttp"
"go.opencensus.io/plugin/ochttp/propagation/b3"
)
@@ -20,37 +18,40 @@ type ServeMux interface {
Handle(path string, h http.Handler)
}
type PodHandlerConfig struct {
RunInContainer ContainerExecHandlerFunc
GetContainerLogs ContainerLogsHandlerFunc
GetPods PodListerFunc
}
// PodHandler creates an http handler for interacting with pods/containers.
func PodHandler(p providers.Provider, debug bool) http.Handler {
func PodHandler(p PodHandlerConfig, debug bool) http.Handler {
r := mux.NewRouter()
// This matches the behaviour in the reference kubelet
r.StrictSlash(true)
if debug {
r.HandleFunc("/runningpods/", api.RunningPodsHandlerFunc(p)).Methods("GET")
r.HandleFunc("/runningpods/", HandleRunningPods(p.GetPods)).Methods("GET")
}
r.HandleFunc("/containerLogs/{namespace}/{pod}/{container}", api.PodLogsHandlerFunc(p)).Methods("GET")
r.HandleFunc("/exec/{namespace}/{pod}/{container}", api.PodExecHandlerFunc(p)).Methods("POST")
r.HandleFunc("/containerLogs/{namespace}/{pod}/{container}", HandleContainerLogs(p.GetContainerLogs)).Methods("GET")
r.HandleFunc("/exec/{namespace}/{pod}/{container}", HandleContainerExec(p.RunInContainer)).Methods("POST")
r.NotFoundHandler = http.HandlerFunc(NotFound)
return r
}
// MetricsSummaryHandler creates an http handler for serving pod metrics.
// PodStatsSummaryHandler creates an http handler for serving pod metrics.
//
// If the passed in provider does not implement providers.PodMetricsProvider,
// it will create handlers that just serves http.StatusNotImplemented
func MetricsSummaryHandler(p providers.Provider) http.Handler {
// If the passed in handler func is nil this will create handlers which only
// serves http.StatusNotImplemented
func PodStatsSummaryHandler(f PodStatsSummaryHandlerFunc) http.Handler {
if f == nil {
return http.HandlerFunc(NotImplemented)
}
r := mux.NewRouter()
const summaryRoute = "/stats/summary"
var h http.HandlerFunc
mp, ok := p.(providers.PodMetricsProvider)
if !ok {
h = NotImplemented
} else {
h = api.PodMetricsHandlerFunc(mp)
}
h := HandlePodStatsSummary(f)
r.Handle(summaryRoute, ochttp.WithRouteTag(h, "PodStatsSummaryHandler")).Methods("GET")
r.Handle(summaryRoute+"/", ochttp.WithRouteTag(h, "PodStatsSummaryHandler")).Methods("GET")
@@ -63,16 +64,25 @@ func MetricsSummaryHandler(p providers.Provider) http.Handler {
//
// Callers should take care to namespace the serve mux as they see fit, however
// these routes get called by the Kubernetes API server.
func AttachPodRoutes(p providers.Provider, mux ServeMux, debug bool) {
func AttachPodRoutes(p PodHandlerConfig, mux ServeMux, debug bool) {
mux.Handle("/", InstrumentHandler(PodHandler(p, debug)))
}
// AttachMetricsRoutes adds the http routes for pod/node metrics to the passed in serve mux.
// PodMetricsConfig stores the handlers for pod metrics routes
// It is used by AttachPodMetrics.
//
// The main reason for this struct is in case of expansion we do not need to break
// the package level API.
type PodMetricsConfig struct {
GetStatsSummary PodStatsSummaryHandlerFunc
}
// AttachPodMetricsRoutes adds the http routes for pod/node metrics to the passed in serve mux.
//
// Callers should take care to namespace the serve mux as they see fit, however
// these routes get called by the Kubernetes API server.
func AttachMetricsRoutes(p providers.Provider, mux ServeMux) {
mux.Handle("/", InstrumentHandler(MetricsSummaryHandler(p)))
func AttachPodMetricsRoutes(p PodMetricsConfig, mux ServeMux) {
mux.Handle("/", InstrumentHandler(HandlePodStatsSummary(p.GetStatsSummary)))
}
func instrumentRequest(r *http.Request) *http.Request {

15
vkubelet/api/stats.go
View File

@@ -10,15 +10,16 @@ import (
stats "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
)
// PodMetricsBackend is used in place of backend implementations to get k8s pod metrics.
type PodMetricsBackend interface {
GetStatsSummary(context.Context) (*stats.Summary, error)
}
// PodStatsSummaryHandlerFunc defines the handler for getting pod stats summaries
type PodStatsSummaryHandlerFunc func(context.Context) (*stats.Summary, error)
// PodMetricsHandlerFunc makes an HTTP handler for implementing the kubelet summary stats endpoint
func PodMetricsHandlerFunc(b PodMetricsBackend) http.HandlerFunc {
// HandlePodStatsSummary makes an HTTP handler for implementing the kubelet summary stats endpoint
func HandlePodStatsSummary(h PodStatsSummaryHandlerFunc) http.HandlerFunc {
if h == nil {
return NotImplemented
}
return handleError(func(w http.ResponseWriter, req *http.Request) error {
stats, err := b.GetStatsSummary(req.Context())
stats, err := h(req.Context())
if err != nil {
if errors.Cause(err) == context.Canceled {
return strongerrors.Cancelled(err)

6
vkubelet/doc.go
View File

@@ -19,11 +19,11 @@ controller.
Up to this point you have a running virtual kubelet controller, but no HTTP
handlers to deal with requests forwarded from the API server for things like
pod logs (e.g. user calls `kubectl logs myVKPod`). This package provides some
helpers for this: `AttachPodRoutes` and `AttachMetricsRoutes`.
pod logs (e.g. user calls `kubectl logs myVKPod`). The api package provides some
helpers for this: `api.AttachPodRoutes` and `api.AttachMetricsRoutes`.
mux := http.NewServeMux()
vkubelet.AttachPodRoutes(provider, mux)
api.AttachPodRoutes(provider, mux)
You must configure your own HTTP server, but these helpers will add handlers at
the correct URI paths to your serve mux. You are not required to use go's

26
vkubelet/node.go
View File

@@ -9,7 +9,6 @@ import (
"github.com/cpuguy83/strongerrors/status/ocstatus"
pkgerrors "github.com/pkg/errors"
"github.com/virtual-kubelet/virtual-kubelet/log"
"github.com/virtual-kubelet/virtual-kubelet/providers"
"github.com/virtual-kubelet/virtual-kubelet/trace"
coord "k8s.io/api/coordination/v1beta1"
corev1 "k8s.io/api/core/v1"
@@ -21,12 +20,33 @@ import (
v1 "k8s.io/client-go/kubernetes/typed/core/v1"
)
// NodeProvider is the interface used for registering a node and updating its
// status in Kubernetes.
//
// Note: Implementers can choose to manage a node themselves, in which case
// it is not needed to provide an implementation for this interface.
type NodeProvider interface {
// Ping checks if the node is still active.
// This is intended to be lightweight as it will be called periodically as a
// heartbeat to keep the node marked as ready in Kubernetes.
Ping(context.Context) error
// NotifyNodeStatus is used to asynchronously monitor the node.
// The passed in callback should be called any time there is a change to the
// node's status.
// This will generally trigger a call to the Kubernetes API server to update
// the status.
//
// NotifyNodeStatus should not block callers.
NotifyNodeStatus(ctx context.Context, cb func(*corev1.Node))
}
// NewNode creates a new node.
// This does not have any side-effects on the system or kubernetes.
//
// Use the node's `Run` method to register and run the loops to update the node
// in Kubernetes.
func NewNode(p providers.NodeProvider, node *corev1.Node, leases v1beta1.LeaseInterface, nodes v1.NodeInterface, opts ...NodeOpt) (*Node, error) {
func NewNode(p NodeProvider, node *corev1.Node, leases v1beta1.LeaseInterface, nodes v1.NodeInterface, opts ...NodeOpt) (*Node, error) {
n := &Node{p: p, n: node, leases: leases, nodes: nodes}
for _, o := range opts {
if err := o(n); err != nil {
@@ -79,7 +99,7 @@ func WithNodeLease(l *coord.Lease) NodeOpt {
// Node deals with creating and managing a node object in Kubernetes.
// It can register a node with Kubernetes and periodically update its status.
type Node struct {
p providers.NodeProvider
p NodeProvider
n *corev1.Node
leases v1beta1.LeaseInterface

3
vkubelet/node_test.go
View File

@@ -6,7 +6,6 @@ import (
"testing"
"time"
"github.com/virtual-kubelet/virtual-kubelet/providers"
"gotest.tools/assert"
"gotest.tools/assert/cmp"
coord "k8s.io/api/coordination/v1beta1"
@@ -232,7 +231,7 @@ func testNode(t *testing.T) *corev1.Node {
}
type testNodeProvider struct {
providers.NodeProvider
NodeProvider
statusHandlers []func(*corev1.Node)
}

33
vkubelet/podcontroller.go
View File

@@ -39,6 +39,39 @@ import (
"github.com/virtual-kubelet/virtual-kubelet/log"
)
// PodLifecycleHandler defines the interface used by the PodController to react
// to new and changed pods scheduled to the node that is being managed.
type PodLifecycleHandler interface {
// CreatePod takes a Kubernetes Pod and deploys it within the provider.
CreatePod(ctx context.Context, pod *corev1.Pod) error
// UpdatePod takes a Kubernetes Pod and updates it within the provider.
UpdatePod(ctx context.Context, pod *corev1.Pod) error
// DeletePod takes a Kubernetes Pod and deletes it from the provider.
DeletePod(ctx context.Context, pod *corev1.Pod) error
// GetPod retrieves a pod by name from the provider (can be cached).
GetPod(ctx context.Context, namespace, name string) (*corev1.Pod, error)
// GetPodStatus retrieves the status of a pod by name from the provider.
GetPodStatus(ctx context.Context, namespace, name string) (*corev1.PodStatus, error)
// GetPods retrieves a list of all pods running on the provider (can be cached).
GetPods(context.Context) ([]*corev1.Pod, error)
}
// PodNotifier notifies callers of pod changes.
// Providers should implement this interface to enable callers to be notified
// of pod status updates asyncronously.
type PodNotifier interface {
// NotifyPods instructs the notifier to call the passed in function when
// the pod status changes.
//
// NotifyPods should not block callers.
NotifyPods(context.Context, func(*corev1.Pod))
}
// PodController is the controller implementation for Pod resources.
type PodController struct {
// server is the instance to which this controller belongs.

7
vkubelet/vkubelet.go
View File

@@ -6,7 +6,6 @@ import (
"time"
"github.com/virtual-kubelet/virtual-kubelet/manager"
"github.com/virtual-kubelet/virtual-kubelet/providers"
"github.com/virtual-kubelet/virtual-kubelet/trace"
corev1 "k8s.io/api/core/v1"
corev1informers "k8s.io/client-go/informers/core/v1"
@@ -23,7 +22,7 @@ type Server struct {
namespace string
nodeName string
k8sClient kubernetes.Interface
provider providers.Provider
provider PodLifecycleHandler
resourceManager *manager.ResourceManager
podSyncWorkers int
podInformer corev1informers.PodInformer
@@ -35,7 +34,7 @@ type Config struct {
Client *kubernetes.Clientset
Namespace string
NodeName string
Provider providers.Provider
Provider PodLifecycleHandler
ResourceManager *manager.ResourceManager
PodSyncWorkers int
PodInformer corev1informers.PodInformer
@@ -68,7 +67,7 @@ func (s *Server) Run(ctx context.Context) error {
q := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "podStatusUpdate")
s.runProviderSyncWorkers(ctx, q)
if pn, ok := s.provider.(providers.PodNotifier); ok {
if pn, ok := s.provider.(PodNotifier); ok {
pn.NotifyPods(ctx, func(pod *corev1.Pod) {
s.enqueuePodStatusUpdate(ctx, q, pod)
})