This adds the v1 lease controller

This refactors the v1 lease controller. It makes two functional differences
to the lease controller:
* It no longer ties lease updates to node pings or node status updates
* There is no fallback mechanism to status updates

This also moves vk_envtest, allowing for future brown-box testing of the
lease controller with envtest
This commit is contained in:
Sargun Dhillon
2020-12-07 10:37:17 -08:00
parent 8affa1c42a
commit 735eb34829
10 changed files with 525 additions and 366 deletions

View File

@@ -1,153 +0,0 @@
package e2e_test
import (
"context"
"flag"
"os"
"strings"
"testing"
"time"
"github.com/bombsimon/logrusr"
"github.com/sirupsen/logrus"
"github.com/virtual-kubelet/virtual-kubelet/log"
logruslogger "github.com/virtual-kubelet/virtual-kubelet/log/logrus"
"github.com/virtual-kubelet/virtual-kubelet/node"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
klogv1 "k8s.io/klog"
klogv2 "k8s.io/klog/v2"
"sigs.k8s.io/controller-runtime/pkg/envtest"
)
var enableEnvTest = flag.Bool("envtest", false, "Enable envtest based tests")
func TestMain(m *testing.M) {
flagset := flag.NewFlagSet("klog", flag.PanicOnError)
klogv1.InitFlags(flagset)
flagset.VisitAll(func(f *flag.Flag) {
flag.Var(f.Value, "klog."+f.Name, f.Usage)
})
flag.Parse()
os.Exit(m.Run())
}
func TestEnvtest(t *testing.T) {
if !*enableEnvTest || os.Getenv("VK_ENVTEST") != "" {
t.Skip("test only runs when -envtest is passed or if VK_ENVTEST is set to a non-empty value")
}
env := &envtest.Environment{}
_, err := env.Start()
assert.NilError(t, err)
defer func() {
assert.NilError(t, env.Stop())
}()
t.Log("Env test environment ready")
t.Run("E2ERunWithoutLeases", func(t *testing.T) {
testNodeE2ERun(t, env, false)
})
t.Run("E2ERunWithLeases", func(t *testing.T) {
testNodeE2ERun(t, env, true)
})
}
func nodeNameForTest(t *testing.T) string {
name := t.Name()
name = strings.ToLower(name)
name = strings.ReplaceAll(name, "/", "-")
name = strings.ReplaceAll(name, "_", "-")
return name
}
func testNodeE2ERun(t *testing.T, env *envtest.Environment, withLeases bool) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
sl := logrus.StandardLogger()
sl.SetLevel(logrus.DebugLevel)
logger := logruslogger.FromLogrus(sl.WithField("test", t.Name()))
ctx = log.WithLogger(ctx, logger)
log.L = logger
klogv2.SetLogger(logrusr.NewLogger(sl))
clientset, err := kubernetes.NewForConfig(env.Config)
assert.NilError(t, err)
nodes := clientset.CoreV1().Nodes()
_, err = clientset.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
assert.NilError(t, err)
testProvider := node.NewNaiveNodeProvider()
testNode := &corev1.Node{
ObjectMeta: metav1.ObjectMeta{
Name: nodeNameForTest(t),
},
}
testNodeCopy := testNode.DeepCopy()
opts := []node.NodeControllerOpt{}
leasesClient := clientset.CoordinationV1beta1().Leases(corev1.NamespaceNodeLease)
if withLeases {
opts = append(opts, node.WithNodeEnableLeaseV1Beta1(leasesClient, nil))
}
node, err := node.NewNodeController(testProvider, testNode, nodes, opts...)
assert.NilError(t, err)
chErr := make(chan error, 1)
go func() {
chErr <- node.Run(ctx)
}()
log.G(ctx).Debug("Waiting for node ready")
select {
case <-node.Ready():
case err = <-chErr:
t.Fatal(err)
case <-ctx.Done():
t.Fatal(ctx.Err())
}
now := time.Now()
var n *corev1.Node
for time.Since(now) < time.Minute*5 {
n, err = nodes.Get(ctx, testNodeCopy.Name, metav1.GetOptions{})
if errors.IsNotFound(err) {
continue
}
if err == nil {
t.Log(n)
goto node_found
}
time.Sleep(100 * time.Millisecond)
}
t.Fatal("Node never found")
node_found:
if withLeases {
for time.Since(now) < time.Minute*5 {
l, err := leasesClient.Get(ctx, testNodeCopy.Name, metav1.GetOptions{})
if errors.IsNotFound(err) {
continue
}
if err == nil {
t.Log(l)
assert.Assert(t, is.Len(l.OwnerReferences, 1))
assert.Assert(t, is.Equal(l.OwnerReferences[0].Name, n.Name))
assert.Assert(t, is.Equal(l.OwnerReferences[0].UID, n.UID))
goto lease_found
}
time.Sleep(100 * time.Millisecond)
}
}
lease_found:
cancel()
err = <-chErr
assert.NilError(t, err)
}