diff --git a/README.md b/README.md index f686a11aa..e4f741e79 100644 --- a/README.md +++ b/README.md @@ -259,49 +259,7 @@ Running the unit tests locally is as simple as `make test`. ### End-to-end tests -Virtual Kubelet includes an end-to-end (e2e) test suite which is used to validate its implementation. -The current e2e suite **does not** run for any providers other than the `mock` provider. - -To run the e2e suite, three things are required: -- a local Kubernetes cluster (we have tested with [Docker for Mac](https://docs.docker.com/docker-for-mac/install/) and [Minikube](https://github.com/kubernetes/minikube)); -- Your _kubeconfig_ default context points to the local Kubernetes cluster; -- [`skaffold`](https://github.com/GoogleContainerTools/skaffold). - -Since our CI uses Minikube, we describe below how to run e2e on top of it. - -To create a Minikube cluster, run the following command after [installing Minikube](https://github.com/kubernetes/minikube#installation): - -```console -$ minikube start -``` - -The e2e suite requires Virtual Kubelet to be running as a pod inside the Kubernetes cluster. -In order to make the testing process easier, the build toolchain leverages on `skaffold` to automatically deploy the Virtual Kubelet to the Kubernetes cluster using the mock provider. - -To run the e2e test suite, you can run the following command: - -```console -$ make e2e -``` - -When you're done testing, you can run the following command to cleanup the resources created by `skaffold`: - -```console -$ make skaffold MODE=delete -``` - -Please note that this will not unregister the Virtual Kubelet as a node in the Kubernetes cluster. -In order to do so, you should run: - -```console -$ kubectl delete node vkubelet-mock-0 -``` - -To clean up all resources you can run: - -```console -$ make e2e.clean -``` +Check out [`test/e2e`](./test/e2e) for more details. ## Known quirks and workarounds diff --git a/go.mod b/go.mod index 968aa905e..ca195826d 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/davecgh/go-spew v1.1.1 github.com/docker/spdystream v0.0.0-20170912183627-bc6354cbbc29 // indirect github.com/elazarl/goproxy v0.0.0-20190421051319-9d40249d3c2f // indirect - github.com/elazarl/goproxy/ext v0.0.0-20190421051319-9d40249d3c2f // indirect + github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2 // indirect github.com/evanphx/json-patch v4.1.0+incompatible // indirect github.com/gogo/protobuf v1.2.1 // indirect github.com/golang/groupcache v0.0.0-20181024230925-c65c006176ff // indirect @@ -40,7 +40,7 @@ require ( gotest.tools v2.2.0+incompatible k8s.io/api v0.0.0 k8s.io/apimachinery v0.0.0 - k8s.io/client-go v0.0.0 + k8s.io/client-go v10.0.0+incompatible k8s.io/klog v0.3.1 k8s.io/kube-openapi v0.0.0-20190510232812-a01b7d5d6c22 // indirect k8s.io/kubernetes v1.15.2 diff --git a/go.sum b/go.sum index 94ace051e..7a91a9cc2 100644 --- a/go.sum +++ b/go.sum @@ -85,8 +85,8 @@ github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFP github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/elazarl/goproxy v0.0.0-20190421051319-9d40249d3c2f h1:8GDPb0tCY8LQ+OJ3dbHb5sA6YZWXFORQYZx5sdsTlMs= github.com/elazarl/goproxy v0.0.0-20190421051319-9d40249d3c2f/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= -github.com/elazarl/goproxy/ext v0.0.0-20190421051319-9d40249d3c2f h1:AUj1VoZUfhPhOPHULCQQDnGhRelpFWHMLhQVWDsS0v4= -github.com/elazarl/goproxy/ext v0.0.0-20190421051319-9d40249d3c2f/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8= +github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2 h1:dWB6v3RcOy03t/bUadywsbyrQwCqZeNIEX6M1OtSZOM= +github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= github.com/euank/go-kmsg-parser v2.0.0+incompatible/go.mod h1:MhmAMZ8V4CYH4ybgdRwPr2TU5ThnS43puaKEMpja1uw= @@ -155,9 +155,7 @@ github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e/go.mod h1:0AA// github.com/google/btree v0.0.0-20160524151835-7d79101e329e/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/cadvisor v0.33.2-0.20190411163913-9db8c7dee20a/go.mod h1:1nql6U13uTHaLYB8rLS5x9IJc2qT6Xd/Tr1sTX6NE48= github.com/google/certificate-transparency-go v1.0.21/go.mod h1:QeJfpSbVSfYc7RgB3gJFj9cbuQMMchQxrWXz8Ruopmg= -github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -173,9 +171,7 @@ github.com/googleapis/gnostic v0.1.0 h1:rVsPeBmXbYv4If/cumu1AzZPwV58q433hvONV1UE github.com/googleapis/gnostic v0.1.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/gophercloud/gophercloud v0.0.0-20190126172459-c818fa66e4c8/go.mod h1:3WdhXV3rUYy9p6AUW8d94kr+HS62Y4VL9mBnFxsD8q4= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/context v1.1.1 h1:AWwleXJkX/nhcU9bZSnZoi3h/qGYqQAGhq6zZe/aQW8= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= -github.com/gorilla/mux v1.6.2 h1:Pgr17XVTNXAk3q/r4CpKzC5xBM/qW1uVLV+IhRZpIIk= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.0 h1:tOSd0UKHQd6urX6ApfOn4XdBMY6Sh1MfxV3kmaazO+U= github.com/gorilla/mux v1.7.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= @@ -279,7 +275,6 @@ github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJ github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.0.1/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -352,7 +347,6 @@ github.com/xanzy/go-cloudstack v0.0.0-20160728180336-1e2cbf647e57/go.mod h1:s3eL github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1/go.mod h1:QcJo0QPSfTONNIgpN5RA8prR7fF8nkF6cTWTcNerRO8= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= -go.opencensus.io v0.20.2 h1:NAfh7zF0/3/HqtMvJNZ/RFrSlCE6ZTlHmKfhL/Dm1Jk= go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0 h1:mU6zScU4U1YAFPHEHYk+3JC4SY7JxgkqS10ZOSyksNg= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= @@ -437,7 +431,6 @@ google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMt google.golang.org/api v0.3.2 h1:iTp+3yyl/KOtxa/d1/JUE0GGSoR6FuW5udver22iwpw= google.golang.org/api v0.3.2/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= diff --git a/internal/test/e2e/framework/env.go b/internal/test/e2e/framework/env.go index 94ab25995..6ce9e8376 100644 --- a/internal/test/e2e/framework/env.go +++ b/internal/test/e2e/framework/env.go @@ -75,7 +75,7 @@ func (f *Framework) CreatePodObjectWithOptionalSecretKey(testName string) *corev // CreatePodObjectWithEnv creates a pod object whose name starts with "env-test-" and that uses the specified environment configuration for its first container. func (f *Framework) CreatePodObjectWithEnv(testName string, env []corev1.EnvVar) *corev1.Pod { - pod := f.CreateDummyPodObjectWithPrefix(testName, "env-test-", "foo") + pod := f.CreateDummyPodObjectWithPrefix(testName, "env-test", "foo") pod.Spec.Containers[0].Env = env return pod } diff --git a/internal/test/e2e/framework/framework.go b/internal/test/e2e/framework/framework.go index e63086d4c..328f865b2 100644 --- a/internal/test/e2e/framework/framework.go +++ b/internal/test/e2e/framework/framework.go @@ -1,6 +1,8 @@ package framework import ( + "time" + "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" @@ -8,17 +10,19 @@ import ( // Framework encapsulates the configuration for the current run, and provides helper methods to be used during testing. type Framework struct { - KubeClient kubernetes.Interface - Namespace string - NodeName string + KubeClient kubernetes.Interface + Namespace string + NodeName string + WatchTimeout time.Duration } // NewTestingFramework returns a new instance of the testing framework. -func NewTestingFramework(kubeconfig, namespace, nodeName string) *Framework { +func NewTestingFramework(kubeconfig, namespace, nodeName string, watchTimeout time.Duration) *Framework { return &Framework{ - KubeClient: createKubeClient(kubeconfig), - Namespace: namespace, - NodeName: nodeName, + KubeClient: createKubeClient(kubeconfig), + Namespace: namespace, + NodeName: nodeName, + WatchTimeout: watchTimeout, } } diff --git a/internal/test/e2e/framework/node.go b/internal/test/e2e/framework/node.go index d1fa4892e..4ded8496f 100644 --- a/internal/test/e2e/framework/node.go +++ b/internal/test/e2e/framework/node.go @@ -31,7 +31,7 @@ func (f *Framework) WaitUntilNodeCondition(fn watch.ConditionFunc) error { } // Watch for updates to the Pod resource until fn is satisfied, or until the timeout is reached. - ctx, cancel := context.WithTimeout(context.Background(), defaultWatchTimeout) + ctx, cancel := context.WithTimeout(context.Background(), f.WatchTimeout) defer cancel() last, err := watch.UntilWithSync(ctx, lw, &corev1.Node{}, nil, fn) if err != nil { diff --git a/internal/test/e2e/framework/pod.go b/internal/test/e2e/framework/pod.go index 2699761ca..124ac8ad8 100644 --- a/internal/test/e2e/framework/pod.go +++ b/internal/test/e2e/framework/pod.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "strings" - "time" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -16,8 +15,6 @@ import ( podutil "k8s.io/kubernetes/pkg/api/v1/pod" ) -const defaultWatchTimeout = 2 * time.Minute - // CreateDummyPodObjectWithPrefix creates a dujmmy pod object using the specified prefix as the value of .metadata.generateName. // A variable number of strings can be provided. // For each one of these strings, a container that uses the string as its image will be appended to the pod. @@ -25,8 +22,7 @@ const defaultWatchTimeout = 2 * time.Minute func (f *Framework) CreateDummyPodObjectWithPrefix(testName string, prefix string, images ...string) *corev1.Pod { // Safe the test name if testName != "" { - testName = strings.Replace(testName, "/", "-", -1) - testName = strings.ToLower(testName) + testName = stripParentTestName(strings.ToLower(testName)) prefix = prefix + "-" + testName + "-" } enableServiceLink := false @@ -88,7 +84,7 @@ func (f *Framework) WaitUntilPodCondition(namespace, name string, fn watch.Condi }, } // Watch for updates to the Pod resource until fn is satisfied, or until the timeout is reached. - ctx, cfn := context.WithTimeout(context.Background(), defaultWatchTimeout) + ctx, cfn := context.WithTimeout(context.Background(), f.WatchTimeout) defer cfn() last, err := watch.UntilWithSync(ctx, lw, &corev1.Pod{}, nil, fn) if err != nil { @@ -147,7 +143,7 @@ func (f *Framework) WaitUntilPodEventWithReason(pod *corev1.Pod, reason string) }, } // Watch for updates to the Event resource until fn is satisfied, or until the timeout is reached. - ctx, cfn := context.WithTimeout(context.Background(), defaultWatchTimeout) + ctx, cfn := context.WithTimeout(context.Background(), f.WatchTimeout) defer cfn() last, err := watch.UntilWithSync(ctx, lw, &corev1.Event{}, nil, func(event watchapi.Event) (b bool, e error) { switch event.Type { @@ -184,3 +180,15 @@ func (f *Framework) GetRunningPods() (*corev1.PodList, error) { return result, err } + +// stripParentTestName strips out the parent's test name from the input (in the form of 'TestParent/TestChild'). +// Some test cases use their name as the pod name for testing purpose, and sometimes it might exceed 63 +// characters (Kubernetes's limit for pod name). This function ensures that we strip out the parent's +// test name to decrease the length of the pod name +func stripParentTestName(name string) string { + parts := strings.Split(name, "/") + if len(parts) == 1 { + return parts[0] + } + return parts[len(parts)-1] +} diff --git a/internal/test/e2e/main_test.go b/internal/test/e2e/main_test.go index e13fe92f2..c41563762 100644 --- a/internal/test/e2e/main_test.go +++ b/internal/test/e2e/main_test.go @@ -4,12 +4,12 @@ package e2e import ( "flag" - "os" + "fmt" "testing" - v1 "k8s.io/api/core/v1" + vke2e "github.com/virtual-kubelet/virtual-kubelet/test/e2e" - "github.com/virtual-kubelet/virtual-kubelet/internal/test/e2e/framework" + v1 "k8s.io/api/core/v1" ) const ( @@ -18,15 +18,9 @@ const ( ) var ( - // f is the testing framework used for running the test suite. - f *framework.Framework - - // kubeconfig is the path to the kubeconfig file to use when running the test suite outside a Kubernetes cluster. kubeconfig string - // namespace is the name of the Kubernetes namespace to use for running the test suite (i.e. where to create pods). - namespace string - // nodeName is the name of the virtual-kubelet node to test. - nodeName string + namespace string + nodeName string ) func init() { @@ -36,17 +30,36 @@ func init() { flag.Parse() } -func TestMain(m *testing.M) { - // Set sane defaults in case no values (or empty ones) have been provided. +// Provider-specific setup function +func setup() error { + fmt.Println("Setting up end-to-end test suite for mock provider...") + return nil +} + +// Provider-specific teardown function +func teardown() error { + fmt.Println("Tearing down end-to-end test suite for mock provider...") + return nil +} + +// Provider-specific shouldSkipTest function +func shouldSkipTest(testName string) bool { + return false +} + +// TestEndToEnd creates and runs the end-to-end test suite for virtual kubelet +func TestEndToEnd(t *testing.T) { setDefaults() - // Create a new instance of the test framework targeting the specified node. - f = framework.NewTestingFramework(kubeconfig, namespace, nodeName) - // Wait for the virtual-kubelet pod to be ready. - if _, err := f.WaitUntilPodReady(namespace, nodeName); err != nil { - panic(err) + config := vke2e.EndToEndTestSuiteConfig{ + Kubeconfig: kubeconfig, + Namespace: namespace, + NodeName: nodeName, + Setup: setup, + Teardown: teardown, + ShouldSkipTest: shouldSkipTest, } - // Run the test suite. - os.Exit(m.Run()) + ts := vke2e.NewEndToEndTestSuite(config) + ts.Run(t) } // setDefaults sets sane defaults in case no values (or empty ones) have been provided. diff --git a/internal/test/suite/suite.go b/internal/test/suite/suite.go new file mode 100644 index 000000000..f3dc7db6e --- /dev/null +++ b/internal/test/suite/suite.go @@ -0,0 +1,85 @@ +package suite + +import ( + "reflect" + "runtime/debug" + "strings" + "testing" +) + +// TestFunc defines the test function in a test case +type TestFunc func(*testing.T) + +// SetUpFunc sets up provider-specific resource in the test suite +type SetUpFunc func() error + +// TeardownFunc tears down provider-specific resources from the test suite +type TeardownFunc func() error + +// ShouldSkipTestFunc determines whether the test suite should skip certain tests +type ShouldSkipTestFunc func(string) bool + +// TestSuite contains methods that defines the lifecycle of a test suite +type TestSuite interface { + Setup() + Teardown() +} + +// TestSkipper allows providers to skip certain tests +type TestSkipper interface { + ShouldSkipTest(string) bool +} + +type testCase struct { + name string + f TestFunc +} + +// Run runs tests registered in the test suite +func Run(t *testing.T, ts TestSuite) { + defer failOnPanic(t) + + ts.Setup() + defer ts.Teardown() + + // The implementation below is based on https://github.com/stretchr/testify + testFinder := reflect.TypeOf(ts) + tests := []testCase{} + for i := 0; i < testFinder.NumMethod(); i++ { + method := testFinder.Method(i) + if !isValidTestFunc(method) { + continue + } + + test := testCase{ + name: method.Name, + f: func(t *testing.T) { + defer failOnPanic(t) + if tSkipper, ok := ts.(TestSkipper); ok && tSkipper.ShouldSkipTest(method.Name) { + t.Skipf("Skipped due to shouldSkipTest()") + } + method.Func.Call([]reflect.Value{reflect.ValueOf(ts), reflect.ValueOf(t)}) + }, + } + tests = append(tests, test) + } + + for _, test := range tests { + t.Run(test.name, test.f) + } +} + +// failOnPanic recovers panic occurred in the test suite and marks the test / test suite as failed +func failOnPanic(t *testing.T) { + if r := recover(); r != nil { + t.Fatalf("%v\n%s", r, debug.Stack()) + } +} + +// isValidTestFunc determines whether or not a given method is a valid test function +func isValidTestFunc(method reflect.Method) bool { + return strings.HasPrefix(method.Name, "Test") && // Test function name must start with "Test", + method.Type.NumIn() == 2 && // the number of function input should be 2 (*TestSuite ts and t *testing.T), + method.Type.In(1) == reflect.TypeOf(&testing.T{}) && + method.Type.NumOut() == 0 // and the number of function output should be 0 +} diff --git a/internal/test/suite/suite_test.go b/internal/test/suite/suite_test.go new file mode 100644 index 000000000..e6a767c9e --- /dev/null +++ b/internal/test/suite/suite_test.go @@ -0,0 +1,126 @@ +package suite + +import ( + "strings" + "testing" + + "gotest.tools/assert" + is "gotest.tools/assert/cmp" +) + +type basicTestSuite struct { + setupCount int + testFooCount int + testBarCount int + bazCount int + testFooBarCount int + testFooBazCount int + testBarBazCount int + teardownCount int + testsRan []string +} + +func (bts *basicTestSuite) Setup() { + bts.setupCount++ +} + +func (bts *basicTestSuite) Teardown() { + bts.teardownCount++ +} + +func (bts *basicTestSuite) TestFoo(t *testing.T) { + bts.testFooCount++ + bts.testsRan = append(bts.testsRan, t.Name()) +} + +func (bts *basicTestSuite) TestBar(t *testing.T) { + bts.testBarCount++ + bts.testsRan = append(bts.testsRan, t.Name()) +} + +// Baz should not be executed by the test suite +// because it does not have the prefix 'Test' +func (bts *basicTestSuite) Baz(t *testing.T) { + bts.bazCount++ + bts.testsRan = append(bts.testsRan, t.Name()) +} + +// TestFooBar should not be executed by the test suite +// because the number of function input is not 2 (*basicTestSuite and *testing.T) +func (bts *basicTestSuite) TestFooBar() { + bts.testFooBarCount++ + bts.testsRan = append(bts.testsRan, "TestFooBar") +} + +// TestFooBaz should not be executed by the test suite +// because the number of function output is not 0 +func (bts *basicTestSuite) TestFooBaz(t *testing.T) error { + bts.testFooBazCount++ + bts.testsRan = append(bts.testsRan, t.Name()) + return nil +} + +// TestBarBaz should not be executed by the test suite +// because the type of the function input is not *testing.T +func (bts *basicTestSuite) TestBarBaz(t string) { + bts.testBarBazCount++ + bts.testsRan = append(bts.testsRan, "TestBarBaz") +} + +func TestBasicTestSuite(t *testing.T) { + bts := new(basicTestSuite) + Run(t, bts) + + assert.Equal(t, bts.setupCount, 1) + assert.Equal(t, bts.testFooCount, 1) + assert.Equal(t, bts.testBarCount, 1) + assert.Equal(t, bts.teardownCount, 1) + assert.Assert(t, is.Len(bts.testsRan, 2)) + assertTestsRan(t, bts.testsRan) + assertNonTests(t, bts) +} + +type skipTestSuite struct { + basicTestSuite + skippedTestCount int +} + +func (sts *skipTestSuite) ShouldSkipTest(testName string) bool { + if testName == "TestBar" { + sts.skippedTestCount++ + return true + } + return false +} + +func TestSkipTest(t *testing.T) { + sts := new(skipTestSuite) + Run(t, sts) + + assert.Equal(t, sts.setupCount, 1) + assert.Equal(t, sts.testFooCount, 1) + assert.Equal(t, sts.testBarCount, 0) + assert.Equal(t, sts.teardownCount, 1) + assert.Equal(t, sts.skippedTestCount, 1) + assert.Assert(t, is.Len(sts.testsRan, 1)) + assertTestsRan(t, sts.testsRan) + assertNonTests(t, &sts.basicTestSuite) +} + +func assertTestsRan(t *testing.T, testsRan []string) { + for _, testRan := range testsRan { + parts := strings.Split(testRan, "/") + // Make sure that the name of the test has exactly one parent name and one subtest name + assert.Assert(t, is.Len(parts, 2)) + // Check the parent test's name + assert.Equal(t, parts[0], t.Name()) + } +} + +// assertNonTests ensures that any malformed test functions are not run by the test suite +func assertNonTests(t *testing.T, bts *basicTestSuite) { + assert.Equal(t, bts.bazCount, 0) + assert.Equal(t, bts.testFooBarCount, 0) + assert.Equal(t, bts.testFooBazCount, 0) + assert.Equal(t, bts.testBarBazCount, 0) +} diff --git a/test/e2e/README.md b/test/e2e/README.md new file mode 100644 index 000000000..8f759a77a --- /dev/null +++ b/test/e2e/README.md @@ -0,0 +1,191 @@ +# Importable End-To-End Test Suite + +Virtual Kubelet (VK) provides an importable end-to-end (E2E) test suite containing a set of common integration tests. As a provider, you can import the test suite and use it to validate your VK implementation. + +## Prerequisite + +To run the E2E test suite, three things are required: + +- A local Kubernetes cluster (we have tested with [Docker for Mac](https://docs.docker.com/docker-for-mac/install/) and [Minikube](https://kubernetes.io/docs/tasks/tools/install-minikube/)); +- Your _kubeconfig_ default context points to the local Kubernetes cluster; +- [skaffold](https://skaffold.dev/docs/getting-started/#installing-skaffold) + +> The test suite is based on [VK 1.0](https://github.com/virtual-kubelet/virtual-kubelet/releases/tag/v1.0.0). If your VK implementation is based on legacy VK library (< v1.0.0), you will have to upgrade it to VK 1.0 using [virtual-kubelet/node-cli](https://github.com/virtual-kubelet/node-cli). + +### Skaffold Folder + +Before running the E2E test suite, you will need to copy the [`./hack`](../../hack) folder containing Skaffold-related files such as Dockerfile, manifests, and certificates to your VK project root. Skaffold essentially helps package your virtual kubelet into a container based on the given [`Dockerfile`](../../hack/skaffold/virtual-kubelet/Dockerfile) and deploy it as a pod (see [`pod.yml`](../../hack/skaffold/virtual-kubelet/pod.yml)) to your Kubernetes test cluster. In summary, you will likely need to modify the VK name in those files, customize the VK configuration file, and the API server certificates (`-crt.pem` and `-key.pem`) before running the test suite. + +### Makefile.e2e + +Also, you will need to copy [`Makefile.e2e`](../../Makefile.e2e) to your VK project root. It contains necessary `make` commands to run the E2E test suite. Do not forget to add `include Makefile.e2e` in your `Makefile`. + +### File Structure + +A minimal VK provider should now have a file structure similar to the one below: + +```console +. +├── Makefile +├── Makefile.e2e +├── README.md +├── cmd +│   └── virtual-kubelet +│   └── main.go +├── go.mod +├── go.sum +├── hack +│   └── skaffold +│   └── virtual-kubelet +│   ├── Dockerfile +│   ├── base.yml +│   ├── pod.yml +│   ├── skaffold.yml +│   ├── vkubelet-provider-0-cfg.json +│   ├── vkubelet-provider-0-crt.pem +│   └── vkubelet-provider-0-key.pem +├── test +│   └── e2e +│ └── main_test.go # import and run the E2E test suite here +├── provider.go # provider-specific VK implementation +├── provider_test.go # unit test +``` + +## Importing the Test Suite + +The test suite can be easily imported in your test files (e.g. `./test/e2e/main_test.go`) with the following import statement: +```go +import ( + vke2e "github.com/virtual-kubelet/virtual-kubelet/test/e2e" +) +``` + +### Test Suite Customization + +The test suite allows providers to customize the test suite using `EndToEndTestSuiteConfig`: + +```go +// EndToEndTestSuiteConfig is the config passed to initialize the testing framework and test suite. +type EndToEndTestSuiteConfig struct { + // Kubeconfig is the path to the kubeconfig file to use when running the test suite outside a Kubernetes cluster. + Kubeconfig string + // Namespace is the name of the Kubernetes namespace to use for running the test suite (i.e. where to create pods). + Namespace string + // NodeName is the name of the virtual-kubelet node to test. + NodeName string + // WatchTimeout is the duration for which the framework watch a particular condition to be satisfied (e.g. watches a pod becoming ready) + WatchTimeout time.Duration + // Setup is a function that sets up provider-specific resource in the test suite + Setup suite.SetUpFunc + // Teardown is a function that tears down provider-specific resources from the test suite + Teardown suite.TeardownFunc + // ShouldSkipTest is a function that determines whether the test suite should skip certain tests + ShouldSkipTest suite.ShouldSkipTestFunc +} +``` + +> `Setup()` is invoked before running the E2E test suite, and `Teardown()` is invoked after all the E2E tests are finished. + +You will need an `EndToEndTestSuiteConfig` to create an `EndToEndTestSuite` using `NewEndToEndTestSuite`. After that, invoke `Run` from `EndToEndTestSuite` to start the test suite. The code snippet below is a minimal example of how to import and run the test suite in your test file. + +```go +package e2e + +import ( + "time" + + vke2e "github.com/virtual-kubelet/virtual-kubelet/test/e2e" +) + +var ( + kubeconfig string + namespace string + nodeName string +) + +// Read the following variables from command-line flags +func init() { + flag.StringVar(&kubeconfig, "kubeconfig", "", "path to the kubeconfig file to use when running the test suite outside a kubernetes cluster") + flag.StringVar(&namespace, "namespace", defaultNamespace, "the name of the kubernetes namespace to use for running the test suite (i.e. where to create pods)") + flag.StringVar(&nodeName, "node-name", defaultNodeName, "the name of the virtual-kubelet node to test") + flag.Parse() +} + +func setup() error { + fmt.Println("Setting up end-to-end test suite...") + return nil +} + +func teardown() error { + fmt.Println("Tearing down end-to-end test suite...") + return nil +} + +func shouldSkipTest(testName string) bool { + // Skip the test 'TestGetStatsSummary' + return testName == "TestGetStatsSummary" +} + +func TestEndToEnd(t *testing.T) { + config := vke2e.EndToEndTestSuiteConfig{ + Kubeconfig: kubeconfig, + Namespace: namespace, + NodeName: nodeName, + Setup: setup, + Teardown: teardown, + ShouldSkipTest: shouldSkipTest, + WaitTimeout: 5 * time.Minute, + } + ts := vke2e.NewEndToEndTestSuite(config) + ts.Run(t) +} +``` + +## Running the Test Suite + +Since our CI uses Minikube, we describe below how to run E2E on top of it. + +To create a Minikube cluster, run the following command after [installing Minikube](https://github.com/kubernetes/minikube#installation): + +```bash +minikube start +``` + +To run the E2E test suite, you can run the following command: + +```bash +make e2e +``` + +You can see from the console output whether the tests in the test suite pass or not. + +```console +... +=== RUN TestEndToEnd +=== RUN TestEndToEnd/TestCreatePodWithMandatoryInexistentConfigMap +=== RUN TestEndToEnd/TestCreatePodWithMandatoryInexistentSecrets +=== RUN TestEndToEnd/TestCreatePodWithOptionalInexistentConfigMap +=== RUN TestEndToEnd/TestCreatePodWithOptionalInexistentSecrets +=== RUN TestEndToEnd/TestGetStatsSummary +=== RUN TestEndToEnd/TestNodeCreateAfterDelete +=== RUN TestEndToEnd/TestPodLifecycleForceDelete +=== RUN TestEndToEnd/TestPodLifecycleGracefulDelete +--- PASS: TestEndToEnd (21.93s) + --- PASS: TestEndToEnd/TestCreatePodWithMandatoryInexistentConfigMap (0.03s) + --- PASS: TestEndToEnd/TestCreatePodWithMandatoryInexistentSecrets (0.03s) + --- PASS: TestEndToEnd/TestCreatePodWithOptionalInexistentConfigMap (0.55s) + --- PASS: TestEndToEnd/TestCreatePodWithOptionalInexistentSecrets (0.99s) + --- PASS: TestEndToEnd/TestGetStatsSummary (0.80s) + --- PASS: TestEndToEnd/TestNodeCreateAfterDelete (9.63s) + --- PASS: TestEndToEnd/TestPodLifecycleForceDelete (2.05s) + basic.go:158: Created pod: nginx-testpodlifecycleforcedelete-jz84g + basic.go:164: Pod nginx-testpodlifecycleforcedelete-jz84g ready + basic.go:197: Force deleted pod: nginx-testpodlifecycleforcedelete-jz84g + basic.go:214: Pod ended as phase: Running + --- PASS: TestEndToEnd/TestPodLifecycleGracefulDelete (1.04s) + basic.go:87: Created pod: nginx-testpodlifecyclegracefuldelete-r84v7 + basic.go:93: Pod nginx-testpodlifecyclegracefuldelete-r84v7 ready + basic.go:120: Deleted pod: nginx-testpodlifecyclegracefuldelete-r84v7 +PASS +... +``` diff --git a/internal/test/e2e/basic_test.go b/test/e2e/basic.go similarity index 93% rename from internal/test/e2e/basic_test.go rename to test/e2e/basic.go index 1e644d608..01a81b509 100644 --- a/internal/test/e2e/basic_test.go +++ b/test/e2e/basic.go @@ -1,5 +1,3 @@ -// +build e2e - package e2e import ( @@ -22,9 +20,9 @@ const ( // TestGetStatsSummary creates a pod having two containers and queries the /stats/summary endpoint of the virtual-kubelet. // It expects this endpoint to return stats for the current node, as well as for the aforementioned pod and each of its two containers. -func TestGetStatsSummary(t *testing.T) { +func (ts *EndToEndTestSuite) TestGetStatsSummary(t *testing.T) { // Create a pod with prefix "nginx-" having three containers. - pod, err := f.CreatePod(f.CreateDummyPodObjectWithPrefix(t.Name(), "nginx-", "foo", "bar", "baz")) + pod, err := f.CreatePod(f.CreateDummyPodObjectWithPrefix(t.Name(), "nginx", "foo", "bar", "baz")) if err != nil { t.Fatal(err) } @@ -69,10 +67,10 @@ func TestGetStatsSummary(t *testing.T) { // Then, it deletes the pods and verifies that the provider has been asked to delete it. // These verifications are made using the /stats/summary endpoint of the virtual-kubelet, by checking for the presence or absence of the pods. // Hence, the provider being tested must implement the PodMetricsProvider interface. -func TestPodLifecycleGracefulDelete(t *testing.T) { +func (ts *EndToEndTestSuite) TestPodLifecycleGracefulDelete(t *testing.T) { // Create a pod with prefix "nginx-" having a single container. - podSpec := f.CreateDummyPodObjectWithPrefix(t.Name(), "nginx-", "foo") - podSpec.Spec.NodeName = nodeName + podSpec := f.CreateDummyPodObjectWithPrefix(t.Name(), "nginx", "foo") + podSpec.Spec.NodeName = f.NodeName pod, err := f.CreatePod(podSpec) if err != nil { @@ -139,11 +137,11 @@ func TestPodLifecycleGracefulDelete(t *testing.T) { assert.Assert(t, *podLast.ObjectMeta.GetDeletionGracePeriodSeconds() > 0) } -// TestPodLifecycleNonGracefulDelete creates one podsand verifies that the provider has created them +// TestPodLifecycleForceDelete creates one podsand verifies that the provider has created them // and put them in the running lifecycle. It then does a force delete on the pod, and verifies the provider // has deleted it. -func TestPodLifecycleForceDelete(t *testing.T) { - podSpec := f.CreateDummyPodObjectWithPrefix(t.Name(), "nginx-", "foo") +func (ts *EndToEndTestSuite) TestPodLifecycleForceDelete(t *testing.T) { + podSpec := f.CreateDummyPodObjectWithPrefix(t.Name(), "nginx", "foo") // Create a pod with prefix having a single container. pod, err := f.CreatePod(podSpec) if err != nil { @@ -217,7 +215,7 @@ func TestPodLifecycleForceDelete(t *testing.T) { // TestCreatePodWithOptionalInexistentSecrets tries to create a pod referencing optional, inexistent secrets. // It then verifies that the pod is created successfully. -func TestCreatePodWithOptionalInexistentSecrets(t *testing.T) { +func (ts *EndToEndTestSuite) TestCreatePodWithOptionalInexistentSecrets(t *testing.T) { // Create a pod with a single container referencing optional, inexistent secrets. pod, err := f.CreatePod(f.CreatePodObjectWithOptionalSecretKey(t.Name())) if err != nil { @@ -251,7 +249,7 @@ func TestCreatePodWithOptionalInexistentSecrets(t *testing.T) { // TestCreatePodWithMandatoryInexistentSecrets tries to create a pod referencing inexistent secrets. // It then verifies that the pod is not created. -func TestCreatePodWithMandatoryInexistentSecrets(t *testing.T) { +func (ts *EndToEndTestSuite) TestCreatePodWithMandatoryInexistentSecrets(t *testing.T) { // Create a pod with a single container referencing inexistent secrets. pod, err := f.CreatePod(f.CreatePodObjectWithMandatorySecretKey(t.Name())) if err != nil { @@ -280,7 +278,7 @@ func TestCreatePodWithMandatoryInexistentSecrets(t *testing.T) { // TestCreatePodWithOptionalInexistentConfigMap tries to create a pod referencing optional, inexistent config map. // It then verifies that the pod is created successfully. -func TestCreatePodWithOptionalInexistentConfigMap(t *testing.T) { +func (ts *EndToEndTestSuite) TestCreatePodWithOptionalInexistentConfigMap(t *testing.T) { // Create a pod with a single container referencing optional, inexistent config map. pod, err := f.CreatePod(f.CreatePodObjectWithOptionalConfigMapKey(t.Name())) if err != nil { @@ -314,7 +312,7 @@ func TestCreatePodWithOptionalInexistentConfigMap(t *testing.T) { // TestCreatePodWithMandatoryInexistentConfigMap tries to create a pod referencing inexistent secrets. // It then verifies that the pod is not created. -func TestCreatePodWithMandatoryInexistentConfigMap(t *testing.T) { +func (ts *EndToEndTestSuite) TestCreatePodWithMandatoryInexistentConfigMap(t *testing.T) { // Create a pod with a single container referencing inexistent config map. pod, err := f.CreatePod(f.CreatePodObjectWithMandatoryConfigMapKey(t.Name())) if err != nil { diff --git a/internal/test/e2e/node_test.go b/test/e2e/node.go similarity index 95% rename from internal/test/e2e/node_test.go rename to test/e2e/node.go index 5c81022b4..c928ea3d1 100644 --- a/internal/test/e2e/node_test.go +++ b/test/e2e/node.go @@ -1,5 +1,3 @@ -// +build e2e - package e2e import ( @@ -17,7 +15,7 @@ import ( // TestNodeCreateAfterDelete makes sure that a node is automatically recreated // if it is deleted while VK is running. -func TestNodeCreateAfterDelete(t *testing.T) { +func (ts *EndToEndTestSuite) TestNodeCreateAfterDelete(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() diff --git a/test/e2e/suite.go b/test/e2e/suite.go new file mode 100644 index 000000000..e91f7ab9b --- /dev/null +++ b/test/e2e/suite.go @@ -0,0 +1,104 @@ +package e2e + +import ( + "testing" + "time" + + "github.com/virtual-kubelet/virtual-kubelet/internal/test/e2e/framework" + "github.com/virtual-kubelet/virtual-kubelet/internal/test/suite" +) + +const defaultWatchTimeout = 2 * time.Minute + +// f is a testing framework that is accessible across the e2e package +var f *framework.Framework + +// EndToEndTestSuite holds the setup, teardown, and shouldSkipTest functions for a specific provider +type EndToEndTestSuite struct { + setup suite.SetUpFunc + teardown suite.TeardownFunc + shouldSkipTest suite.ShouldSkipTestFunc +} + +// EndToEndTestSuiteConfig is the config passed to initialize the testing framework and test suite. +type EndToEndTestSuiteConfig struct { + // Kubeconfig is the path to the kubeconfig file to use when running the test suite outside a Kubernetes cluster. + Kubeconfig string + // Namespace is the name of the Kubernetes namespace to use for running the test suite (i.e. where to create pods). + Namespace string + // NodeName is the name of the virtual-kubelet node to test. + NodeName string + // WatchTimeout is the duration for which the framework watch a particular condition to be satisfied (e.g. watches a pod becoming ready) + WatchTimeout time.Duration + // Setup is a function that sets up provider-specific resource in the test suite + Setup suite.SetUpFunc + // Teardown is a function that tears down provider-specific resources from the test suite + Teardown suite.TeardownFunc + // ShouldSkipTest is a function that determines whether the test suite should skip certain tests + ShouldSkipTest suite.ShouldSkipTestFunc +} + +// Setup runs the setup function from the provider and other +// procedures before running the test suite +func (ts *EndToEndTestSuite) Setup() { + if err := ts.setup(); err != nil { + panic(err) + } + + // Wait for the virtual kubelet (deployed as a pod) to become fully ready + if _, err := f.WaitUntilPodReady(f.Namespace, f.NodeName); err != nil { + panic(err) + } +} + +// Teardown runs the teardown function from the provider and other +// procedures after running the test suite +func (ts *EndToEndTestSuite) Teardown() { + if err := ts.teardown(); err != nil { + panic(err) + } +} + +// ShouldSkipTest returns true if a provider wants to skip running a particular test +func (ts *EndToEndTestSuite) ShouldSkipTest(testName string) bool { + return ts.shouldSkipTest(testName) +} + +// Run runs tests registered in the test suite +func (ts *EndToEndTestSuite) Run(t *testing.T) { + suite.Run(t, ts) +} + +// NewEndToEndTestSuite returns a new EndToEndTestSuite given a test suite configuration, +// setup, and teardown functions from provider +func NewEndToEndTestSuite(cfg EndToEndTestSuiteConfig) *EndToEndTestSuite { + if cfg.Namespace == "" { + panic("Empty namespace") + } else if cfg.NodeName == "" { + panic("Empty node name") + } + + if cfg.WatchTimeout == time.Duration(0) { + cfg.WatchTimeout = defaultWatchTimeout + } + + f = framework.NewTestingFramework(cfg.Kubeconfig, cfg.Namespace, cfg.NodeName, cfg.WatchTimeout) + + emptyFunc := func() error { return nil } + if cfg.Setup == nil { + cfg.Setup = emptyFunc + } + if cfg.Teardown == nil { + cfg.Teardown = emptyFunc + } + if cfg.ShouldSkipTest == nil { + // This will not skip any test in the test suite + cfg.ShouldSkipTest = func(_ string) bool { return false } + } + + return &EndToEndTestSuite{ + setup: cfg.Setup, + teardown: cfg.Teardown, + shouldSkipTest: cfg.ShouldSkipTest, + } +}