Importable End-To-End Test Suite (#758)

* Rename VK to chewong for development purpose

* Rename basic_test.go to basic.go

* Add e2e.go and suite.go

* Disable tests in node.go

* End to end tests are now importable as a testing suite

* Remove 'test' from test files

* Add documentations

* Rename chewong back to virtual-kubelet

* Change 'Testing Suite' to 'Test Suite'

* Add the ability to skip certain testss

* Add unit tests for suite.go

* Add README.md for importable e2e test suite

* VK implementation has to be based on VK v1.0.0

* Stricter checks on validating test functions

* Move certain files back to internal folder

* Add WatchTimeout as a config field

* Add slight modifications

internal/test/suite/suite.go

@@ -0,0 +1,85 @@
+package suite
+
+import (
+	"reflect"
+	"runtime/debug"
+	"strings"
+	"testing"
+)
+
+// TestFunc defines the test function in a test case
+type TestFunc func(*testing.T)
+
+// SetUpFunc sets up provider-specific resource in the test suite
+type SetUpFunc func() error
+
+// TeardownFunc tears down provider-specific resources from the test suite
+type TeardownFunc func() error
+
+// ShouldSkipTestFunc determines whether the test suite should skip certain tests
+type ShouldSkipTestFunc func(string) bool
+
+// TestSuite contains methods that defines the lifecycle of a test suite
+type TestSuite interface {
+	Setup()
+	Teardown()
+}
+
+// TestSkipper allows providers to skip certain tests
+type TestSkipper interface {
+	ShouldSkipTest(string) bool
+}
+
+type testCase struct {
+	name string
+	f    TestFunc
+}
+
+// Run runs tests registered in the test suite
+func Run(t *testing.T, ts TestSuite) {
+	defer failOnPanic(t)
+
+	ts.Setup()
+	defer ts.Teardown()
+
+	// The implementation below is based on https://github.com/stretchr/testify
+	testFinder := reflect.TypeOf(ts)
+	tests := []testCase{}
+	for i := 0; i < testFinder.NumMethod(); i++ {
+		method := testFinder.Method(i)
+		if !isValidTestFunc(method) {
+			continue
+		}
+
+		test := testCase{
+			name: method.Name,
+			f: func(t *testing.T) {
+				defer failOnPanic(t)
+				if tSkipper, ok := ts.(TestSkipper); ok && tSkipper.ShouldSkipTest(method.Name) {
+					t.Skipf("Skipped due to shouldSkipTest()")
+				}
+				method.Func.Call([]reflect.Value{reflect.ValueOf(ts), reflect.ValueOf(t)})
+			},
+		}
+		tests = append(tests, test)
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, test.f)
+	}
+}
+
+// failOnPanic recovers panic occurred in the test suite and marks the test / test suite as failed
+func failOnPanic(t *testing.T) {
+	if r := recover(); r != nil {
+		t.Fatalf("%v\n%s", r, debug.Stack())
+	}
+}
+
+// isValidTestFunc determines whether or not a given method is a valid test function
+func isValidTestFunc(method reflect.Method) bool {
+	return strings.HasPrefix(method.Name, "Test") && // Test function name must start with "Test",
+		method.Type.NumIn() == 2 && // the number of function input should be 2 (*TestSuite ts and t *testing.T),
+		method.Type.In(1) == reflect.TypeOf(&testing.T{}) &&
+		method.Type.NumOut() == 0 // and the number of function output should be 0
+}

internal/test/suite/suite_test.go

@@ -0,0 +1,126 @@
+package suite
+
+import (
+	"strings"
+	"testing"
+
+	"gotest.tools/assert"
+	is "gotest.tools/assert/cmp"
+)
+
+type basicTestSuite struct {
+	setupCount      int
+	testFooCount    int
+	testBarCount    int
+	bazCount        int
+	testFooBarCount int
+	testFooBazCount int
+	testBarBazCount int
+	teardownCount   int
+	testsRan        []string
+}
+
+func (bts *basicTestSuite) Setup() {
+	bts.setupCount++
+}
+
+func (bts *basicTestSuite) Teardown() {
+	bts.teardownCount++
+}
+
+func (bts *basicTestSuite) TestFoo(t *testing.T) {
+	bts.testFooCount++
+	bts.testsRan = append(bts.testsRan, t.Name())
+}
+
+func (bts *basicTestSuite) TestBar(t *testing.T) {
+	bts.testBarCount++
+	bts.testsRan = append(bts.testsRan, t.Name())
+}
+
+// Baz should not be executed by the test suite
+// because it does not have the prefix 'Test'
+func (bts *basicTestSuite) Baz(t *testing.T) {
+	bts.bazCount++
+	bts.testsRan = append(bts.testsRan, t.Name())
+}
+
+// TestFooBar should not be executed by the test suite
+// because the number of function input is not 2 (*basicTestSuite and *testing.T)
+func (bts *basicTestSuite) TestFooBar() {
+	bts.testFooBarCount++
+	bts.testsRan = append(bts.testsRan, "TestFooBar")
+}
+
+// TestFooBaz should not be executed by the test suite
+// because the number of function output is not 0
+func (bts *basicTestSuite) TestFooBaz(t *testing.T) error {
+	bts.testFooBazCount++
+	bts.testsRan = append(bts.testsRan, t.Name())
+	return nil
+}
+
+// TestBarBaz should not be executed by the test suite
+// because the type of the function input is not *testing.T
+func (bts *basicTestSuite) TestBarBaz(t string) {
+	bts.testBarBazCount++
+	bts.testsRan = append(bts.testsRan, "TestBarBaz")
+}
+
+func TestBasicTestSuite(t *testing.T) {
+	bts := new(basicTestSuite)
+	Run(t, bts)
+
+	assert.Equal(t, bts.setupCount, 1)
+	assert.Equal(t, bts.testFooCount, 1)
+	assert.Equal(t, bts.testBarCount, 1)
+	assert.Equal(t, bts.teardownCount, 1)
+	assert.Assert(t, is.Len(bts.testsRan, 2))
+	assertTestsRan(t, bts.testsRan)
+	assertNonTests(t, bts)
+}
+
+type skipTestSuite struct {
+	basicTestSuite
+	skippedTestCount int
+}
+
+func (sts *skipTestSuite) ShouldSkipTest(testName string) bool {
+	if testName == "TestBar" {
+		sts.skippedTestCount++
+		return true
+	}
+	return false
+}
+
+func TestSkipTest(t *testing.T) {
+	sts := new(skipTestSuite)
+	Run(t, sts)
+
+	assert.Equal(t, sts.setupCount, 1)
+	assert.Equal(t, sts.testFooCount, 1)
+	assert.Equal(t, sts.testBarCount, 0)
+	assert.Equal(t, sts.teardownCount, 1)
+	assert.Equal(t, sts.skippedTestCount, 1)
+	assert.Assert(t, is.Len(sts.testsRan, 1))
+	assertTestsRan(t, sts.testsRan)
+	assertNonTests(t, &sts.basicTestSuite)
+}
+
+func assertTestsRan(t *testing.T, testsRan []string) {
+	for _, testRan := range testsRan {
+		parts := strings.Split(testRan, "/")
+		// Make sure that the name of the test has exactly one parent name and one subtest name
+		assert.Assert(t, is.Len(parts, 2))
+		// Check the parent test's name
+		assert.Equal(t, parts[0], t.Name())
+	}
+}
+
+// assertNonTests ensures that any malformed test functions are not run by the test suite
+func assertNonTests(t *testing.T, bts *basicTestSuite) {
+	assert.Equal(t, bts.bazCount, 0)
+	assert.Equal(t, bts.testFooBarCount, 0)
+	assert.Equal(t, bts.testFooBazCount, 0)
+	assert.Equal(t, bts.testBarBazCount, 0)
+}