tests: introduce e2e suite (#422)

* mock: implement GetStatsSummary

Signed-off-by: Paulo Pires <pjpires@gmail.com>

* make: use skaffold to deploy vk

Signed-off-by: Paulo Pires <pjpires@gmail.com>

* test: add an e2e test suite

Signed-off-by: Paulo Pires <pjpires@gmail.com>

* test: add vendored code

Signed-off-by: Paulo Pires <pjpires@gmail.com>

* docs: update README.md

Signed-off-by: Paulo Pires <pjpires@gmail.com>

* ci: run e2e on circleci

Signed-off-by: Paulo Pires <pjpires@gmail.com>

* make: improve the skaffold target

Signed-off-by: Paulo Pires <pjpires@gmail.com>

* e2e: fix defer pod deletion

Signed-off-by: Paulo Pires <pjpires@gmail.com>

* e2e: improve instructions

Signed-off-by: Paulo Pires <pjpires@gmail.com>

* makefile: default shell is bash

Signed-off-by: Paulo Pires <pjpires@gmail.com>

test/e2e/framework/framework.go

@@ -0,0 +1,47 @@
+package framework
+
+import (
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+)
+
+// Framework encapsulates the configuration for the current run, and provides helper methods to be used during testing.
+type Framework struct {
+	KubeClient  kubernetes.Interface
+	Namespace   string
+	NodeName    string
+	TaintKey    string
+	TaintValue  string
+	TaintEffect string
+}
+
+// NewTestingFramework returns a new instance of the testing framework.
+func NewTestingFramework(kubeconfig, namespace, nodeName, taintKey, taintValue, taintEffect string) *Framework {
+	return &Framework{
+		KubeClient:  createKubeClient(kubeconfig),
+		Namespace:   namespace,
+		NodeName:    nodeName,
+		TaintKey:    taintKey,
+		TaintValue:  taintValue,
+		TaintEffect: taintEffect,
+	}
+}
+
+// createKubeClient creates a new Kubernetes client based on the specified kubeconfig file.
+// If no value for kubeconfig is specified, in-cluster configuration is assumed.
+func createKubeClient(kubeconfig string) *kubernetes.Clientset {
+	var (
+		cfg *rest.Config
+		err error
+	)
+	if kubeconfig == "" {
+		cfg, err = rest.InClusterConfig()
+	} else {
+		cfg, err = clientcmd.BuildConfigFromFlags("", kubeconfig)
+	}
+	if err != nil {
+		panic(err)
+	}
+	return kubernetes.NewForConfigOrDie(cfg)
+}

test/e2e/framework/pod.go

@@ -0,0 +1,108 @@
+package framework
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/runtime"
+	watchapi "k8s.io/apimachinery/pkg/watch"
+	"k8s.io/client-go/tools/cache"
+	"k8s.io/client-go/tools/watch"
+	podutil "k8s.io/kubernetes/pkg/api/v1/pod"
+)
+
+const (
+	defaultWatchTimeout       = 2 * time.Minute
+	hostnameNodeSelectorLabel = "kubernetes.io/hostname"
+)
+
+// CreateDummyPodObjectWithPrefix creates a dujmmy pod object using the specified prefix as the value of .metadata.generateName.
+// A variable number of strings can be provided.
+// For each one of these strings, a container that uses the string as its image will be appended to the pod.
+// This method DOES NOT create the pod in the Kubernetes API.
+func (f *Framework) CreateDummyPodObjectWithPrefix(prefix string, images ...string) *corev1.Pod {
+	pod := &corev1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			GenerateName: prefix,
+			Namespace:    f.Namespace,
+		},
+		Spec: corev1.PodSpec{
+			NodeSelector: map[string]string{
+				hostnameNodeSelectorLabel: f.NodeName,
+			},
+			Tolerations: []corev1.Toleration{
+				{
+					Key:    f.TaintKey,
+					Value:  f.TaintValue,
+					Effect: corev1.TaintEffect(f.TaintEffect),
+				},
+			},
+		},
+	}
+	for idx, img := range images {
+		pod.Spec.Containers = append(pod.Spec.Containers, corev1.Container{
+			Name:  fmt.Sprintf("%s%d", prefix, idx),
+			Image: img,
+		})
+	}
+	return pod
+}
+
+// CreatePod creates the specified pod in the Kubernetes API.
+func (f *Framework) CreatePod(pod *corev1.Pod) (*corev1.Pod, error) {
+	return f.KubeClient.CoreV1().Pods(f.Namespace).Create(pod)
+}
+
+// DeletePod deletes the pod with the specified name and namespace in the Kubernetes API.
+func (f *Framework) DeletePod(namespace, name string) error {
+	return f.KubeClient.CoreV1().Pods(namespace).Delete(name, &metav1.DeleteOptions{})
+}
+
+// WaitUntilPodCondition establishes a watch on the pod with the specified name and namespace.
+// Then, it waits for the specified condition function to be verified.
+func (f *Framework) WaitUntilPodCondition(namespace, name string, fn watch.ConditionFunc) error {
+	// Create a field selector that matches the specified Pod resource.
+	fs := fields.ParseSelectorOrDie(fmt.Sprintf("metadata.namespace==%s,metadata.name==%s", namespace, name))
+	// Create a ListWatch so we can receive events for the matched Pod resource.
+	lw := &cache.ListWatch{
+		ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
+			options.FieldSelector = fs.String()
+			return f.KubeClient.CoreV1().Pods(namespace).List(options)
+		},
+		WatchFunc: func(options metav1.ListOptions) (watchapi.Interface, error) {
+			options.FieldSelector = fs.String()
+			return f.KubeClient.CoreV1().Pods(namespace).Watch(options)
+		},
+	}
+	// Watch for updates to the Pod resource until fn is satisfied, or until the timeout is reached.
+	ctx, cfn := context.WithTimeout(context.Background(), defaultWatchTimeout)
+	defer cfn()
+	last, err := watch.UntilWithSync(ctx, lw, &corev1.Pod{}, nil, fn)
+	if err != nil {
+		return err
+	}
+	if last == nil {
+		return fmt.Errorf("no events received for pod %q", name)
+	}
+	return nil
+}
+
+// WaitUntilPodReady blocks until the pod with the specified name and namespace is reported to be running and ready.
+func (f *Framework) WaitUntilPodReady(namespace, name string) error {
+	return f.WaitUntilPodCondition(namespace, name, func(event watchapi.Event) (bool, error) {
+		pod := event.Object.(*corev1.Pod)
+		return pod.Status.Phase == corev1.PodRunning && podutil.IsPodReady(pod) && pod.Status.PodIP != "", nil
+	})
+}
+
+// WaitUntilPodDeleted blocks until the pod with the specified name and namespace is marked for deletion (or, alternatively, effectively deleted).
+func (f *Framework) WaitUntilPodDeleted(namespace, name string) error {
+	return f.WaitUntilPodCondition(namespace, name, func(event watchapi.Event) (bool, error) {
+		pod := event.Object.(*corev1.Pod)
+		return event.Type == watchapi.Deleted || pod.DeletionTimestamp != nil, nil
+	})
+}

test/e2e/framework/stats.go

@@ -0,0 +1,31 @@
+package framework
+
+import (
+	"encoding/json"
+	"strconv"
+
+	"k8s.io/apimachinery/pkg/util/net"
+	stats "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
+)
+
+// GetStatsSummary queries the /stats/summary endpoint of the virtual-kubelet and returns the Summary object obtained as a response.
+func (f *Framework) GetStatsSummary() (*stats.Summary, error) {
+	// Query the /stats/summary endpoint.
+	b, err := f.KubeClient.CoreV1().
+		RESTClient().
+		Get().
+		Namespace(f.Namespace).
+		Resource("pods").
+		SubResource("proxy").
+		Name(net.JoinSchemeNamePort("http", f.NodeName, strconv.Itoa(10255))).
+		Suffix("/stats/summary").DoRaw()
+	if err != nil {
+		return nil, err
+	}
+	// Unmarshal the response as a Summary object and return it.
+	res := &stats.Summary{}
+	if err := json.Unmarshal(b, res); err != nil {
+		return nil, err
+	}
+	return res, nil
+}