d8dfun/virtual-kubelet
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

tests: introduce e2e suite (#422)

Browse Source 
* mock: implement GetStatsSummary

Signed-off-by: Paulo Pires <pjpires@gmail.com>

* make: use skaffold to deploy vk

Signed-off-by: Paulo Pires <pjpires@gmail.com>

* test: add an e2e test suite

Signed-off-by: Paulo Pires <pjpires@gmail.com>

* test: add vendored code

Signed-off-by: Paulo Pires <pjpires@gmail.com>

* docs: update README.md

Signed-off-by: Paulo Pires <pjpires@gmail.com>

* ci: run e2e on circleci

Signed-off-by: Paulo Pires <pjpires@gmail.com>

* make: improve the skaffold target

Signed-off-by: Paulo Pires <pjpires@gmail.com>

* e2e: fix defer pod deletion

Signed-off-by: Paulo Pires <pjpires@gmail.com>

* e2e: improve instructions

Signed-off-by: Paulo Pires <pjpires@gmail.com>

* makefile: default shell is bash

Signed-off-by: Paulo Pires <pjpires@gmail.com>
This commit is contained in:
Paulo Pires
2018-11-28 17:01:36 +00:00
committed by Brian Goff
parent 688c10fa8b
commit 579823e6a5
22 changed files with 1487 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

144
test/e2e/basic_test.go Normal file
View File

@@ -0,0 +1,144 @@
// +build e2e
package e2e
import (
"fmt"
"testing"
"k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
)
// TestGetStatsSummary creates a pod having two containers and queries the /stats/summary endpoint of the virtual-kubelet.
// It expects this endpoint to return stats for the current node, as well as for the aforementioned pod and each of its two containers.
func TestGetStatsSummary(t *testing.T) {
// Create a pod with prefix "nginx-0-" having three containers.
pod, err := f.CreatePod(f.CreateDummyPodObjectWithPrefix("nginx-0-", "foo", "bar", "baz"))
if err != nil {
t.Fatal(err)
}
// Delete the "nginx-0-X" pod after the test finishes.
defer func() {
if err := f.DeletePod(pod.Namespace, pod.Name); err != nil && !apierrors.IsNotFound(err) {
t.Error(err)
}
}()
// Wait for the "nginx-0-X" pod to be reported as running and ready.
if err := f.WaitUntilPodReady(pod.Namespace, pod.Name); err != nil {
t.Fatal(err)
}
// Grab the stats from the provider.
stats, err := f.GetStatsSummary()
if err != nil {
t.Fatal(err)
}
// Make sure that we've got stats for the current node.
if stats.Node.NodeName != f.NodeName {
t.Fatalf("expected stats for node %s, got stats for node %s", f.NodeName, stats.Node.NodeName)
}
// Make sure the "nginx-0-X" pod exists in the slice of PodStats.
idx, err := findPodInPodStats(stats, pod)
if err != nil {
t.Fatal(err)
}
// Make sure that we've got stats for all the containers in the "nginx-0-X" pod.
desiredContainerStatsCount := len(pod.Spec.Containers)
currentContainerStatsCount := len(stats.Pods[idx].Containers)
if currentContainerStatsCount != desiredContainerStatsCount {
t.Fatalf("expected stats for %d containers, got stats for %d containers", desiredContainerStatsCount, currentContainerStatsCount)
}
}
// TestPodLifecycle creates two pods and verifies that the provider has been asked to create them.
// Then, it deletes one of the pods and verifies that the provider has been asked to delete it.
// These verifications are made using the /stats/summary endpoint of the virtual-kubelet, by checking for the presence or absence of the pods.
// Hence, the provider being tested must implement the PodMetricsProvider interface.
func TestPodLifecycle(t *testing.T) {
// Create a pod with prefix "nginx-0-" having a single container.
pod0, err := f.CreatePod(f.CreateDummyPodObjectWithPrefix("nginx-0-", "foo"))
if err != nil {
t.Fatal(err)
}
// Delete the "nginx-0-X" pod after the test finishes.
defer func() {
if err := f.DeletePod(pod0.Namespace, pod0.Name); err != nil && !apierrors.IsNotFound(err) {
t.Error(err)
}
}()
// Create a pod with prefix "nginx-1-" having a single container.
pod1, err := f.CreatePod(f.CreateDummyPodObjectWithPrefix("nginx-1-", "bar"))
if err != nil {
t.Fatal(err)
}
// Delete the "nginx-1-Y" pod after the test finishes.
defer func() {
if err := f.DeletePod(pod1.Namespace, pod1.Name); err != nil && !apierrors.IsNotFound(err) {
t.Error(err)
}
}()
// Wait for the "nginx-0-X" pod to be reported as running and ready.
if err := f.WaitUntilPodReady(pod0.Namespace, pod0.Name); err != nil {
t.Fatal(err)
}
// Wait for the "nginx-1-Y" pod to be reported as running and ready.
if err := f.WaitUntilPodReady(pod1.Namespace, pod1.Name); err != nil {
t.Fatal(err)
}
// Grab the stats from the provider.
stats, err := f.GetStatsSummary()
if err != nil {
t.Fatal(err)
}
// Make sure the "nginx-0-X" pod exists in the slice of PodStats.
if _, err := findPodInPodStats(stats, pod0); err != nil {
t.Fatal(err)
}
// Make sure the "nginx-1-Y" pod exists in the slice of PodStats.
if _, err := findPodInPodStats(stats, pod1); err != nil {
t.Fatal(err)
}
// Delete the "nginx-1" pod.
if err := f.DeletePod(pod1.Namespace, pod1.Name); err != nil {
t.Fatal(err)
}
// Wait for the "nginx-1-Y" pod to be reported as having been marked for deletion.
if err := f.WaitUntilPodDeleted(pod1.Namespace, pod1.Name); err != nil {
t.Fatal(err)
}
// Grab the stats from the provider.
stats, err = f.GetStatsSummary()
if err != nil {
t.Fatal(err)
}
// Make sure the "nginx-1-Y" pod DOES NOT exist in the slice of PodStats anymore.
if _, err := findPodInPodStats(stats, pod1); err == nil {
t.Fatalf("expected to NOT find pod \"%s/%s\" in the slice of pod stats", pod1.Namespace, pod1.Name)
}
}
// findPodInPodStats returns the index of the specified pod in the .pods field of the specified Summary object.
// It returns an error if the specified pod is not found.
func findPodInPodStats(summary *v1alpha1.Summary, pod *v1.Pod) (int, error) {
for i, p := range summary.Pods {
if p.PodRef.Namespace == pod.Namespace && p.PodRef.Name == pod.Name && string(p.PodRef.UID) == string(pod.UID) {
return i, nil
}
}
return -1, fmt.Errorf("failed to find pod \"%s/%s\" in the slice of pod stats", pod.Namespace, pod.Name)
}

47
test/e2e/framework/framework.go Normal file
View File

@@ -0,0 +1,47 @@
package framework
import (
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
)
// Framework encapsulates the configuration for the current run, and provides helper methods to be used during testing.
type Framework struct {
KubeClient kubernetes.Interface
Namespace string
NodeName string
TaintKey string
TaintValue string
TaintEffect string
}
// NewTestingFramework returns a new instance of the testing framework.
func NewTestingFramework(kubeconfig, namespace, nodeName, taintKey, taintValue, taintEffect string) *Framework {
return &Framework{
KubeClient: createKubeClient(kubeconfig),
Namespace: namespace,
NodeName: nodeName,
TaintKey: taintKey,
TaintValue: taintValue,
TaintEffect: taintEffect,
}
}
// createKubeClient creates a new Kubernetes client based on the specified kubeconfig file.
// If no value for kubeconfig is specified, in-cluster configuration is assumed.
func createKubeClient(kubeconfig string) *kubernetes.Clientset {
var (
cfg *rest.Config
err error
)
if kubeconfig == "" {
cfg, err = rest.InClusterConfig()
} else {
cfg, err = clientcmd.BuildConfigFromFlags("", kubeconfig)
}
if err != nil {
panic(err)
}
return kubernetes.NewForConfigOrDie(cfg)
}

108
test/e2e/framework/pod.go Normal file
View File

@@ -0,0 +1,108 @@
package framework
import (
"context"
"fmt"
"time"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/runtime"
watchapi "k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/tools/watch"
podutil "k8s.io/kubernetes/pkg/api/v1/pod"
)
const (
defaultWatchTimeout = 2 * time.Minute
hostnameNodeSelectorLabel = "kubernetes.io/hostname"
)
// CreateDummyPodObjectWithPrefix creates a dujmmy pod object using the specified prefix as the value of .metadata.generateName.
// A variable number of strings can be provided.
// For each one of these strings, a container that uses the string as its image will be appended to the pod.
// This method DOES NOT create the pod in the Kubernetes API.
func (f *Framework) CreateDummyPodObjectWithPrefix(prefix string, images ...string) *corev1.Pod {
pod := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
GenerateName: prefix,
Namespace: f.Namespace,
},
Spec: corev1.PodSpec{
NodeSelector: map[string]string{
hostnameNodeSelectorLabel: f.NodeName,
},
Tolerations: []corev1.Toleration{
{
Key: f.TaintKey,
Value: f.TaintValue,
Effect: corev1.TaintEffect(f.TaintEffect),
},
},
},
}
for idx, img := range images {
pod.Spec.Containers = append(pod.Spec.Containers, corev1.Container{
Name: fmt.Sprintf("%s%d", prefix, idx),
Image: img,
})
}
return pod
}
// CreatePod creates the specified pod in the Kubernetes API.
func (f *Framework) CreatePod(pod *corev1.Pod) (*corev1.Pod, error) {
return f.KubeClient.CoreV1().Pods(f.Namespace).Create(pod)
}
// DeletePod deletes the pod with the specified name and namespace in the Kubernetes API.
func (f *Framework) DeletePod(namespace, name string) error {
return f.KubeClient.CoreV1().Pods(namespace).Delete(name, &metav1.DeleteOptions{})
}
// WaitUntilPodCondition establishes a watch on the pod with the specified name and namespace.
// Then, it waits for the specified condition function to be verified.
func (f *Framework) WaitUntilPodCondition(namespace, name string, fn watch.ConditionFunc) error {
// Create a field selector that matches the specified Pod resource.
fs := fields.ParseSelectorOrDie(fmt.Sprintf("metadata.namespace==%s,metadata.name==%s", namespace, name))
// Create a ListWatch so we can receive events for the matched Pod resource.
lw := &cache.ListWatch{
ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
options.FieldSelector = fs.String()
return f.KubeClient.CoreV1().Pods(namespace).List(options)
},
WatchFunc: func(options metav1.ListOptions) (watchapi.Interface, error) {
options.FieldSelector = fs.String()
return f.KubeClient.CoreV1().Pods(namespace).Watch(options)
},
}
// Watch for updates to the Pod resource until fn is satisfied, or until the timeout is reached.
ctx, cfn := context.WithTimeout(context.Background(), defaultWatchTimeout)
defer cfn()
last, err := watch.UntilWithSync(ctx, lw, &corev1.Pod{}, nil, fn)
if err != nil {
return err
}
if last == nil {
return fmt.Errorf("no events received for pod %q", name)
}
return nil
}
// WaitUntilPodReady blocks until the pod with the specified name and namespace is reported to be running and ready.
func (f *Framework) WaitUntilPodReady(namespace, name string) error {
return f.WaitUntilPodCondition(namespace, name, func(event watchapi.Event) (bool, error) {
pod := event.Object.(*corev1.Pod)
return pod.Status.Phase == corev1.PodRunning && podutil.IsPodReady(pod) && pod.Status.PodIP != "", nil
})
}
// WaitUntilPodDeleted blocks until the pod with the specified name and namespace is marked for deletion (or, alternatively, effectively deleted).
func (f *Framework) WaitUntilPodDeleted(namespace, name string) error {
return f.WaitUntilPodCondition(namespace, name, func(event watchapi.Event) (bool, error) {
pod := event.Object.(*corev1.Pod)
return event.Type == watchapi.Deleted || pod.DeletionTimestamp != nil, nil
})
}

31
test/e2e/framework/stats.go Normal file
View File

@@ -0,0 +1,31 @@
package framework
import (
"encoding/json"
"strconv"
"k8s.io/apimachinery/pkg/util/net"
stats "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
)
// GetStatsSummary queries the /stats/summary endpoint of the virtual-kubelet and returns the Summary object obtained as a response.
func (f *Framework) GetStatsSummary() (*stats.Summary, error) {
// Query the /stats/summary endpoint.
b, err := f.KubeClient.CoreV1().
RESTClient().
Get().
Namespace(f.Namespace).
Resource("pods").
SubResource("proxy").
Name(net.JoinSchemeNamePort("http", f.NodeName, strconv.Itoa(10255))).
Suffix("/stats/summary").DoRaw()
if err != nil {
return nil, err
}
// Unmarshal the response as a Summary object and return it.
res := &stats.Summary{}
if err := json.Unmarshal(b, res); err != nil {
return nil, err
}
return res, nil
}

81
test/e2e/main_test.go Normal file
View File

@@ -0,0 +1,81 @@
// +build e2e
package e2e
import (
"flag"
"os"
"testing"
"k8s.io/api/core/v1"
"github.com/virtual-kubelet/virtual-kubelet/test/e2e/framework"
)
const (
defaultNamespace = v1.NamespaceDefault
defaultNodeName = "vkubelet-mock-0"
defaultTaintKey = "virtual-kubelet.io/provider"
defaultTaintValue = "mock"
defaultTaintEffect = string(v1.TaintEffectNoSchedule)
)
var (
// f is the testing framework used for running the test suite.
f *framework.Framework
// kubeconfig is the path to the kubeconfig file to use when running the test suite outside a Kubernetes cluster.
kubeconfig string
// namespace is the name of the Kubernetes namespace to use for running the test suite (i.e. where to create pods).
namespace string
// nodeName is the name of the virtual-kubelet node to test.
nodeName string
// taintKey is the key of the taint that is expected to be associated with the virtual-kubelet node to test.
taintKey string
// taintValue is the value of the taint that is expected to be associated with the virtual-kubelet node to test.
taintValue string
// taintEffect is the effect of the taint that is expected to be associated with the virtual-kubelet node to test.
taintEffect string
)
func init() {
flag.StringVar(&kubeconfig, "kubeconfig", "", "path to the kubeconfig file to use when running the test suite outside a kubernetes cluster")
flag.StringVar(&namespace, "namespace", defaultNamespace, "the name of the kubernetes namespace to use for running the test suite (i.e. where to create pods)")
flag.StringVar(&nodeName, "node-name", defaultNodeName, "the name of the virtual-kubelet node to test")
flag.StringVar(&taintKey, "taint-key", defaultTaintKey, "the key of the taint that is expected to be associated with the virtual-kubelet node to test")
flag.StringVar(&taintValue, "taint-value", defaultTaintValue, "the value of the taint that is expected to be associated with the virtual-kubelet node to test")
flag.StringVar(&taintEffect, "taint-effect", defaultTaintEffect, "the effect of the taint that is expected to be associated with the virtual-kubelet node to test")
flag.Parse()
}
func TestMain(m *testing.M) {
// Set sane defaults in case no values (or empty ones) have been provided.
setDefaults()
// Create a new instance of the test framework targeting the specified node.
f = framework.NewTestingFramework(kubeconfig, namespace, nodeName, taintKey, taintValue, taintEffect)
// Wait for the virtual-kubelet pod to be ready.
if err := f.WaitUntilPodReady(namespace, nodeName); err != nil {
panic(err)
}
// Run the test suite.
os.Exit(m.Run())
}
// setDefaults sets sane defaults in case no values (or empty ones) have been provided.
func setDefaults() {
if namespace == "" {
namespace = defaultNamespace
}
if nodeName == "" {
nodeName = defaultNodeName
}
if taintKey == "" {
taintKey = defaultTaintKey
}
if taintValue == "" {
taintValue = defaultTaintValue
}
if taintEffect == "" {
taintEffect = defaultTaintEffect
}
}