Move test/* to internal/test/*

This ensures this code cannot be imported by other repositories as this
is only intended for internal testing packages.

internal/test/e2e/framework/env.go

@@ -0,0 +1,81 @@
+package framework
+
+import (
+	corev1 "k8s.io/api/core/v1"
+)
+
+var (
+	bFalse = false
+	bTrue  = true
+)
+
+// CreatePodObjectWithMandatoryConfigMapKey creates a pod object that references the "key_0" key from the "config-map-0" config map as mandatory.
+func (f *Framework) CreatePodObjectWithMandatoryConfigMapKey(testName string) *corev1.Pod {
+	return f.CreatePodObjectWithEnv(testName, []corev1.EnvVar{
+		{
+			Name: "CONFIG_MAP_0_KEY_0",
+			ValueFrom: &corev1.EnvVarSource{
+				ConfigMapKeyRef: &corev1.ConfigMapKeySelector{
+					LocalObjectReference: corev1.LocalObjectReference{Name: "config-map-0"},
+					Key:                  "key_0",
+					Optional:             &bFalse,
+				},
+			},
+		},
+	})
+}
+
+// CreatePodObjectWithOptionalConfigMapKey creates a pod object that references the "key_0" key from the "config-map-0" config map as optional.
+func (f *Framework) CreatePodObjectWithOptionalConfigMapKey(testName string) *corev1.Pod {
+	return f.CreatePodObjectWithEnv(testName, []corev1.EnvVar{
+		{
+			Name: "CONFIG_MAP_0_KEY_0",
+			ValueFrom: &corev1.EnvVarSource{
+				ConfigMapKeyRef: &corev1.ConfigMapKeySelector{
+					LocalObjectReference: corev1.LocalObjectReference{Name: "config-map-0"},
+					Key:                  "key_0",
+					Optional:             &bTrue,
+				},
+			},
+		},
+	})
+}
+
+// CreatePodObjectWithMandatorySecretKey creates a pod object that references the "key_0" key from the "secret-0" config map as mandatory.
+func (f *Framework) CreatePodObjectWithMandatorySecretKey(testName string) *corev1.Pod {
+	return f.CreatePodObjectWithEnv(testName, []corev1.EnvVar{
+		{
+			Name: "SECRET_0_KEY_0",
+			ValueFrom: &corev1.EnvVarSource{
+				SecretKeyRef: &corev1.SecretKeySelector{
+					LocalObjectReference: corev1.LocalObjectReference{Name: "secret-0"},
+					Key:                  "key_0",
+					Optional:             &bFalse,
+				},
+			},
+		},
+	})
+}
+
+// CreatePodObjectWithOptionalSecretKey creates a pod object that references the "key_0" key from the "secret-0" config map as optional.
+func (f *Framework) CreatePodObjectWithOptionalSecretKey(testName string) *corev1.Pod {
+	return f.CreatePodObjectWithEnv(testName, []corev1.EnvVar{
+		{
+			Name: "SECRET_0_KEY_0",
+			ValueFrom: &corev1.EnvVarSource{
+				SecretKeyRef: &corev1.SecretKeySelector{
+					LocalObjectReference: corev1.LocalObjectReference{Name: "secret-0"},
+					Key:                  "key_0",
+					Optional:             &bTrue,
+				},
+			},
+		},
+	})
+}
+
+// CreatePodObjectWithEnv creates a pod object whose name starts with "env-test-" and that uses the specified environment configuration for its first container.
+func (f *Framework) CreatePodObjectWithEnv(testName string, env []corev1.EnvVar) *corev1.Pod {
+	pod := f.CreateDummyPodObjectWithPrefix(testName, "env-test-", "foo")
+	pod.Spec.Containers[0].Env = env
+	return pod
+}

internal/test/e2e/framework/framework.go

@@ -0,0 +1,41 @@
+package framework
+
+import (
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+)
+
+// Framework encapsulates the configuration for the current run, and provides helper methods to be used during testing.
+type Framework struct {
+	KubeClient kubernetes.Interface
+	Namespace  string
+	NodeName   string
+}
+
+// NewTestingFramework returns a new instance of the testing framework.
+func NewTestingFramework(kubeconfig, namespace, nodeName string) *Framework {
+	return &Framework{
+		KubeClient: createKubeClient(kubeconfig),
+		Namespace:  namespace,
+		NodeName:   nodeName,
+	}
+}
+
+// createKubeClient creates a new Kubernetes client based on the specified kubeconfig file.
+// If no value for kubeconfig is specified, in-cluster configuration is assumed.
+func createKubeClient(kubeconfig string) *kubernetes.Clientset {
+	var (
+		cfg *rest.Config
+		err error
+	)
+	if kubeconfig == "" {
+		cfg, err = rest.InClusterConfig()
+	} else {
+		cfg, err = clientcmd.BuildConfigFromFlags("", kubeconfig)
+	}
+	if err != nil {
+		panic(err)
+	}
+	return kubernetes.NewForConfigOrDie(cfg)
+}

internal/test/e2e/framework/node.go

@@ -0,0 +1,60 @@
+package framework
+
+import (
+	"context"
+	"fmt"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/runtime"
+	watchapi "k8s.io/apimachinery/pkg/watch"
+	"k8s.io/client-go/tools/cache"
+	"k8s.io/client-go/tools/watch"
+)
+
+// WaitUntilNodeCondition establishes a watch on the vk node.
+// Then, it waits for the specified condition function to be verified.
+func (f *Framework) WaitUntilNodeCondition(fn watch.ConditionFunc) error {
+	// Create a field selector that matches the specified Pod resource.
+	fs := fields.OneTermEqualSelector("metadata.name", f.NodeName).String()
+	// Create a ListWatch so we can receive events for the matched Pod resource.
+	lw := &cache.ListWatch{
+		ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
+			options.FieldSelector = fs
+			return f.KubeClient.CoreV1().Nodes().List(options)
+		},
+		WatchFunc: func(options metav1.ListOptions) (watchapi.Interface, error) {
+			options.FieldSelector = fs
+			return f.KubeClient.CoreV1().Nodes().Watch(options)
+		},
+	}
+
+	// Watch for updates to the Pod resource until fn is satisfied, or until the timeout is reached.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultWatchTimeout)
+	defer cancel()
+	last, err := watch.UntilWithSync(ctx, lw, &corev1.Node{}, nil, fn)
+	if err != nil {
+		return err
+	}
+	if last == nil {
+		return fmt.Errorf("no events received for node %q", f.NodeName)
+	}
+	return nil
+}
+
+// DeleteNode deletes the vk node used by the framework
+func (f *Framework) DeleteNode() error {
+	var gracePeriod int64
+	propagation := metav1.DeletePropagationBackground
+	opts := metav1.DeleteOptions{
+		PropagationPolicy:  &propagation,
+		GracePeriodSeconds: &gracePeriod,
+	}
+	return f.KubeClient.CoreV1().Nodes().Delete(f.NodeName, &opts)
+}
+
+// GetNode gets the vk nodeused by the framework
+func (f *Framework) GetNode() (*corev1.Node, error) {
+	return f.KubeClient.CoreV1().Nodes().Get(f.NodeName, metav1.GetOptions{})
+}

internal/test/e2e/framework/pod.go

@@ -0,0 +1,186 @@
+package framework
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/runtime"
+	watchapi "k8s.io/apimachinery/pkg/watch"
+	"k8s.io/client-go/tools/cache"
+	"k8s.io/client-go/tools/watch"
+	podutil "k8s.io/kubernetes/pkg/api/v1/pod"
+)
+
+const defaultWatchTimeout = 2 * time.Minute
+
+// CreateDummyPodObjectWithPrefix creates a dujmmy pod object using the specified prefix as the value of .metadata.generateName.
+// A variable number of strings can be provided.
+// For each one of these strings, a container that uses the string as its image will be appended to the pod.
+// This method DOES NOT create the pod in the Kubernetes API.
+func (f *Framework) CreateDummyPodObjectWithPrefix(testName string, prefix string, images ...string) *corev1.Pod {
+	// Safe the test name
+	if testName != "" {
+		testName = strings.Replace(testName, "/", "-", -1)
+		testName = strings.ToLower(testName)
+		prefix = prefix + "-" + testName + "-"
+	}
+	enableServiceLink := false
+
+	pod := &corev1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			GenerateName: prefix,
+			Namespace:    f.Namespace,
+		},
+		Spec: corev1.PodSpec{
+			NodeName:           f.NodeName,
+			EnableServiceLinks: &enableServiceLink,
+		},
+	}
+	for idx, img := range images {
+		pod.Spec.Containers = append(pod.Spec.Containers, corev1.Container{
+			Name:  fmt.Sprintf("%s%d", prefix, idx),
+			Image: img,
+		})
+	}
+	return pod
+}
+
+// CreatePod creates the specified pod in the Kubernetes API.
+func (f *Framework) CreatePod(pod *corev1.Pod) (*corev1.Pod, error) {
+	return f.KubeClient.CoreV1().Pods(f.Namespace).Create(pod)
+}
+
+// DeletePod deletes the pod with the specified name and namespace in the Kubernetes API using the default grace period.
+func (f *Framework) DeletePod(namespace, name string) error {
+	return f.KubeClient.CoreV1().Pods(namespace).Delete(name, &metav1.DeleteOptions{})
+}
+
+// DeletePodImmediately forcibly deletes the pod with the specified name and namespace in the Kubernetes API.
+// This is equivalent to running "kubectl delete --force --grace-period 0 --namespace <namespace> pod <name>".
+func (f *Framework) DeletePodImmediately(namespace, name string) error {
+	grace := int64(0)
+	propagation := metav1.DeletePropagationBackground
+	return f.KubeClient.CoreV1().Pods(namespace).Delete(name, &metav1.DeleteOptions{
+		GracePeriodSeconds: &grace,
+		PropagationPolicy:  &propagation,
+	})
+}
+
+// WaitUntilPodCondition establishes a watch on the pod with the specified name and namespace.
+// Then, it waits for the specified condition function to be verified.
+func (f *Framework) WaitUntilPodCondition(namespace, name string, fn watch.ConditionFunc) (*corev1.Pod, error) {
+	// Create a field selector that matches the specified Pod resource.
+	fs := fields.ParseSelectorOrDie(fmt.Sprintf("metadata.namespace==%s,metadata.name==%s", namespace, name))
+	// Create a ListWatch so we can receive events for the matched Pod resource.
+	lw := &cache.ListWatch{
+		ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
+			options.FieldSelector = fs.String()
+			return f.KubeClient.CoreV1().Pods(namespace).List(options)
+		},
+		WatchFunc: func(options metav1.ListOptions) (watchapi.Interface, error) {
+			options.FieldSelector = fs.String()
+			return f.KubeClient.CoreV1().Pods(namespace).Watch(options)
+		},
+	}
+	// Watch for updates to the Pod resource until fn is satisfied, or until the timeout is reached.
+	ctx, cfn := context.WithTimeout(context.Background(), defaultWatchTimeout)
+	defer cfn()
+	last, err := watch.UntilWithSync(ctx, lw, &corev1.Pod{}, nil, fn)
+	if err != nil {
+		return nil, err
+	}
+	if last == nil {
+		return nil, fmt.Errorf("no events received for pod %q", name)
+	}
+	pod := last.Object.(*corev1.Pod)
+	return pod, nil
+}
+
+// WaitUntilPodReady blocks until the pod with the specified name and namespace is reported to be running and ready.
+func (f *Framework) WaitUntilPodReady(namespace, name string) (*corev1.Pod, error) {
+	return f.WaitUntilPodCondition(namespace, name, func(event watchapi.Event) (bool, error) {
+		pod := event.Object.(*corev1.Pod)
+		return pod.Status.Phase == corev1.PodRunning && podutil.IsPodReady(pod) && pod.Status.PodIP != "", nil
+	})
+}
+
+// WaitUntilPodDeleted blocks until the pod with the specified name and namespace is deleted from apiserver.
+func (f *Framework) WaitUntilPodDeleted(namespace, name string) (*corev1.Pod, error) {
+	return f.WaitUntilPodCondition(namespace, name, func(event watchapi.Event) (bool, error) {
+		pod := event.Object.(*corev1.Pod)
+		return event.Type == watchapi.Deleted || pod.ObjectMeta.DeletionTimestamp != nil, nil
+	})
+}
+
+// WaitUntilPodInPhase blocks until the pod with the specified name and namespace is in one of the specified phases
+func (f *Framework) WaitUntilPodInPhase(namespace, name string, phases ...corev1.PodPhase) (*corev1.Pod, error) {
+	return f.WaitUntilPodCondition(namespace, name, func(event watchapi.Event) (bool, error) {
+		pod := event.Object.(*corev1.Pod)
+		for _, p := range phases {
+			if pod.Status.Phase == p {
+				return true, nil
+			}
+		}
+		return false, nil
+	})
+}
+
+// WaitUntilPodEventWithReason establishes a watch on events involving the specified pod.
+// Then, it waits for an event with the specified reason to be created/updated.
+func (f *Framework) WaitUntilPodEventWithReason(pod *corev1.Pod, reason string) error {
+	// Create a field selector that matches Event resources involving the specified pod.
+	fs := fields.ParseSelectorOrDie(fmt.Sprintf("involvedObject.kind==Pod,involvedObject.uid==%s", pod.UID))
+	// Create a ListWatch so we can receive events for the matched Event resource.
+	lw := &cache.ListWatch{
+		ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
+			options.FieldSelector = fs.String()
+			return f.KubeClient.CoreV1().Events(pod.Namespace).List(options)
+		},
+		WatchFunc: func(options metav1.ListOptions) (watchapi.Interface, error) {
+			options.FieldSelector = fs.String()
+			return f.KubeClient.CoreV1().Events(pod.Namespace).Watch(options)
+		},
+	}
+	// Watch for updates to the Event resource until fn is satisfied, or until the timeout is reached.
+	ctx, cfn := context.WithTimeout(context.Background(), defaultWatchTimeout)
+	defer cfn()
+	last, err := watch.UntilWithSync(ctx, lw, &corev1.Event{}, nil, func(event watchapi.Event) (b bool, e error) {
+		switch event.Type {
+		case watchapi.Error:
+			fallthrough
+		case watchapi.Deleted:
+			return false, fmt.Errorf("got event of unexpected type %q", event.Type)
+		default:
+			return event.Object.(*corev1.Event).Reason == reason, nil
+		}
+	})
+	if err != nil {
+		return err
+	}
+	if last == nil {
+		return fmt.Errorf("no events involving pod \"%s/%s\" have been seen", pod.Namespace, pod.Name)
+	}
+	return nil
+}
+
+// GetRunningPods gets the running pods from the provider of the virtual kubelet
+func (f *Framework) GetRunningPods() (*corev1.PodList, error) {
+	result := &corev1.PodList{}
+
+	err := f.KubeClient.CoreV1().
+		RESTClient().
+		Get().
+		Resource("nodes").
+		Name(f.NodeName).
+		SubResource("proxy").
+		Suffix("runningpods/").
+		Do().
+		Into(result)
+
+	return result, err
+}

internal/test/e2e/framework/stats.go

@@ -0,0 +1,31 @@
+package framework
+
+import (
+	"encoding/json"
+	"strconv"
+
+	"k8s.io/apimachinery/pkg/util/net"
+	stats "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
+)
+
+// GetStatsSummary queries the /stats/summary endpoint of the virtual-kubelet and returns the Summary object obtained as a response.
+func (f *Framework) GetStatsSummary() (*stats.Summary, error) {
+	// Query the /stats/summary endpoint.
+	b, err := f.KubeClient.CoreV1().
+		RESTClient().
+		Get().
+		Namespace(f.Namespace).
+		Resource("pods").
+		SubResource("proxy").
+		Name(net.JoinSchemeNamePort("http", f.NodeName, strconv.Itoa(10255))).
+		Suffix("/stats/summary").DoRaw()
+	if err != nil {
+		return nil, err
+	}
+	// Unmarshal the response as a Summary object and return it.
+	res := &stats.Summary{}
+	if err := json.Unmarshal(b, res); err != nil {
+		return nil, err
+	}
+	return res, nil
+}