Use Running Pods Endpoint for testing

* Use the VKUBELET API for fetching the pods during testing, which
   makes it easier to run testing without wiring up an entire cluster
 * Runningpods does not require each provider to implement the
   optional stats interface for testing.
This commit is contained in:
Sargun Dhillon
2019-05-17 16:18:18 -07:00
parent d183dbbe17
commit ec8972fef3
5 changed files with 237 additions and 164 deletions

View File

@@ -8,33 +8,35 @@ import (
"time"
"github.com/virtual-kubelet/virtual-kubelet/vkubelet"
v1 "k8s.io/api/core/v1"
"gotest.tools/assert"
"k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
)
const (
// deleteGracePeriodForProvider is the amount of time we allow for the provider to react to deletion of a pod before proceeding to assert that the pod has been deleted.
deleteGracePeriodForProvider = 100 * time.Millisecond
// deleteGracePeriodForProvider is the maximum amount of time we allow for the provider to react to deletion of a pod
// before proceeding to assert that the pod has been deleted.
deleteGracePeriodForProvider = 1 * time.Second
)
// TestGetStatsSummary creates a pod having two containers and queries the /stats/summary endpoint of the virtual-kubelet.
// It expects this endpoint to return stats for the current node, as well as for the aforementioned pod and each of its two containers.
func TestGetStatsSummary(t *testing.T) {
// Create a pod with prefix "nginx-0-" having three containers.
pod, err := f.CreatePod(f.CreateDummyPodObjectWithPrefix("nginx-0-", "foo", "bar", "baz"))
// Create a pod with prefix "nginx-" having three containers.
pod, err := f.CreatePod(f.CreateDummyPodObjectWithPrefix(t.Name(), "nginx-", "foo", "bar", "baz"))
if err != nil {
t.Fatal(err)
}
// Delete the "nginx-0-X" pod after the test finishes.
defer func() {
if err := f.DeletePod(pod.Namespace, pod.Name); err != nil && !apierrors.IsNotFound(err) {
if err := f.DeletePodImmediately(pod.Namespace, pod.Name); err != nil && !apierrors.IsNotFound(err) {
t.Error(err)
}
}()
// Wait for the "nginx-0-X" pod to be reported as running and ready.
if err := f.WaitUntilPodReady(pod.Namespace, pod.Name); err != nil {
// Wait for the "nginx-" pod to be reported as running and ready.
if _, err := f.WaitUntilPodReady(pod.Namespace, pod.Name); err != nil {
t.Fatal(err)
}
@@ -49,13 +51,13 @@ func TestGetStatsSummary(t *testing.T) {
t.Fatalf("expected stats for node %s, got stats for node %s", f.NodeName, stats.Node.NodeName)
}
// Make sure the "nginx-0-X" pod exists in the slice of PodStats.
// Make sure the "nginx-" pod exists in the slice of PodStats.
idx, err := findPodInPodStats(stats, pod)
if err != nil {
t.Fatal(err)
}
// Make sure that we've got stats for all the containers in the "nginx-0-X" pod.
// Make sure that we've got stats for all the containers in the "nginx-" pod.
desiredContainerStatsCount := len(pod.Spec.Containers)
currentContainerStatsCount := len(stats.Pods[idx].Containers)
if currentContainerStatsCount != desiredContainerStatsCount {
@@ -63,151 +65,174 @@ func TestGetStatsSummary(t *testing.T) {
}
}
// TestPodLifecycle creates two pods and verifies that the provider has been asked to create them.
// Then, it deletes one of the pods and verifies that the provider has been asked to delete it.
// TestPodLifecycleGracefulDelete creates a pod and verifies that the provider has been asked to create it.
// Then, it deletes the pods and verifies that the provider has been asked to delete it.
// These verifications are made using the /stats/summary endpoint of the virtual-kubelet, by checking for the presence or absence of the pods.
// Hence, the provider being tested must implement the PodMetricsProvider interface.
func TestPodLifecycle(t *testing.T) {
// Create a pod with prefix "nginx-0-" having a single container.
pod0, err := f.CreatePod(f.CreateDummyPodObjectWithPrefix("nginx-0-", "foo"))
func TestPodLifecycleGracefulDelete(t *testing.T) {
// Create a pod with prefix "nginx-" having a single container.
podSpec := f.CreateDummyPodObjectWithPrefix(t.Name(), "nginx-", "foo")
podSpec.Spec.NodeName = nodeName
pod, err := f.CreatePod(podSpec)
if err != nil {
t.Fatal(err)
}
// Delete the "nginx-0-X" pod after the test finishes.
// Delete the pod after the test finishes.
defer func() {
if err := f.DeletePod(pod0.Namespace, pod0.Name); err != nil && !apierrors.IsNotFound(err) {
if err := f.DeletePodImmediately(pod.Namespace, pod.Name); err != nil && !apierrors.IsNotFound(err) {
t.Error(err)
}
}()
t.Logf("Created pod: %s", pod.Name)
// Create a pod with prefix "nginx-1-" having a single container.
pod1, err := f.CreatePod(f.CreateDummyPodObjectWithPrefix("nginx-1-", "bar"))
if err != nil {
// Wait for the "nginx-" pod to be reported as running and ready.
if _, err := f.WaitUntilPodReady(pod.Namespace, pod.Name); err != nil {
t.Fatal(err)
}
// Delete the "nginx-1-Y" pod after the test finishes.
defer func() {
if err := f.DeletePod(pod1.Namespace, pod1.Name); err != nil && !apierrors.IsNotFound(err) {
t.Error(err)
}
}()
t.Logf("Pod %s ready", pod.Name)
// Wait for the "nginx-0-X" pod to be reported as running and ready.
if err := f.WaitUntilPodReady(pod0.Namespace, pod0.Name); err != nil {
t.Fatal(err)
}
// Wait for the "nginx-1-Y" pod to be reported as running and ready.
if err := f.WaitUntilPodReady(pod1.Namespace, pod1.Name); err != nil {
t.Fatal(err)
}
// Grab the pods from the provider.
pods, err := f.GetRunningPods()
assert.NilError(t, err)
// Grab the stats from the provider.
stats, err := f.GetStatsSummary()
if err != nil {
t.Fatal(err)
}
// Check if the pod exists in the slice of PodStats.
assert.NilError(t, findPodInPods(pods, pod))
// Make sure the "nginx-0-X" pod exists in the slice of PodStats.
if _, err := findPodInPodStats(stats, pod0); err != nil {
t.Fatal(err)
}
// Make sure the "nginx-1-Y" pod exists in the slice of PodStats.
if _, err := findPodInPodStats(stats, pod1); err != nil {
t.Fatal(err)
}
// Wait for the "nginx-1-Y" pod to be deleted in a separate goroutine.
// This ensures that we don't possibly miss the MODIFIED/DELETED events due to establishing the watch too late in the process.
pod1Ch := make(chan error)
podCh := make(chan error)
var podLast *v1.Pod
go func() {
// Wait for the "nginx-1-Y" pod to be reported as having been marked for deletion.
if err := f.WaitUntilPodDeleted(pod1.Namespace, pod1.Name); err != nil {
// Close the podCh channel, signaling we've observed deletion of the pod.
defer close(podCh)
var err error
podLast, err = f.WaitUntilPodDeleted(pod.Namespace, pod.Name)
if err != nil {
// Propagate the error to the outside so we can fail the test.
pod1Ch <- err
} else {
// Close the pod0Ch channel, signaling we've observed deletion of the pod.
close(pod1Ch)
podCh <- err
}
}()
// Delete the "nginx-1" pod.
if err := f.DeletePod(pod1.Namespace, pod1.Name); err != nil {
// Gracefully delete the "nginx-" pod.
if err := f.DeletePod(pod.Namespace, pod.Name); err != nil {
t.Fatal(err)
}
t.Logf("Deleted pod: %s", pod.Name)
// Wait for the delete event to be ACKed.
if err := <-pod1Ch; err != nil {
if err := <-podCh; err != nil {
t.Fatal(err)
}
time.Sleep(deleteGracePeriodForProvider)
// Give the provider some time to react to the MODIFIED/DELETED events before proceeding.
// Grab the pods from the provider.
pods, err = f.GetRunningPods()
assert.NilError(t, err)
// Make sure the pod DOES NOT exist in the provider's set of running pods
assert.Assert(t, findPodInPods(pods, pod) != nil)
// Make sure we saw the delete event, and the delete event was graceful
assert.Assert(t, podLast != nil)
assert.Assert(t, podLast.ObjectMeta.GetDeletionGracePeriodSeconds() != nil)
assert.Assert(t, *podLast.ObjectMeta.GetDeletionGracePeriodSeconds() > 0)
}
// TestPodLifecycleNonGracefulDelete creates one podsand verifies that the provider has created them
// and put them in the running lifecycle. It then does a force delete on the pod, and verifies the provider
// has deleted it.
func TestPodLifecycleForceDelete(t *testing.T) {
podSpec := f.CreateDummyPodObjectWithPrefix(t.Name(), "nginx-", "foo")
// Create a pod with prefix having a single container.
pod, err := f.CreatePod(podSpec)
if err != nil {
t.Fatal(err)
}
// Delete the pod after the test finishes.
defer func() {
if err := f.DeletePodImmediately(pod.Namespace, pod.Name); err != nil && !apierrors.IsNotFound(err) {
t.Error(err)
}
}()
t.Logf("Created pod: %s", pod.Name)
// Wait for the "nginx-" pod to be reported as running and ready.
if _, err := f.WaitUntilPodReady(pod.Namespace, pod.Name); err != nil {
t.Fatal(err)
}
t.Logf("Pod %s ready", pod.Name)
// Grab the pods from the provider.
pods, err := f.GetRunningPods()
assert.NilError(t, err)
// Check if the pod exists in the slice of Pods.
assert.NilError(t, findPodInPods(pods, pod))
// Wait for the pod to be deleted in a separate goroutine.
// This ensures that we don't possibly miss the MODIFIED/DELETED events due to establishing the watch too late in the process.
// It also makes sure that in light of soft deletes, we properly handle non-graceful pod deletion
podCh := make(chan error)
var podLast *v1.Pod
go func() {
// Close the podCh channel, signaling we've observed deletion of the pod.
defer close(podCh)
var err error
// Wait for the pod to be reported as having been deleted.
podLast, err = f.WaitUntilPodDeleted(pod.Namespace, pod.Name)
if err != nil {
// Propagate the error to the outside so we can fail the test.
podCh <- err
}
}()
time.Sleep(deleteGracePeriodForProvider)
// Forcibly delete the pod.
if err := f.DeletePodImmediately(pod.Namespace, pod.Name); err != nil {
t.Logf("Last saw pod in state: %+v", podLast)
t.Fatal(err)
}
t.Log("Force deleted pod: ", pod.Name)
// Wait for the delete event to be ACKed.
if err := <-podCh; err != nil {
t.Logf("Last saw pod in state: %+v", podLast)
t.Fatal(err)
}
// Give the provider some time to react to the MODIFIED/DELETED events before proceeding.
time.Sleep(deleteGracePeriodForProvider)
// Grab the stats from the provider.
stats, err = f.GetStatsSummary()
if err != nil {
t.Fatal(err)
}
// Grab the pods from the provider.
pods, err = f.GetRunningPods()
assert.NilError(t, err)
// Make sure the "nginx-1-Y" pod DOES NOT exist in the slice of PodStats anymore.
if _, err := findPodInPodStats(stats, pod1); err == nil {
t.Fatalf("expected to NOT find pod \"%s/%s\" in the slice of pod stats", pod1.Namespace, pod1.Name)
}
// Make sure the "nginx-" pod DOES NOT exist in the slice of Pods anymore.
assert.Assert(t, findPodInPods(pods, pod) != nil)
// Wait for the "nginx-0-X" pod to be deleted in a separate goroutine.
// This ensures that we don't possibly miss the MODIFIED/DELETED events due to establishing the watch too late in the process.
pod0Ch := make(chan error)
go func() {
// Wait for the "nginx-0-X" pod to be reported as having been deleted.
if err := f.WaitUntilPodDeleted(pod0.Namespace, pod0.Name); err != nil {
// Propagate the error to the outside so we can fail the test.
pod0Ch <- err
} else {
// Close the pod0Ch channel, signaling we've observed deletion of the pod.
close(pod0Ch)
}
}()
t.Logf("Pod ended as phase: %+v", podLast.Status.Phase)
// Forcibly delete the "nginx-0" pod.
if err := f.DeletePodImmediately(pod0.Namespace, pod0.Name); err != nil {
t.Fatal(err)
}
// Wait for the delete event to be ACKed.
if err := <-pod0Ch; err != nil {
t.Fatal(err)
}
// Give the provider some time to react to the MODIFIED/DELETED events before proceeding.
time.Sleep(deleteGracePeriodForProvider)
// Grab the stats from the provider.
stats, err = f.GetStatsSummary()
if err != nil {
t.Fatal(err)
}
// Make sure the "nginx-0-X" pod DOES NOT exist in the slice of PodStats anymore.
if _, err := findPodInPodStats(stats, pod0); err == nil {
t.Fatalf("expected to NOT find pod \"%s/%s\" in the slice of pod stats", pod0.Namespace, pod0.Name)
}
}
// TestCreatePodWithOptionalInexistentSecrets tries to create a pod referencing optional, inexistent secrets.
// It then verifies that the pod is created successfully.
func TestCreatePodWithOptionalInexistentSecrets(t *testing.T) {
// Create a pod with a single container referencing optional, inexistent secrets.
pod, err := f.CreatePod(f.CreatePodObjectWithOptionalSecretKey())
pod, err := f.CreatePod(f.CreatePodObjectWithOptionalSecretKey(t.Name()))
if err != nil {
t.Fatal(err)
}
// Delete the pod after the test finishes.
defer func() {
if err := f.DeletePod(pod.Namespace, pod.Name); err != nil && !apierrors.IsNotFound(err) {
if err := f.DeletePodImmediately(pod.Namespace, pod.Name); err != nil && !apierrors.IsNotFound(err) {
t.Error(err)
}
}()
// Wait for the pod to be reported as running and ready.
if err := f.WaitUntilPodReady(pod.Namespace, pod.Name); err != nil {
if _, err := f.WaitUntilPodReady(pod.Namespace, pod.Name); err != nil {
t.Fatal(err)
}
@@ -216,21 +241,19 @@ func TestCreatePodWithOptionalInexistentSecrets(t *testing.T) {
t.Fatal(err)
}
// Check that the pod is known to the provider.
stats, err := f.GetStatsSummary()
if err != nil {
t.Fatal(err)
}
if _, err := findPodInPodStats(stats, pod); err != nil {
t.Fatal(err)
}
// Grab the pods from the provider.
pods, err := f.GetRunningPods()
assert.NilError(t, err)
// Check if the pod exists in the slice of Pods.
assert.NilError(t, findPodInPods(pods, pod))
}
// TestCreatePodWithMandatoryInexistentSecrets tries to create a pod referencing inexistent secrets.
// It then verifies that the pod is not created.
func TestCreatePodWithMandatoryInexistentSecrets(t *testing.T) {
// Create a pod with a single container referencing inexistent secrets.
pod, err := f.CreatePod(f.CreatePodObjectWithMandatorySecretKey())
pod, err := f.CreatePod(f.CreatePodObjectWithMandatorySecretKey(t.Name()))
if err != nil {
t.Fatal(err)
}
@@ -247,34 +270,32 @@ func TestCreatePodWithMandatoryInexistentSecrets(t *testing.T) {
t.Fatal(err)
}
// Check that the pod is NOT known to the provider.
stats, err := f.GetStatsSummary()
if err != nil {
t.Fatal(err)
}
if _, err := findPodInPodStats(stats, pod); err == nil {
t.Fatalf("Expecting to NOT find pod \"%s/%s\" having mandatory, inexistent secrets.", pod.Namespace, pod.Name)
}
// Grab the pods from the provider.
pods, err := f.GetRunningPods()
assert.NilError(t, err)
// Check if the pod exists in the slice of PodStats.
assert.Assert(t, findPodInPods(pods, pod) != nil)
}
// TestCreatePodWithOptionalInexistentConfigMap tries to create a pod referencing optional, inexistent config map.
// It then verifies that the pod is created successfully.
func TestCreatePodWithOptionalInexistentConfigMap(t *testing.T) {
// Create a pod with a single container referencing optional, inexistent config map.
pod, err := f.CreatePod(f.CreatePodObjectWithOptionalConfigMapKey())
pod, err := f.CreatePod(f.CreatePodObjectWithOptionalConfigMapKey(t.Name()))
if err != nil {
t.Fatal(err)
}
// Delete the pod after the test finishes.
defer func() {
if err := f.DeletePod(pod.Namespace, pod.Name); err != nil && !apierrors.IsNotFound(err) {
if err := f.DeletePodImmediately(pod.Namespace, pod.Name); err != nil && !apierrors.IsNotFound(err) {
t.Error(err)
}
}()
// Wait for the pod to be reported as running and ready.
if err := f.WaitUntilPodReady(pod.Namespace, pod.Name); err != nil {
if _, err := f.WaitUntilPodReady(pod.Namespace, pod.Name); err != nil {
t.Fatal(err)
}
@@ -283,21 +304,19 @@ func TestCreatePodWithOptionalInexistentConfigMap(t *testing.T) {
t.Fatal(err)
}
// Check that the pod is known to the provider.
stats, err := f.GetStatsSummary()
if err != nil {
t.Fatal(err)
}
if _, err := findPodInPodStats(stats, pod); err != nil {
t.Fatal(err)
}
// Grab the pods from the provider.
pods, err := f.GetRunningPods()
assert.NilError(t, err)
// Check if the pod exists in the slice of PodStats.
assert.NilError(t, findPodInPods(pods, pod))
}
// TestCreatePodWithMandatoryInexistentConfigMap tries to create a pod referencing inexistent secrets.
// It then verifies that the pod is not created.
func TestCreatePodWithMandatoryInexistentConfigMap(t *testing.T) {
// Create a pod with a single container referencing inexistent config map.
pod, err := f.CreatePod(f.CreatePodObjectWithMandatoryConfigMapKey())
pod, err := f.CreatePod(f.CreatePodObjectWithMandatoryConfigMapKey(t.Name()))
if err != nil {
t.Fatal(err)
}
@@ -314,14 +333,12 @@ func TestCreatePodWithMandatoryInexistentConfigMap(t *testing.T) {
t.Fatal(err)
}
// Check that the pod is NOT known to the provider.
stats, err := f.GetStatsSummary()
if err != nil {
t.Fatal(err)
}
if _, err := findPodInPodStats(stats, pod); err == nil {
t.Fatalf("Expecting to NOT find pod \"%s/%s\" having mandatory, inexistent config map.", pod.Namespace, pod.Name)
}
// Grab the pods from the provider.
pods, err := f.GetRunningPods()
assert.NilError(t, err)
// Check if the pod exists in the slice of PodStats.
assert.Assert(t, findPodInPods(pods, pod) != nil)
}
// findPodInPodStats returns the index of the specified pod in the .pods field of the specified Summary object.
@@ -334,3 +351,14 @@ func findPodInPodStats(summary *v1alpha1.Summary, pod *v1.Pod) (int, error) {
}
return -1, fmt.Errorf("failed to find pod \"%s/%s\" in the slice of pod stats", pod.Namespace, pod.Name)
}
// findPodInPodStats returns the index of the specified pod in the .pods field of the specified PodList object.
// It returns error if the pod doesn't exist in the podlist
func findPodInPods(pods *v1.PodList, pod *v1.Pod) error {
for _, p := range pods.Items {
if p.Namespace == pod.Namespace && p.Name == pod.Name && string(p.UID) == string(pod.UID) {
return nil
}
}
return fmt.Errorf("failed to find pod \"%s/%s\" in the slice of pod list", pod.Namespace, pod.Name)
}