Add tracing of the kubernetes cluster during testing (#608)

* Add tracing of the kubernetes cluster during testing

This adds tracing to the testing to get the kubelet's logs upon
failure. In addition, it keeps track of the pods, and the node
statuses throughout the test.

* Add arguments to make virtual kubelet's log more useful

.circleci/config.yml

@@ -98,10 +98,23 @@ jobs:
             until kubectl get nodes -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do
               sleep 1;
             done
+      - run:
+          name: Watch pods
+          command: kubectl get pods -o json --watch
+          background: true
+      - run:
+          name: Watch nodes
+          command: kubectl get nodes -o json --watch
+          background: true
       - run:
           name: Run the end-to-end test suite
           command: |
             make e2e
+      - run:
+          name: Collect logs on failure from vkubelet-mock-0
+          command: |
+            kubectl logs vkubelet-mock-0
+          when: on_fail
 
 workflows:
   version: 2

hack/skaffold/virtual-kubelet/pod.yml

@@ -16,6 +16,13 @@ spec:
     - mock
     - --provider-config
     - /vkubelet-mock-0-cfg.json
+    - --startup-timeout
+    - 10s
+    - --klog.v
+    - "2"
+    - --klog.logtostderr
+    - --log-level
+    - debug
     ports:
     - name: metrics
       containerPort: 10255