Revert caching go builds (#631)

It turns out that using gocache on circleci like this makes it
so that builds are not properly invalidated, and breaks a
whole load of things.

.circleci/config.yml

@@ -58,7 +58,6 @@ jobs:
       MINIKUBE_WANTUPDATENOTIFICATION: false
       MINIKUBE_WANTREPORTERRORPROMPT: false
       SKAFFOLD_VERSION: v0.18.0
-      GOCACHE: /home/circleci/.cache/go
     steps:
       - checkout
       - run:
@@ -100,28 +99,10 @@ jobs:
           name: Watch nodes
           command: kubectl get nodes -o json --watch
           background: true
-
-      - restore_cache:
-          keys:
-            - pkg-cache-{{ .Branch }}
-            - go-cache-{{ .Branch }}
-
       - run:
           name: Run the end-to-end test suite
           command: |
             make e2e
-
-      - save_cache:
-          key: pkg-cache-{{ .Branch }}
-          paths:
-            - "/go/pkg"
-            - "/home/circleci/go/"
-
-      - save_cache:
-          key: go-cache-{{ .Branch }}
-          paths:
-            - "/home/circleci/.cache/go"
-
       - run:
           name: Collect logs on failure from vkubelet-mock-0
           command: |