Merge pull request #973 from cpuguy83/ci_store_test_results

Output test results in junit and export to circle
2021-06-01 11:32:21 -07:00
parent fbf6a1957f db5bf2b0d3
commit 92f8661031
5 changed files with 59 additions and 37 deletions
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -30,7 +30,7 @@ jobs:
  test:
    resource_class: xlarge
    docker:
-      - image: circleci/golang:1.15
+      - image: circleci/golang:1.16
        environment:
          GO111MODULE: "on"
    working_directory: /go/src/github.com/virtual-kubelet/virtual-kubelet
@@ -42,13 +42,22 @@ jobs:
      - run:
          name: Build
          command: V=1 make build
+      - run: go install gotest.tools/gotestsum@latest
      - run:
          name: Tests
-          command: V=1 CI=1 make test envtest
+          environment:
+            GOTEST: gotestsum -- -timeout=9m
+            GOTESTSUM_JUNITFILE: output/unit/results.xml
+            GODEBUG: cgocheck=2
+          command: |
+            mkdir -p output/unit
+            V=1 make test envtest
      - save_cache:
          key: test-{{ checksum "go.mod" }}-{{ checksum "go.sum" }}
          paths:
            - "/go/pkg/mod"
+      - store_test_results:
+          path: output

  e2e:
    machine:
@@ -121,13 +130,21 @@ jobs:
            - e2e-{{ checksum "go.mod" }}-{{ checksum "go.sum" }}-2
      - run:
          name: Run the end-to-end test suite
+          environment:
+            GOTEST: gotestsum --
          command: |
            mkdir $HOME/.go
            export PATH=$HOME/.go/bin:${PATH}
-            curl -fsSL -o "/tmp/go.tar.gz" "https://dl.google.com/go/go1.15.6.linux-amd64.tar.gz"
+            curl -fsSL -o "/tmp/go.tar.gz" "https://dl.google.com/go/go1.16.4.linux-amd64.tar.gz"
            tar -C $HOME/.go --strip-components=1 -xzf "/tmp/go.tar.gz"
            go version
+            mkdir -p output/e2e
+            export GOTESTSUM_JUNITFILE="$(pwd)/output/e2e/results.xml"
+            export PATH="${GOPATH}/bin:${PATH}"
+            go install gotest.tools/gotestsum@latest
            make e2e
+      - store_test_results:
+          path: output
      - save_cache:
          key: e2e-{{ checksum "go.mod" }}-{{ checksum "go.sum" }}-2
          paths:
--- a/32
+++ b/32
@@ -5,6 +5,8 @@ exec := $(DOCKER_IMAGE)
 github_repo := virtual-kubelet/virtual-kubelet
 binary := virtual-kubelet

+GOTEST ?= go test $(if $V,-v)
+
 export GO111MODULE ?= on

 include Makefile.e2e
@@ -71,36 +73,28 @@ vet:
 	@echo "go vet'ing..."
 ifndef CI
 	@echo "go vet'ing Outside CI..."
-	go vet $(allpackages)
+	go vet $(TESTDIRS)
 else
 	@echo "go vet'ing in CI..."
 	mkdir -p test
-	( go vet $(allpackages); echo $$? ) | \
+	( go vet $(TESTDIRS); echo $$? ) | \
       tee test/vet.txt | sed '$$ d'; exit $$(tail -1 test/vet.txt)
 endif

 test:
-ifndef CI
-	@echo "Testing..."
-	go test  $(if $V,-v) $(allpackages)
-else
-	@echo "Testing in CI..."
-	mkdir -p test
-	( GODEBUG=cgocheck=2 go test -timeout=9m -v $(allpackages); echo $$? ) | \
-       tee test/output.txt | sed '$$ d'; exit $$(tail -1 test/output.txt)
-endif
+	$(GOTEST) $(TESTDIRS)

 list:
 	@echo "List..."
-	@echo $(allpackages)
+	@echo $(TESTDIRS)

 cover: gocovmerge
 	@echo "Coverage Report..."
 	@echo "NOTE: make cover does not exit 1 on failure, don't use it to check for tests success!"
 	rm -f .GOPATH/cover/*.out cover/all.merged
 	$(if $V,@echo "-- go test -coverpkg=./... -coverprofile=cover/... ./...")
-	@for MOD in $(allpackages); do \
-        go test -coverpkg=`echo $(allpackages)|tr " " ","` \
+	@for MOD in $(TESTDIRS); do \
+        go test -coverpkg=`echo $(TESTDIRS)|tr " " ","` \
            -coverprofile=cover/unit-`echo $$MOD|tr "/" "_"`.out \
            $$MOD 2>&1 | grep -v "no packages being tested depend on"; \
    done
@@ -142,11 +136,7 @@ VERSION          := $(shell git describe --tags --always --dirty="-dev")
 DATE             := $(shell date -u '+%Y-%m-%d-%H:%M UTC')
 VERSION_FLAGS    := -ldflags='-X "main.buildVersion=$(VERSION)" -X "main.buildTime=$(DATE)"'

-# assuming go 1.9 here!!
-_allpackages = $(shell go list ./...)
-
-# memoize allpackages, so that it's executed only once and only if used
-allpackages = $(if $(__allpackages),,$(eval __allpackages := $$(_allpackages)))$(__allpackages)
+TESTDIRS ?= ./...

 .PHONY: goimports
 goimports: $(gobin_tool)
@@ -187,7 +177,7 @@ kubebuilder_2.3.1_${TEST_OS}_${TEST_ARCH}: kubebuilder_2.3.1_${TEST_OS}_${TEST_A
 envtest: kubebuilder_2.3.1_${TEST_OS}_${TEST_ARCH}
 	# You can add klog flags for debugging, like: -klog.v=10 -klog.logtostderr
 	# klogv2 flags just wraps our existing logrus.
-	KUBEBUILDER_ASSETS=$(PWD)/kubebuilder_2.3.1_${TEST_OS}_${TEST_ARCH}/bin go test -v -run=TestEnvtest ./node -envtest=true
+	KUBEBUILDER_ASSETS=$(PWD)/kubebuilder_2.3.1_${TEST_OS}_${TEST_ARCH}/bin $(GOTEST) -run=TestEnvtest ./node -envtest=true

 .PHONY: fmt
 fmt:
@@ -195,4 +185,4 @@ fmt:

 .PHONY: lint
 lint: $(gobin_tool)
-	gobin -run github.com/golangci/golangci-lint/cmd/golangci-lint@v1.33.0 run ./...
+	gobin -run github.com/golangci/golangci-lint/cmd/golangci-lint@v1.33.0 run ./...
--- a/Makefile.e2e
+++ b/Makefile.e2e
@@ -39,7 +39,7 @@ e2e: NODE_NAME := vkubelet-mock-0
 e2e: export VK_BUILD_TAGS += mock_provider
 e2e: e2e.clean bin/e2e/virtual-kubelet skaffold/run
 	@echo Running tests...
-	cd $(PWD)/internal/test/e2e && go test -v -timeout 5m -tags e2e ./... \
+	cd $(PWD)/internal/test/e2e && $(GOTEST) -timeout 5m -tags e2e ./... \
 		-kubeconfig=$(KUBECONFIG) \
 		-namespace=$(NAMESPACE) \
 		-node-name=$(NODE_NAME)
--- a/internal/expansion/expand_test.go
+++ b/internal/expansion/expand_test.go
@@ -5,10 +5,17 @@ import (
 )

 func TestMapReference(t *testing.T) {
-	envs := map[string]string{
-		"FOO": "bar",
-		"ZOO": "$(FOO)-1",
-		"BLU": "$(ZOO)-2",
+	// We use a struct here instead of a map because we need mappings to happen in order.
+	// Go maps are randomized.
+	type envVar struct {
+		Name  string
+		Value string
+	}
+
+	envs := []envVar{
+		{"FOO", "bar"},
+		{"ZOO", "$(FOO)-1"},
+		{"BLU", "$(ZOO)-2"},
 	}

 	declaredEnv := map[string]string{
@@ -21,8 +28,8 @@ func TestMapReference(t *testing.T) {

 	mapping := MappingFuncFor(declaredEnv, serviceEnv)

-	for k, v := range envs {
-		declaredEnv[k] = Expand(v, mapping)
+	for _, env := range envs {
+		declaredEnv[env.Name] = Expand(env.Value, mapping)
 	}

 	expectedEnv := map[string]string{
--- a/node/lifecycle_test.go
+++ b/node/lifecycle_test.go
@@ -367,6 +367,14 @@ func testDanglingPodScenario(ctx context.Context, t *testing.T, s *system, m tes

 }

+func sendErr(ctx context.Context, ch chan error, err error) {
+	select {
+	case <-ctx.Done():
+		log.G(ctx).WithError(err).Warn("timeout waiting to send test error")
+	case ch <- err:
+	}
+}
+
 func testDanglingPodScenarioWithDeletionTimestamp(ctx context.Context, t *testing.T, s *system, m testingProvider) {
 	t.Parallel()

@@ -396,7 +404,7 @@ func testDanglingPodScenarioWithDeletionTimestamp(ctx context.Context, t *testin
 			func(ev watch.Event) (bool, error) {
 				return ev.Type == watch.Deleted, nil
 			})
-		watchErrCh <- watchErr
+		sendErr(ctx, watchErrCh, watchErr)
 	}()

 	// Start the pod controller
@@ -436,7 +444,7 @@ func testCreateStartDeleteScenario(ctx context.Context, t *testing.T, s *system,
 				return pod.Name == p.ObjectMeta.Name, nil
 			})

-		watchErrCh <- watchErr
+		sendErr(ctx, watchErrCh, watchErr)
 	}()

 	// Create the Pod
@@ -465,7 +473,7 @@ func testCreateStartDeleteScenario(ctx context.Context, t *testing.T, s *system,
 				return pod.Status.Phase == corev1.PodRunning, nil
 			})

-		watchErrCh <- watchErr
+		sendErr(ctx, watchErrCh, watchErr)
 	}()

 	assert.NilError(t, s.start(ctx))
@@ -487,7 +495,7 @@ func testCreateStartDeleteScenario(ctx context.Context, t *testing.T, s *system,
 		_, watchDeleteErr := watchutils.UntilWithoutRetry(ctx, watcher2, func(ev watch.Event) (bool, error) {
 			return ev.Type == watch.Deleted, nil
 		})
-		waitDeleteCh <- watchDeleteErr
+		sendErr(ctx, waitDeleteCh, watchDeleteErr)
 	}()

 	// Setup a watch prior to pod deletion
@@ -495,7 +503,7 @@ func testCreateStartDeleteScenario(ctx context.Context, t *testing.T, s *system,
 	assert.NilError(t, err)
 	defer watcher.Stop()
 	go func() {
-		watchErrCh <- waitFunction(ctx, watcher)
+		sendErr(ctx, watchErrCh, waitFunction(ctx, watcher))
 	}()

 	// Delete the pod via deletiontimestamp
@@ -559,7 +567,7 @@ func testUpdatePodWhileRunningScenario(ctx context.Context, t *testing.T, s *sys
 			})
 		// This deepcopy is required to please the race detector
 		p = newPod.Object.(*corev1.Pod).DeepCopy()
-		watchErrCh <- watchErr
+		sendErr(ctx, watchErrCh, watchErr)
 	}()

 	// Start the pod controller