Fargate: Refactor e2e tests and make the region configurable

providers/aws/provider_test.go

@@ -22,8 +22,17 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 )
 
-const testRegion = "us-east-1"
+const (
+	// E2E test configuration.
+	testName          = "vk-fargate-e2e-test"
+	defaultTestRegion = "us-east-1"
 
+	// Environment variables that modify the test behavior.
+	envSkipTests  = "SKIP_AWS_E2E"
+	envTestRegion = "VK_TEST_FARGATE_REGION"
+)
+
+// executorRoleAssumePolicy is the policy used by task execution role.
 const executorRoleAssumePolicy = `{
   "Version": "2012-10-17",
   "Statement": [
@@ -38,70 +47,135 @@ const executorRoleAssumePolicy = `{
   ]
 }`
 
+// testConfig contains the Fargate provider test configuration template.
 const testConfig = `
 Region = "%s"
 ClusterName = "%s"
-CloudWatchLogGroupName = "%s"
-ExecutionRoleArn = "%s"
 Subnets = [ "%s" ]
 SecurityGroups = [ ]
-# Required to pull the busybox image
 AssignPublicIPv4Address = true
+ExecutionRoleArn = "%s"
+CloudWatchLogGroupName = "%s"
 `
 
-const testName = "vk-aws-e2e-test"
+var (
+	ecsClient        *ecs.ECS
+	testRegion       string
+	subnetID         *string
+	executorRoleName *string
+	logGroupName     *string
+)
 
-func TestAWS(t *testing.T) {
+// TestMain wraps the tests with the extra setup and teardown of AWS resources.
-	if os.Getenv("SKIP_AWS_E2E") == "1" {
+func TestMain(m *testing.M) {
-		t.Skip("skipping AWS e2e tests")
+	var err error
+
+	// Skip the tests in this package if the environment variable is set.
+	if os.Getenv(envSkipTests) == "1" {
+		fmt.Println("Skipping AWS E2E tests.")
+		os.Exit(0)
 	}
 
+	// Query the test region.
+	region, ok := os.LookupEnv(envTestRegion)
+	if ok {
+		testRegion = region
+	} else {
+		testRegion = defaultTestRegion
+	}
+	fmt.Printf("Starting provider tests in region %s\n", testRegion)
+
+	// Create the session and clients.
 	session := session.New(&aws.Config{
 		Region: aws.String(testRegion),
 	})
+
+	ecsClient = ecs.New(session)
 	ec2Client := ec2.New(session)
-	ecsClient := ecs.New(session)
 	cloudwatchClient := cloudwatchlogs.New(session)
 	iamClient := iam.New(session)
 
-	// Create a new VPC with one subnet and internet access
+	// Create a test VPC with one subnet and internet access.
-	// Internet access is required to pull public images from the docker registry
+	// Internet access is required to pull public images from the docker registry.
-	subnetID, err := createVpcWithInternetAccess(ec2Client)
+	subnetID, err = createVpcWithInternetAccess(ec2Client)
 	if err != nil {
-		t.Error(err)
+		fmt.Printf("Failed to create VPC: %+v\n", err)
+		os.Exit(-1)
 	}
 
-	// Create the Cloudwatch Logs Log Group used by container logs
+	// Create the AWS CloudWatch Logs log group used by containers.
-	logGroupdID := aws.String("/ecs/vk-aws-e2e-test")
+	logGroupName = aws.String("/ecs/" + testName)
 	_, err = cloudwatchClient.CreateLogGroup(&cloudwatchlogs.CreateLogGroupInput{
-		LogGroupName: logGroupdID,
+		LogGroupName: logGroupName,
 	})
 	if err != nil {
-		t.Error(err)
+		fmt.Printf("Failed to create CloudWatch Logs log group: %+v\n", err)
+		os.Exit(-1)
 	}
 
-	// Create the role used by Fargate to write logs and pull ECR images
+	// Create the role used by Fargate to write logs and pull ECR images.
-	executorRoleID := aws.String("vk-aws-e2e-test")
+	executorRoleName = aws.String(testName)
 	_, err = iamClient.CreateRole(&iam.CreateRoleInput{
-		RoleName:                 executorRoleID,
+		RoleName:                 executorRoleName,
 		AssumeRolePolicyDocument: aws.String(executorRoleAssumePolicy),
 	})
 	if err != nil {
-		t.Error(err)
+		fmt.Printf("Failed to create task execution role: %+v", err)
+		os.Exit(-1)
 	}
 
-	// Default policy allowing log writes and ECR pulls
+	// Attach the default policy allowing log writes and ECR pulls.
 	iamClient.AttachRolePolicy(&iam.AttachRolePolicyInput{
 		PolicyArn: aws.String("arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy"),
-		RoleName:  executorRoleID,
+		RoleName:  executorRoleName,
 	})
 	if err != nil {
-		t.Error(err)
+		fmt.Printf("Failed to attach role policy: %+v", err)
+		os.Exit(-1)
 	}
 
-	// Create a cluster for the E2E test
+	// Run the tests.
+	exitCode := m.Run()
+
+	// Delete the task execution role.
+	iamClient.DetachRolePolicy(&iam.DetachRolePolicyInput{
+		PolicyArn: aws.String("arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy"),
+		RoleName:  executorRoleName,
+	})
+	if err != nil {
+		fmt.Printf("Failed to delete task execution role: %+v", err)
+	}
+
+	// Delete the role.
+	_, err = iamClient.DeleteRole(&iam.DeleteRoleInput{
+		RoleName: executorRoleName,
+	})
+	if err != nil {
+		fmt.Printf("Failed to delete task execution role: %+v", err)
+	}
+
+	// Delete the log group.
+	_, err = cloudwatchClient.DeleteLogGroup(&cloudwatchlogs.DeleteLogGroupInput{
+		LogGroupName: logGroupName,
+	})
+	if err != nil {
+		fmt.Printf("Failed to delete CloudWatch Logs log group: %+v\n", err)
+	}
+
+	// Delete the test VPC.
+	err = deleteVpc(ec2Client)
+	if err != nil {
+		fmt.Printf("Failed to delete VPC: %+v\n", err)
+	}
+
+	os.Exit(exitCode)
+}
+
+// TestAWSFargateProviderPodLifecycle validates basic pod lifecycle by starting and stopping a pod.
+func TestAWSFargateProviderPodLifecycle(t *testing.T) {
+	// Create a cluster for the E2E test.
 	createResponse, err := ecsClient.CreateCluster(&ecs.CreateClusterInput{
-		ClusterName: aws.String("vk-aws-e2e-test"),
+		ClusterName: aws.String(testName),
 	})
 	if err != nil {
 		t.Error(err)
@@ -111,13 +185,9 @@ func TestAWS(t *testing.T) {
 	time.Sleep(10 * time.Second)
 
 	t.Run("Create, list and delete pod", func(t *testing.T) {
-		if clusterID == nil || logGroupdID == nil || executorRoleID == nil || subnetID == nil {
+		// Write provider config file with test configuration.
-			t.Fatal("Can't start tests without all requirements.")
+		config := fmt.Sprintf(testConfig, testRegion, testName, *subnetID, *executorRoleName, *logGroupName)
-		}
+		fmt.Printf("Fargate provider test configuration:%s", config)
-
-		// Write config file with our test configuration
-		config := fmt.Sprintf(testConfig, testRegion, "vk-aws-e2e-test", *logGroupdID, *executorRoleID, *subnetID)
-		fmt.Printf("Test with config:\n%s", config)
 
 		tmpfile, err := ioutil.TempFile("", "example")
 		if err != nil {
@@ -134,13 +204,14 @@ func TestAWS(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		// Start the FargateProvider
+		// Start the Fargate provider.
-		provider, err := vkAWS.NewFargateProvider(tmpfile.Name(), nil, "vk-aws-test", "Linux", "1.2.3.4", 10250)
+		provider, err := vkAWS.NewFargateProvider(
+			tmpfile.Name(), nil, testName, "Linux", "1.2.3.4", 10250)
 		if err != nil {
 			t.Fatal(err)
 		}
 
-		// Its a new cluster, there shouldn't be anything running
+		// Confirm that there are no pods on the cluster.
 		pods, err := provider.GetPods()
 		if err != nil {
 			t.Error(err)
@@ -149,7 +220,7 @@ func TestAWS(t *testing.T) {
 			t.Errorf("Expect zero pods, but received %d pods\n%v", len(pods), pods)
 		}
 
-		// Create a test pod
+		// Create a test pod.
 		podName := fmt.Sprintf("test_%d", time.Now().UnixNano()/1000)
 
 		pod := &v1.Pod{
@@ -187,8 +258,7 @@ func TestAWS(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		// Now there should be a pod running
+		// Now there should be exactly one pod.
-
 		pods, err = provider.GetPods()
 		if err != nil {
 			t.Error(err)
@@ -197,13 +267,13 @@ func TestAWS(t *testing.T) {
 			t.Errorf("Expect one pods, but received %d pods\n%v", len(pods), pods)
 		}
 
-		// Wait until its scheduled and running
+		// Wait until the pod is running.
 		err = waitUntilPodStatus(provider, podName, v1.PodRunning)
 		if err != nil {
 			t.Error(err)
 		}
 
-		// Some addition time for the logs to settle
+		// Wait a few seconds for the logs to settle.
 		time.Sleep(10 * time.Second)
 
 		logs, err := provider.GetContainerLogs("default", podName, "echo-container", 100)
@@ -215,7 +285,7 @@ func TestAWS(t *testing.T) {
 			t.Errorf("Expected logs to be \"Started\\n\", but received \"%v\"", logs)
 		}
 
-		// Remove the pod
+		// Delete the pod.
 		err = provider.DeletePod(pod)
 		if err != nil {
 			t.Fatal(err)
@@ -226,7 +296,7 @@ func TestAWS(t *testing.T) {
 			t.Error(err)
 		}
 
-		// The cluster should be empty again
+		// The cluster should be empty again.
 		pods, err = provider.GetPods()
 		if err != nil {
 			t.Error(err)
@@ -236,46 +306,16 @@ func TestAWS(t *testing.T) {
 		}
 	})
 
-	// Remove the cluster
+	// Delete the test cluster.
 	_, err = ecsClient.DeleteCluster(&ecs.DeleteClusterInput{
 		Cluster: clusterID,
 	})
 	if err != nil {
 		t.Error(err)
 	}
-
-	// Remove the execution role
-	iamClient.DetachRolePolicy(&iam.DetachRolePolicyInput{
-		PolicyArn: aws.String("arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy"),
-		RoleName:  executorRoleID,
-	})
-	if err != nil {
-		t.Error(err)
-	}
-
-	_, err = iamClient.DeleteRole(&iam.DeleteRoleInput{
-		RoleName: executorRoleID,
-	})
-	if err != nil {
-		t.Error(err)
-	}
-
-	// Remove the log group
-	_, err = cloudwatchClient.DeleteLogGroup(&cloudwatchlogs.DeleteLogGroupInput{
-		LogGroupName: logGroupdID,
-	})
-	if err != nil {
-		t.Error(err)
-	}
-
-	// Remove the VPC
-	err = deleteVpc(ec2Client)
-	if err != nil {
-		t.Error(err)
-	}
 }
 
-// waitUntilPodStatus polls pod status until the desired state was reached
+// waitUntilPodStatus polls pod status until the desired state is reached.
 func waitUntilPodStatus(provider *vkAWS.FargateProvider, podName string, desiredStatus v1.PodPhase) error {
 	ctx := context.Background()
 	context.WithTimeout(ctx, time.Duration(time.Second*60))