Add the Kube-Proxy Side Car Container and Volume to the container group (#292)

* Add Kube-Proxy Side Car Container

* Add the Kube-Proxy Side Car Container

* Add test

* Bug fix

providers/azure/aci.go

@@ -31,6 +31,7 @@ import (
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
+	clientcmdv1 "k8s.io/client-go/tools/clientcmd/api/v1"
 	"k8s.io/client-go/tools/remotecommand"
 	stats "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
 )
@@ -74,7 +75,9 @@ type ACIProvider struct {
 	networkProfile     string
 	masterURI          string
 	clusterCIDR        string
-	
+	kubeProxyContainer *aci.Container
+	kubeProxyVolume    *aci.Volume
+		
 	metricsSync     sync.Mutex
 	metricsSyncTime time.Time
 	lastMetric      *stats.Summary
@@ -278,16 +281,23 @@ func NewACIProvider(config string, rm *manager.ResourceManager, nodeName, operat
 		if err := p.setupNetworkProfile(azAuth); err != nil {
 			return nil, fmt.Errorf("error setting up network profile: %v", err)
 		}
-	}
 
-	p.masterURI = "10.0.0.1"
+		masterURI := os.Getenv("MASTER_URI")
-	if masterURI := os.Getenv("MASTER_URI"); masterURI != "" {
+		if masterURI == "" {
-		p.masterURI = masterURI
+			masterURI = "10.0.0.1"
-	}
+		}
 
-	p.clusterCIDR = "10.240.0.0/16"
+		p.kubeProxyVolume, err = getKubeProxyVolume(serviceAccountSecretMountPath, masterURI)
-	if clusterCIDR := os.Getenv("CLUSTER_CIDR"); clusterCIDR != "" {
+		if err != nil {
-		p.clusterCIDR = clusterCIDR
+			return nil, fmt.Errorf("error creating kube proxy volume spec: %v", err)
+		}
+
+		clusterCIDR := os.Getenv("CLUSTER_CIDR")
+		if clusterCIDR == "" {
+			clusterCIDR = "10.240.0.0/16"
+		}
+
+		p.kubeProxyContainer = getKubeProxyContainer(clusterCIDR)
 	}
 
 	return &p, err
@@ -394,6 +404,102 @@ func populateNetworkProfile(p *network.Profile, subnet *network.Subnet) {
 	})
 }
 
+
+func getKubeProxyContainer(clusterCIDR string) *aci.Container {
+	container := aci.Container{
+		Name: kubeProxyContainerName,
+		ContainerProperties: aci.ContainerProperties{
+			Image:   kubeProxyImageName,
+			Command: []string{
+				"/hyperkube",
+				"proxy",
+				"--kubeconfig="+kubeConfigDir+"/"+kubeConfigFile,
+				"--cluster-cidr="+clusterCIDR,
+				"--feature-gates=ExperimentalCriticalPodAnnotation=true",
+			},
+		},
+	}
+
+	container.VolumeMounts = []aci.VolumeMount{
+		aci.VolumeMount{
+			Name:      kubeConfigSecretVolume,
+			MountPath: kubeConfigDir,
+			ReadOnly:  true,
+		},
+	}
+
+	container.Resources = aci.ResourceRequirements{
+		Requests: &aci.ResourceRequests{
+			CPU:        0.1,
+			MemoryInGB: 0.10,
+		},
+	}
+
+	return &container
+}
+
+func getKubeProxyVolume(secretPath, masterURI string) (*aci.Volume, error) {
+	ca, err := ioutil.ReadFile(secretPath + "/ca.crt")
+	if err != nil {
+		return nil, fmt.Errorf("failed to read ca.crt file: %v", err)
+	}
+
+	var token []byte
+	token, err = ioutil.ReadFile(secretPath + "/token")
+	if err != nil {
+		return nil, fmt.Errorf("failed to read token file: %v", err)
+	}
+
+	name := "virtual-kubelet"
+
+	config := clientcmdv1.Config{
+		APIVersion: "v1",
+		Kind:       "Config",
+		Clusters:   []clientcmdv1.NamedCluster{
+			clientcmdv1.NamedCluster{
+				Name:    name,
+				Cluster: clientcmdv1.Cluster{
+					Server:                   masterURI,
+					CertificateAuthorityData: ca,
+				},
+			},
+		},
+		AuthInfos:  []clientcmdv1.NamedAuthInfo{
+			clientcmdv1.NamedAuthInfo{
+				Name:     name,
+				AuthInfo: clientcmdv1.AuthInfo{
+					Token: string(token),
+				},
+			},
+		},
+		Contexts:   []clientcmdv1.NamedContext{
+			clientcmdv1.NamedContext{
+				Name: name,
+				Context: clientcmdv1.Context{
+					Cluster:  name,
+					AuthInfo: name,
+				},
+			},
+		},
+		CurrentContext: name,
+	}
+
+	b := new(bytes.Buffer)
+	if err := json.NewEncoder(b).Encode(config); err != nil {
+		return nil, fmt.Errorf("failed to encode the kubeconfig: %v", err)
+	}
+
+	paths := make(map[string]string)
+	paths[kubeConfigFile] = base64.StdEncoding.EncodeToString(b.Bytes())
+
+	volume := aci.Volume{
+		Name:   kubeConfigSecretVolume,
+		Secret: paths,
+	}
+
+	return &volume, nil
+}
+
 // CreatePod accepts a Pod definition and creates
 // an ACI deployment
 func (p *ACIProvider) CreatePod(pod *v1.Pod) error {
@@ -484,52 +590,8 @@ func (p *ACIProvider) amendVnetResources(containerGroup *aci.ContainerGroup) {
 
 	containerGroup.NetworkProfile = &aci.NetworkProfileDefinition{ID: p.networkProfile}
 
-	//containerGroup.ContainerGroupProperties.Containers = append(containerGroup.ContainerGroupProperties.Containers, *(getKubeProxyContainerSpec(p.clusterCIDR)))
+	containerGroup.ContainerGroupProperties.Containers = append(containerGroup.ContainerGroupProperties.Containers, *(p.kubeProxyContainer))
-	//containerGroup.ContainerGroupProperties.Volumes = append(containerGroup.ContainerGroupProperties.Volumes, *(getKubeProxyVolumeSpec(p.masterURI)))
+	containerGroup.ContainerGroupProperties.Volumes = append(containerGroup.ContainerGroupProperties.Volumes, *(p.kubeProxyVolume))
-}
-
-func getKubeProxyContainerSpec(clusterCIDR string) *aci.Container {
-	container := aci.Container{
-		Name: kubeProxyContainerName,
-		ContainerProperties: aci.ContainerProperties{
-			Image:   kubeProxyImageName,
-			Command: []string{
-				"/hyperkube",
-				"proxy",
-				"--kubeconfig="+kubeConfigDir+"/"+kubeConfigFile,
-				"--cluster-cidr="+clusterCIDR,
-			},
-		},
-	}
-
-	container.VolumeMounts = []aci.VolumeMount{
-		aci.VolumeMount{
-			Name:      kubeConfigSecretVolume,
-			MountPath: kubeConfigDir,
-			ReadOnly:  true,
-		},
-	}
-
-	container.Resources = aci.ResourceRequirements{
-		Requests: &aci.ResourceRequests{
-			CPU:        0.1,
-			MemoryInGB: 0.10,
-		},
-	}
-
-	return &container
-}
-
-func getKubeProxyVolumeSpec(masterURI string) *aci.Volume {
-	paths := make(map[string]string)
-
-
-	volume := aci.Volume{
-		Name:   kubeConfigSecretVolume,
-		Secret: paths,
-	}
-
-	return &volume
 }
 
 // UpdatePod is a noop, ACI currently does not support live updates of a pod.
@@ -1165,6 +1227,10 @@ func containerGroupToPod(cg *aci.ContainerGroup) (*v1.Pod, error) {
 	containers := make([]v1.Container, 0, len(cg.Containers))
 	containerStatuses := make([]v1.ContainerStatus, 0, len(cg.Containers))
 	for _, c := range cg.Containers {
+		if strings.EqualFold(c.Name, kubeProxyContainerName) {
+			continue;
+		}
+
 		container := v1.Container{
 			Name:    c.Name,
 			Image:   c.Image,

providers/azure/aci_test.go

@@ -11,6 +11,7 @@ import (
 	"io/ioutil"
 	"net/http"
 	"os"
+	"path/filepath"
 	"testing"
 
 	"github.com/google/uuid"
@@ -652,3 +653,56 @@ func TestCreatePodWithReadinessProbe(t *testing.T) {
 		t.Fatal("Failed to create pod", err)
 	}
 }
+
+func TestGetKubeProxyContainer(t *testing.T) {
+	clusterCIDR := "cidr-" + uuid.New().String()
+	commands := []string{
+		"/hyperkube",
+		"proxy",
+		"--kubeconfig="+kubeConfigDir+"/"+kubeConfigFile,
+		"--cluster-cidr="+clusterCIDR,
+		"--feature-gates=ExperimentalCriticalPodAnnotation=true",
+	}
+
+	c := getKubeProxyContainer(clusterCIDR)
+	assert.NotNil(t, c, "container should not be nil")
+	assert.Equal(t, c.Name, kubeProxyContainerName, "Container name is not expected")
+	assert.Equal(t, c.ContainerProperties.Command, commands, "Command doesn't match")
+	assert.Equal(t, len(c.ContainerProperties.VolumeMounts), 1, "VolumeMounts number should be 1")
+	assert.Equal(t, c.ContainerProperties.VolumeMounts[0].Name, kubeConfigSecretVolume, "Volume name is not expected")
+	assert.Equal(t, c.ContainerProperties.VolumeMounts[0].MountPath, kubeConfigDir, "Volume mount path is not expected")
+	assert.NotNil(t, c.ContainerProperties.Resources.Requests, "Resource request should be specified")
+	assert.Equal(t, c.ContainerProperties.Resources.Requests.CPU, 0.1, "CPU request should be 0.1")
+	assert.Equal(t, c.ContainerProperties.Resources.Requests.MemoryInGB, 0.10, "CPU request should be 0.10")
+}
+
+func TestGetKubeProxyVolume(t *testing.T) {
+	ca := "this is a fake ca"
+	token := "this is a fake token"
+	masterURI := "this is a fake master URI"
+
+	dir, err := ioutil.TempDir("", "serviceaccount")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	defer os.RemoveAll(dir)
+
+	if err := ioutil.WriteFile(filepath.Join(dir, "ca.crt"), []byte(ca), 0666); err != nil {
+		t.Fatal(err)
+	}
+	if err := ioutil.WriteFile(filepath.Join(dir, "token"), []byte(token), 0666); err != nil {
+		t.Fatal(err)
+	}
+
+	var v *aci.Volume
+	v, err = getKubeProxyVolume(dir, masterURI)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	assert.NotNil(t, v, "volume should not be nil")
+	assert.Equal(t, v.Name, kubeConfigSecretVolume, "Volume name is not expected")
+	assert.NotNil(t, v.Secret, "Secret should not be nil")
+	assert.NotNil(t, v.Secret[kubeConfigFile], "kubeconfig should not be nil")
+}