Adopt ACI Extension and DNSConfig

providers/azure/aci.go

@@ -45,13 +45,6 @@ const (
 	subnetsAction           = "Microsoft.Network/virtualNetworks/subnets/action"
 	subnetDelegationService = "Microsoft.ContainerInstance/containerGroups"
 	networkProfileType      = "Microsoft.Network/networkProfiles"
-
-	// KubeProxy SideCar Container
-	kubeProxyContainerName  = "vk-side-car-kube-proxy"
-	kubeProxyImageName      = "mcr.microsoft.com/aci/sc-proxy:v1.9.10-master_20180816.1"
-	kubeConfigDir	        = "/etc/kube-proxy"
-	kubeConfigFile          = "kubeconfig"
-	kubeConfigSecretVolume  = "vk-side-car-kubeconfig-secret-volume"
 )
 
 // ACIProvider implements the virtual-kubelet provider interface and communicates with Azure's ACI APIs.
@@ -73,11 +66,9 @@ type ACIProvider struct {
 	vnetName           string
 	vnetResourceGroup  string
 	networkProfile     string
-	masterURI          string
+	kubeProxyExtension *aci.Extension
-	clusterCIDR        string
+	kubeDNSIP          string
-	kubeProxyContainer *aci.Container
+
-	kubeProxyVolume    *aci.Volume
-		
 	metricsSync     sync.Mutex
 	metricsSyncTime time.Time
 	lastMetric      *stats.Summary
@@ -287,17 +278,20 @@ func NewACIProvider(config string, rm *manager.ResourceManager, nodeName, operat
 			masterURI = "10.0.0.1"
 		}
 
-		p.kubeProxyVolume, err = getKubeProxyVolume(serviceAccountSecretMountPath, masterURI)
-		if err != nil {
-			return nil, fmt.Errorf("error creating kube proxy volume spec: %v", err)
-		}
-
 		clusterCIDR := os.Getenv("CLUSTER_CIDR")
 		if clusterCIDR == "" {
 			clusterCIDR = "10.240.0.0/16"
 		}
 
-		p.kubeProxyContainer = getKubeProxyContainer(clusterCIDR)
+		p.kubeProxyExtension, err = getKubeProxyExtension(serviceAccountSecretMountPath, masterURI, clusterCIDR)
+		if err != nil {
+			return nil, fmt.Errorf("error creating kube proxy extension: %v", err)
+		}
+
+		p.kubeDNSIP = "10.0.0.10"
+		if kubeDNSIP := os.Getenv("KUBE_DNS_IP"); kubeDNSIP != "" {
+			p.kubeDNSIP = kubeDNSIP
+		}
 	}
 
 	return &p, err
@@ -404,39 +398,7 @@ func populateNetworkProfile(p *network.Profile, subnet *network.Subnet) {
 	})
 }
 
-
+func getKubeProxyExtension(secretPath, masterURI, clusterCIDR string) (*aci.Extension, error) {
-func getKubeProxyContainer(clusterCIDR string) *aci.Container {
-	container := aci.Container{
-		Name: kubeProxyContainerName,
-		ContainerProperties: aci.ContainerProperties{
-			Image:   kubeProxyImageName,
-			Command: []string{
-				"/bin/bash",
-				"-c",
-				"while true; do /setup_iptables.sh && /hyperkube proxy --kubeconfig="+kubeConfigDir+"/"+kubeConfigFile+" --cluster-cidr="+clusterCIDR+" --feature-gates=ExperimentalCriticalPodAnnotation=true; done",
-			},
-		},
-	}
-
-	container.VolumeMounts = []aci.VolumeMount{
-		aci.VolumeMount{
-			Name:      kubeConfigSecretVolume,
-			MountPath: kubeConfigDir,
-			ReadOnly:  true,
-		},
-	}
-
-	container.Resources = aci.ResourceRequirements{
-		Requests: &aci.ResourceRequests{
-			CPU:        0.1,
-			MemoryInGB: 0.10,
-		},
-	}
-
-	return &container
-}
-
-func getKubeProxyVolume(secretPath, masterURI string) (*aci.Volume, error) {
 	ca, err := ioutil.ReadFile(secretPath + "/ca.crt")
 	if err != nil {
 		return nil, fmt.Errorf("failed to read ca.crt file: %v", err)
@@ -487,15 +449,22 @@ func getKubeProxyVolume(secretPath, masterURI string) (*aci.Volume, error) {
 		return nil, fmt.Errorf("failed to encode the kubeconfig: %v", err)
 	}
 
-	paths := make(map[string]string)
+	extension := aci.Extension{
-	paths[kubeConfigFile] = base64.StdEncoding.EncodeToString(b.Bytes())
+		Name:   "kube-proxy",
-
+		Properties: &aci.ExtensionProperties{
-	volume := aci.Volume{
+			Type:    aci.ExtensionTypeKubeProxy,
-		Name:   kubeConfigSecretVolume,
+			Version: aci.ExtensionVersion1_0,
-		Secret: paths,
+			Settings:          map[string]string{
+				aci.KubeProxyExtensionSettingClusterCIDR: clusterCIDR,
+				aci.KubeProxyExtensionSettingKubeVersion: aci.KubeProxyExtensionKubeVersion,
+			},
+			ProtectedSettings: map[string]string{
+				aci.KubeProxyExtensionSettingKubeConfig: base64.StdEncoding.EncodeToString(b.Bytes()),
+			},
+		},
 	}
 
-	return &volume, nil
+	return &extension, nil
 }
 
 // CreatePod accepts a Pod definition and creates
@@ -566,7 +535,7 @@ func (p *ACIProvider) CreatePod(pod *v1.Pod) error {
 		"CreationTimestamp": podCreationTimestamp,
 	}
 
-	p.amendVnetResources(&containerGroup)
+	p.amendVnetResources(&containerGroup, pod)
 
 	_, err = p.aciClient.CreateContainerGroup(
 		p.resourceGroup,
@@ -577,19 +546,54 @@ func (p *ACIProvider) CreatePod(pod *v1.Pod) error {
 	return err
 }
 
-func containerGroupName(pod *v1.Pod) string {
+func (p *ACIProvider) amendVnetResources(containerGroup *aci.ContainerGroup, pod *v1.Pod) {
-	return fmt.Sprintf("%s-%s", pod.Namespace, pod.Name)
-}
-
-func (p *ACIProvider) amendVnetResources(containerGroup *aci.ContainerGroup) {
 	if p.networkProfile == "" {
 		return
 	}
 
 	containerGroup.NetworkProfile = &aci.NetworkProfileDefinition{ID: p.networkProfile}
 
-	containerGroup.ContainerGroupProperties.Containers = append(containerGroup.ContainerGroupProperties.Containers, *(p.kubeProxyContainer))
+	extensions := make([]aci.Extension, 0, 1)
-	containerGroup.ContainerGroupProperties.Volumes = append(containerGroup.ContainerGroupProperties.Volumes, *(p.kubeProxyVolume))
+	extensions = append(extensions, *p.kubeProxyExtension)
+
+	containerGroup.ContainerGroupProperties.Extensions = extensions
+	containerGroup.ContainerGroupProperties.DNSConfig = p.getDNSConfig(pod.Spec.DNSPolicy, pod.Spec.DNSConfig)
+}
+
+func (p *ACIProvider) getDNSConfig(dnsPolicy v1.DNSPolicy, dnsConfig *v1.PodDNSConfig) *aci.DNSConfig {
+	nameServers := make([]string, 0)
+
+	if dnsPolicy == v1.DNSClusterFirst || dnsPolicy == v1.DNSClusterFirstWithHostNet {
+		nameServers = append(nameServers, p.kubeDNSIP)
+	}
+
+	var searchDomains string
+	options := []string{}
+
+	if dnsConfig != nil {
+		nameServers = append(nameServers, dnsConfig.Nameservers...)
+		searchDomains = strings.Join(dnsConfig.Nameservers, " ")
+
+		for _, option := range dnsConfig.Options {
+			op := option.Name
+			if option.Value != nil && *(option.Value) != "" {
+				op = op + ":" + *(option.Value)
+			}
+			options = append(options, op)
+		}
+	}
+
+	if len(nameServers) == 0 {
+		return nil
+	}
+
+	result := aci.DNSConfig{
+		NameServers: nameServers,
+		SearchDomains: searchDomains,
+		Options:       strings.Join(options, " "),
+	}
+
+	return &result
 }
 
 func containerGroupName(pod *v1.Pod) string {
@@ -1229,10 +1233,6 @@ func containerGroupToPod(cg *aci.ContainerGroup) (*v1.Pod, error) {
 	containers := make([]v1.Container, 0, len(cg.Containers))
 	containerStatuses := make([]v1.ContainerStatus, 0, len(cg.Containers))
 	for _, c := range cg.Containers {
-		if strings.EqualFold(c.Name, kubeProxyContainerName) {
-			continue;
-		}
-
 		container := v1.Container{
 			Name:    c.Name,
 			Image:   c.Image,

providers/azure/aci_test.go

@@ -652,57 +652,4 @@ func TestCreatePodWithReadinessProbe(t *testing.T) {
 	if err := provider.CreatePod(pod); err != nil {
 		t.Fatal("Failed to create pod", err)
 	}
-}
+}
-
-func TestGetKubeProxyContainer(t *testing.T) {
-	clusterCIDR := "cidr-" + uuid.New().String()
-	commands := []string{
-		"/hyperkube",
-		"proxy",
-		"--kubeconfig="+kubeConfigDir+"/"+kubeConfigFile,
-		"--cluster-cidr="+clusterCIDR,
-		"--feature-gates=ExperimentalCriticalPodAnnotation=true",
-	}
-
-	c := getKubeProxyContainer(clusterCIDR)
-	assert.NotNil(t, c, "container should not be nil")
-	assert.Equal(t, c.Name, kubeProxyContainerName, "Container name is not expected")
-	assert.Equal(t, c.ContainerProperties.Command, commands, "Command doesn't match")
-	assert.Equal(t, len(c.ContainerProperties.VolumeMounts), 1, "VolumeMounts number should be 1")
-	assert.Equal(t, c.ContainerProperties.VolumeMounts[0].Name, kubeConfigSecretVolume, "Volume name is not expected")
-	assert.Equal(t, c.ContainerProperties.VolumeMounts[0].MountPath, kubeConfigDir, "Volume mount path is not expected")
-	assert.NotNil(t, c.ContainerProperties.Resources.Requests, "Resource request should be specified")
-	assert.Equal(t, c.ContainerProperties.Resources.Requests.CPU, 0.1, "CPU request should be 0.1")
-	assert.Equal(t, c.ContainerProperties.Resources.Requests.MemoryInGB, 0.10, "CPU request should be 0.10")
-}
-
-func TestGetKubeProxyVolume(t *testing.T) {
-	ca := "this is a fake ca"
-	token := "this is a fake token"
-	masterURI := "this is a fake master URI"
-
-	dir, err := ioutil.TempDir("", "serviceaccount")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	defer os.RemoveAll(dir)
-
-	if err := ioutil.WriteFile(filepath.Join(dir, "ca.crt"), []byte(ca), 0666); err != nil {
-		t.Fatal(err)
-	}
-	if err := ioutil.WriteFile(filepath.Join(dir, "token"), []byte(token), 0666); err != nil {
-		t.Fatal(err)
-	}
-
-	var v *aci.Volume
-	v, err = getKubeProxyVolume(dir, masterURI)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	assert.NotNil(t, v, "volume should not be nil")
-	assert.Equal(t, v.Name, kubeConfigSecretVolume, "Volume name is not expected")
-	assert.NotNil(t, v.Secret, "Secret should not be nil")
-	assert.NotNil(t, v.Secret[kubeConfigFile], "kubeconfig should not be nil")
-}

providers/azure/client/aci/client.go

@@ -10,8 +10,8 @@ import (
 const (
 	// BaseURI is the default URI used for compute services.
 	baseURI    = "https://management.azure.com"
-	userAgent  = "virtual-kubelet/azure-arm-aci/2018-07-01"
+	userAgent  = "virtual-kubelet/azure-arm-aci/2018-09-01"
-	apiVersion = "2018-07-01"
+	apiVersion = "2018-09-01"
 
 	containerGroupURLPath                    = "subscriptions/{{.subscriptionId}}/resourceGroups/{{.resourceGroup}}/providers/Microsoft.ContainerInstance/containerGroups/{{.containerGroupName}}"
 	containerGroupListURLPath                = "subscriptions/{{.subscriptionId}}/providers/Microsoft.ContainerInstance/containerGroups"

providers/azure/client/aci/client_test.go

@@ -14,7 +14,7 @@ import (
 
 var (
 	client         *Client
-	location       = "eastus2euap"
+	location       = "westcentralus"
 	resourceGroup  = "virtual-kubelet-tests"
 	containerGroup = "virtual-kubelet-test-container-group"
 	subscriptionID string

providers/azure/client/aci/types.go

@@ -95,6 +95,8 @@ type ContainerGroupProperties struct {
 	InstanceView             ContainerGroupPropertiesInstanceView `json:"instanceView,omitempty"`
 	Diagnostics              *ContainerGroupDiagnostics           `json:"diagnostics,omitempty"`
 	NetworkProfile           *NetworkProfileDefinition            `json:"networkProfile,omitempty"`
+	Extensions               []Extension                          `json:"extension,omitempty"`
+	DNSConfig                *DNSConfig                           `json:"dnsConfig,omitempty"`
 }
 
 // ContainerGroupPropertiesInstanceView is the instance view of the container group. Only valid in response.
@@ -301,7 +303,7 @@ type ExecRequest struct {
 	TerminalSize TerminalSize `json:"terminalSize,omitempty"`
 }
 
-// ExecRequest is a request for Launch Exec API response for ACI.
+// ExecResponse is a request for Launch Exec API response for ACI.
 type ExecResponse struct {
 	WebSocketUri string `json:"webSocketUri,omitempty"`
 	Password     string `json:"password,omitempty"`
@@ -423,3 +425,48 @@ const (
 	AggregationTypeAverage AggregationType = "average"
 	AggregationTypeTotal   AggregationType = "total"
 )
+
+// Extension is the container group extension
+type Extension struct {
+	Name       string               `json:"name"`
+	Properties *ExtensionProperties `json:"properties"`
+}
+
+// ExtensionProperties is the properties for extension
+type ExtensionProperties struct {
+	Type              ExtensionType     `json:"extensionType"`
+	Version           ExtensionVersion  `json:"version"`
+	Settings          map[string]string `json:"settings,omitempty"`
+	ProtectedSettings map[string]string `json:"protectedSettings,omitempty"`
+}
+
+// ExtensionType is an enum type for defining supported extension types
+type ExtensionType string
+
+// Supported extension types
+const (
+	ExtensionTypeKubeProxy ExtensionType = "kube-proxy"
+)
+
+// ExtensionVersion is an enum type for defining supported extension versions
+type ExtensionVersion string
+
+// Supported extension version
+const (
+	ExtensionVersion1_0 ExtensionVersion = "1.0"
+)
+
+// Supported kube-proxy extension constants
+const (
+	KubeProxyExtensionSettingClusterCIDR string = "clusterCidr"
+	KubeProxyExtensionSettingKubeVersion string = "kubeVersion"
+	KubeProxyExtensionSettingKubeConfig  string = "kubeConfig"
+	KubeProxyExtensionKubeVersion        string = "v1.9.10"
+)
+
+// DNSConfig is the DNS config for container group
+type DNSConfig struct {
+	NameServers   []string `json:"nameServers"`
+	SearchDomains string   `json:"searchDomains,omitempty"`
+	Options       string   `json:"options,omitempty"`
+}