From 169e733adbe44484ec9a6bbc5cc7fbfc38c73f7e Mon Sep 17 00:00:00 2001 From: Robbie Zhang Date: Thu, 30 Aug 2018 19:09:15 -0700 Subject: [PATCH] Adopt ACI Extension and DNSConfig --- providers/azure/aci.go | 140 +++++++++++----------- providers/azure/aci_test.go | 55 +-------- providers/azure/client/aci/client.go | 4 +- providers/azure/client/aci/client_test.go | 2 +- providers/azure/client/aci/types.go | 49 +++++++- 5 files changed, 122 insertions(+), 128 deletions(-) diff --git a/providers/azure/aci.go b/providers/azure/aci.go index 260af9d8f..182bd9577 100644 --- a/providers/azure/aci.go +++ b/providers/azure/aci.go @@ -45,13 +45,6 @@ const ( subnetsAction = "Microsoft.Network/virtualNetworks/subnets/action" subnetDelegationService = "Microsoft.ContainerInstance/containerGroups" networkProfileType = "Microsoft.Network/networkProfiles" - - // KubeProxy SideCar Container - kubeProxyContainerName = "vk-side-car-kube-proxy" - kubeProxyImageName = "mcr.microsoft.com/aci/sc-proxy:v1.9.10-master_20180816.1" - kubeConfigDir = "/etc/kube-proxy" - kubeConfigFile = "kubeconfig" - kubeConfigSecretVolume = "vk-side-car-kubeconfig-secret-volume" ) // ACIProvider implements the virtual-kubelet provider interface and communicates with Azure's ACI APIs. @@ -73,11 +66,9 @@ type ACIProvider struct { vnetName string vnetResourceGroup string networkProfile string - masterURI string - clusterCIDR string - kubeProxyContainer *aci.Container - kubeProxyVolume *aci.Volume - + kubeProxyExtension *aci.Extension + kubeDNSIP string + metricsSync sync.Mutex metricsSyncTime time.Time lastMetric *stats.Summary @@ -287,17 +278,20 @@ func NewACIProvider(config string, rm *manager.ResourceManager, nodeName, operat masterURI = "10.0.0.1" } - p.kubeProxyVolume, err = getKubeProxyVolume(serviceAccountSecretMountPath, masterURI) - if err != nil { - return nil, fmt.Errorf("error creating kube proxy volume spec: %v", err) - } - clusterCIDR := os.Getenv("CLUSTER_CIDR") if clusterCIDR == "" { clusterCIDR = "10.240.0.0/16" } - p.kubeProxyContainer = getKubeProxyContainer(clusterCIDR) + p.kubeProxyExtension, err = getKubeProxyExtension(serviceAccountSecretMountPath, masterURI, clusterCIDR) + if err != nil { + return nil, fmt.Errorf("error creating kube proxy extension: %v", err) + } + + p.kubeDNSIP = "10.0.0.10" + if kubeDNSIP := os.Getenv("KUBE_DNS_IP"); kubeDNSIP != "" { + p.kubeDNSIP = kubeDNSIP + } } return &p, err @@ -404,39 +398,7 @@ func populateNetworkProfile(p *network.Profile, subnet *network.Subnet) { }) } - -func getKubeProxyContainer(clusterCIDR string) *aci.Container { - container := aci.Container{ - Name: kubeProxyContainerName, - ContainerProperties: aci.ContainerProperties{ - Image: kubeProxyImageName, - Command: []string{ - "/bin/bash", - "-c", - "while true; do /setup_iptables.sh && /hyperkube proxy --kubeconfig="+kubeConfigDir+"/"+kubeConfigFile+" --cluster-cidr="+clusterCIDR+" --feature-gates=ExperimentalCriticalPodAnnotation=true; done", - }, - }, - } - - container.VolumeMounts = []aci.VolumeMount{ - aci.VolumeMount{ - Name: kubeConfigSecretVolume, - MountPath: kubeConfigDir, - ReadOnly: true, - }, - } - - container.Resources = aci.ResourceRequirements{ - Requests: &aci.ResourceRequests{ - CPU: 0.1, - MemoryInGB: 0.10, - }, - } - - return &container -} - -func getKubeProxyVolume(secretPath, masterURI string) (*aci.Volume, error) { +func getKubeProxyExtension(secretPath, masterURI, clusterCIDR string) (*aci.Extension, error) { ca, err := ioutil.ReadFile(secretPath + "/ca.crt") if err != nil { return nil, fmt.Errorf("failed to read ca.crt file: %v", err) @@ -487,15 +449,22 @@ func getKubeProxyVolume(secretPath, masterURI string) (*aci.Volume, error) { return nil, fmt.Errorf("failed to encode the kubeconfig: %v", err) } - paths := make(map[string]string) - paths[kubeConfigFile] = base64.StdEncoding.EncodeToString(b.Bytes()) - - volume := aci.Volume{ - Name: kubeConfigSecretVolume, - Secret: paths, + extension := aci.Extension{ + Name: "kube-proxy", + Properties: &aci.ExtensionProperties{ + Type: aci.ExtensionTypeKubeProxy, + Version: aci.ExtensionVersion1_0, + Settings: map[string]string{ + aci.KubeProxyExtensionSettingClusterCIDR: clusterCIDR, + aci.KubeProxyExtensionSettingKubeVersion: aci.KubeProxyExtensionKubeVersion, + }, + ProtectedSettings: map[string]string{ + aci.KubeProxyExtensionSettingKubeConfig: base64.StdEncoding.EncodeToString(b.Bytes()), + }, + }, } - return &volume, nil + return &extension, nil } // CreatePod accepts a Pod definition and creates @@ -566,7 +535,7 @@ func (p *ACIProvider) CreatePod(pod *v1.Pod) error { "CreationTimestamp": podCreationTimestamp, } - p.amendVnetResources(&containerGroup) + p.amendVnetResources(&containerGroup, pod) _, err = p.aciClient.CreateContainerGroup( p.resourceGroup, @@ -577,19 +546,54 @@ func (p *ACIProvider) CreatePod(pod *v1.Pod) error { return err } -func containerGroupName(pod *v1.Pod) string { - return fmt.Sprintf("%s-%s", pod.Namespace, pod.Name) -} - -func (p *ACIProvider) amendVnetResources(containerGroup *aci.ContainerGroup) { +func (p *ACIProvider) amendVnetResources(containerGroup *aci.ContainerGroup, pod *v1.Pod) { if p.networkProfile == "" { return } containerGroup.NetworkProfile = &aci.NetworkProfileDefinition{ID: p.networkProfile} - containerGroup.ContainerGroupProperties.Containers = append(containerGroup.ContainerGroupProperties.Containers, *(p.kubeProxyContainer)) - containerGroup.ContainerGroupProperties.Volumes = append(containerGroup.ContainerGroupProperties.Volumes, *(p.kubeProxyVolume)) + extensions := make([]aci.Extension, 0, 1) + extensions = append(extensions, *p.kubeProxyExtension) + + containerGroup.ContainerGroupProperties.Extensions = extensions + containerGroup.ContainerGroupProperties.DNSConfig = p.getDNSConfig(pod.Spec.DNSPolicy, pod.Spec.DNSConfig) +} + +func (p *ACIProvider) getDNSConfig(dnsPolicy v1.DNSPolicy, dnsConfig *v1.PodDNSConfig) *aci.DNSConfig { + nameServers := make([]string, 0) + + if dnsPolicy == v1.DNSClusterFirst || dnsPolicy == v1.DNSClusterFirstWithHostNet { + nameServers = append(nameServers, p.kubeDNSIP) + } + + var searchDomains string + options := []string{} + + if dnsConfig != nil { + nameServers = append(nameServers, dnsConfig.Nameservers...) + searchDomains = strings.Join(dnsConfig.Nameservers, " ") + + for _, option := range dnsConfig.Options { + op := option.Name + if option.Value != nil && *(option.Value) != "" { + op = op + ":" + *(option.Value) + } + options = append(options, op) + } + } + + if len(nameServers) == 0 { + return nil + } + + result := aci.DNSConfig{ + NameServers: nameServers, + SearchDomains: searchDomains, + Options: strings.Join(options, " "), + } + + return &result } func containerGroupName(pod *v1.Pod) string { @@ -1229,10 +1233,6 @@ func containerGroupToPod(cg *aci.ContainerGroup) (*v1.Pod, error) { containers := make([]v1.Container, 0, len(cg.Containers)) containerStatuses := make([]v1.ContainerStatus, 0, len(cg.Containers)) for _, c := range cg.Containers { - if strings.EqualFold(c.Name, kubeProxyContainerName) { - continue; - } - container := v1.Container{ Name: c.Name, Image: c.Image, diff --git a/providers/azure/aci_test.go b/providers/azure/aci_test.go index 74e86c718..8df3f62e5 100644 --- a/providers/azure/aci_test.go +++ b/providers/azure/aci_test.go @@ -652,57 +652,4 @@ func TestCreatePodWithReadinessProbe(t *testing.T) { if err := provider.CreatePod(pod); err != nil { t.Fatal("Failed to create pod", err) } -} - -func TestGetKubeProxyContainer(t *testing.T) { - clusterCIDR := "cidr-" + uuid.New().String() - commands := []string{ - "/hyperkube", - "proxy", - "--kubeconfig="+kubeConfigDir+"/"+kubeConfigFile, - "--cluster-cidr="+clusterCIDR, - "--feature-gates=ExperimentalCriticalPodAnnotation=true", - } - - c := getKubeProxyContainer(clusterCIDR) - assert.NotNil(t, c, "container should not be nil") - assert.Equal(t, c.Name, kubeProxyContainerName, "Container name is not expected") - assert.Equal(t, c.ContainerProperties.Command, commands, "Command doesn't match") - assert.Equal(t, len(c.ContainerProperties.VolumeMounts), 1, "VolumeMounts number should be 1") - assert.Equal(t, c.ContainerProperties.VolumeMounts[0].Name, kubeConfigSecretVolume, "Volume name is not expected") - assert.Equal(t, c.ContainerProperties.VolumeMounts[0].MountPath, kubeConfigDir, "Volume mount path is not expected") - assert.NotNil(t, c.ContainerProperties.Resources.Requests, "Resource request should be specified") - assert.Equal(t, c.ContainerProperties.Resources.Requests.CPU, 0.1, "CPU request should be 0.1") - assert.Equal(t, c.ContainerProperties.Resources.Requests.MemoryInGB, 0.10, "CPU request should be 0.10") -} - -func TestGetKubeProxyVolume(t *testing.T) { - ca := "this is a fake ca" - token := "this is a fake token" - masterURI := "this is a fake master URI" - - dir, err := ioutil.TempDir("", "serviceaccount") - if err != nil { - t.Fatal(err) - } - - defer os.RemoveAll(dir) - - if err := ioutil.WriteFile(filepath.Join(dir, "ca.crt"), []byte(ca), 0666); err != nil { - t.Fatal(err) - } - if err := ioutil.WriteFile(filepath.Join(dir, "token"), []byte(token), 0666); err != nil { - t.Fatal(err) - } - - var v *aci.Volume - v, err = getKubeProxyVolume(dir, masterURI) - if err != nil { - t.Fatal(err) - } - - assert.NotNil(t, v, "volume should not be nil") - assert.Equal(t, v.Name, kubeConfigSecretVolume, "Volume name is not expected") - assert.NotNil(t, v.Secret, "Secret should not be nil") - assert.NotNil(t, v.Secret[kubeConfigFile], "kubeconfig should not be nil") -} +} \ No newline at end of file diff --git a/providers/azure/client/aci/client.go b/providers/azure/client/aci/client.go index 958759895..e693a270f 100644 --- a/providers/azure/client/aci/client.go +++ b/providers/azure/client/aci/client.go @@ -10,8 +10,8 @@ import ( const ( // BaseURI is the default URI used for compute services. baseURI = "https://management.azure.com" - userAgent = "virtual-kubelet/azure-arm-aci/2018-07-01" - apiVersion = "2018-07-01" + userAgent = "virtual-kubelet/azure-arm-aci/2018-09-01" + apiVersion = "2018-09-01" containerGroupURLPath = "subscriptions/{{.subscriptionId}}/resourceGroups/{{.resourceGroup}}/providers/Microsoft.ContainerInstance/containerGroups/{{.containerGroupName}}" containerGroupListURLPath = "subscriptions/{{.subscriptionId}}/providers/Microsoft.ContainerInstance/containerGroups" diff --git a/providers/azure/client/aci/client_test.go b/providers/azure/client/aci/client_test.go index eb8fddf1b..9195bf35b 100644 --- a/providers/azure/client/aci/client_test.go +++ b/providers/azure/client/aci/client_test.go @@ -14,7 +14,7 @@ import ( var ( client *Client - location = "eastus2euap" + location = "westcentralus" resourceGroup = "virtual-kubelet-tests" containerGroup = "virtual-kubelet-test-container-group" subscriptionID string diff --git a/providers/azure/client/aci/types.go b/providers/azure/client/aci/types.go index 4ac29b5a4..4c7fa2501 100644 --- a/providers/azure/client/aci/types.go +++ b/providers/azure/client/aci/types.go @@ -95,6 +95,8 @@ type ContainerGroupProperties struct { InstanceView ContainerGroupPropertiesInstanceView `json:"instanceView,omitempty"` Diagnostics *ContainerGroupDiagnostics `json:"diagnostics,omitempty"` NetworkProfile *NetworkProfileDefinition `json:"networkProfile,omitempty"` + Extensions []Extension `json:"extension,omitempty"` + DNSConfig *DNSConfig `json:"dnsConfig,omitempty"` } // ContainerGroupPropertiesInstanceView is the instance view of the container group. Only valid in response. @@ -301,7 +303,7 @@ type ExecRequest struct { TerminalSize TerminalSize `json:"terminalSize,omitempty"` } -// ExecRequest is a request for Launch Exec API response for ACI. +// ExecResponse is a request for Launch Exec API response for ACI. type ExecResponse struct { WebSocketUri string `json:"webSocketUri,omitempty"` Password string `json:"password,omitempty"` @@ -423,3 +425,48 @@ const ( AggregationTypeAverage AggregationType = "average" AggregationTypeTotal AggregationType = "total" ) + +// Extension is the container group extension +type Extension struct { + Name string `json:"name"` + Properties *ExtensionProperties `json:"properties"` +} + +// ExtensionProperties is the properties for extension +type ExtensionProperties struct { + Type ExtensionType `json:"extensionType"` + Version ExtensionVersion `json:"version"` + Settings map[string]string `json:"settings,omitempty"` + ProtectedSettings map[string]string `json:"protectedSettings,omitempty"` +} + +// ExtensionType is an enum type for defining supported extension types +type ExtensionType string + +// Supported extension types +const ( + ExtensionTypeKubeProxy ExtensionType = "kube-proxy" +) + +// ExtensionVersion is an enum type for defining supported extension versions +type ExtensionVersion string + +// Supported extension version +const ( + ExtensionVersion1_0 ExtensionVersion = "1.0" +) + +// Supported kube-proxy extension constants +const ( + KubeProxyExtensionSettingClusterCIDR string = "clusterCidr" + KubeProxyExtensionSettingKubeVersion string = "kubeVersion" + KubeProxyExtensionSettingKubeConfig string = "kubeConfig" + KubeProxyExtensionKubeVersion string = "v1.9.10" +) + +// DNSConfig is the DNS config for container group +type DNSConfig struct { + NameServers []string `json:"nameServers"` + SearchDomains string `json:"searchDomains,omitempty"` + Options string `json:"options,omitempty"` +} \ No newline at end of file