d8dfun/virtual-kubelet
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Adopt ACI Extension and DNSConfig

Browse Source
This commit is contained in:
Robbie Zhang
2018-08-30 19:09:15 -07:00
parent 3ef512349b
commit 169e733adb
5 changed files with 122 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files

140
providers/azure/aci.go
View File

@@ -45,13 +45,6 @@ const (
subnetsAction = "Microsoft.Network/virtualNetworks/subnets/action"
subnetDelegationService = "Microsoft.ContainerInstance/containerGroups"
networkProfileType = "Microsoft.Network/networkProfiles"
// KubeProxy SideCar Container
kubeProxyContainerName = "vk-side-car-kube-proxy"
kubeProxyImageName = "mcr.microsoft.com/aci/sc-proxy:v1.9.10-master_20180816.1"
kubeConfigDir = "/etc/kube-proxy"
kubeConfigFile = "kubeconfig"
kubeConfigSecretVolume = "vk-side-car-kubeconfig-secret-volume"
)
// ACIProvider implements the virtual-kubelet provider interface and communicates with Azure's ACI APIs.
@@ -73,11 +66,9 @@ type ACIProvider struct {
vnetName string
vnetResourceGroup string
networkProfile string
masterURI string
clusterCIDR string
kubeProxyContainer *aci.Container
kubeProxyVolume *aci.Volume
kubeProxyExtension *aci.Extension
kubeDNSIP string
metricsSync sync.Mutex
metricsSyncTime time.Time
lastMetric *stats.Summary
@@ -287,17 +278,20 @@ func NewACIProvider(config string, rm *manager.ResourceManager, nodeName, operat
masterURI = "10.0.0.1"
}
p.kubeProxyVolume, err = getKubeProxyVolume(serviceAccountSecretMountPath, masterURI)
if err != nil {
return nil, fmt.Errorf("error creating kube proxy volume spec: %v", err)
}
clusterCIDR := os.Getenv("CLUSTER_CIDR")
if clusterCIDR == "" {
clusterCIDR = "10.240.0.0/16"
}
p.kubeProxyContainer = getKubeProxyContainer(clusterCIDR)
p.kubeProxyExtension, err = getKubeProxyExtension(serviceAccountSecretMountPath, masterURI, clusterCIDR)
if err != nil {
return nil, fmt.Errorf("error creating kube proxy extension: %v", err)
}
p.kubeDNSIP = "10.0.0.10"
if kubeDNSIP := os.Getenv("KUBE_DNS_IP"); kubeDNSIP != "" {
p.kubeDNSIP = kubeDNSIP
}
}
return &p, err
@@ -404,39 +398,7 @@ func populateNetworkProfile(p *network.Profile, subnet *network.Subnet) {
})
}
func getKubeProxyContainer(clusterCIDR string) *aci.Container {
container := aci.Container{
Name: kubeProxyContainerName,
ContainerProperties: aci.ContainerProperties{
Image: kubeProxyImageName,
Command: []string{
"/bin/bash",
"-c",
"while true; do /setup_iptables.sh && /hyperkube proxy --kubeconfig="+kubeConfigDir+"/"+kubeConfigFile+" --cluster-cidr="+clusterCIDR+" --feature-gates=ExperimentalCriticalPodAnnotation=true; done",
},
},
}
container.VolumeMounts = []aci.VolumeMount{
aci.VolumeMount{
Name: kubeConfigSecretVolume,
MountPath: kubeConfigDir,
ReadOnly: true,
},
}
container.Resources = aci.ResourceRequirements{
Requests: &aci.ResourceRequests{
CPU: 0.1,
MemoryInGB: 0.10,
},
}
return &container
}
func getKubeProxyVolume(secretPath, masterURI string) (*aci.Volume, error) {
func getKubeProxyExtension(secretPath, masterURI, clusterCIDR string) (*aci.Extension, error) {
ca, err := ioutil.ReadFile(secretPath + "/ca.crt")
if err != nil {
return nil, fmt.Errorf("failed to read ca.crt file: %v", err)
@@ -487,15 +449,22 @@ func getKubeProxyVolume(secretPath, masterURI string) (*aci.Volume, error) {
return nil, fmt.Errorf("failed to encode the kubeconfig: %v", err)
}
paths := make(map[string]string)
paths[kubeConfigFile] = base64.StdEncoding.EncodeToString(b.Bytes())
volume := aci.Volume{
Name: kubeConfigSecretVolume,
Secret: paths,
extension := aci.Extension{
Name: "kube-proxy",
Properties: &aci.ExtensionProperties{
Type: aci.ExtensionTypeKubeProxy,
Version: aci.ExtensionVersion1_0,
Settings: map[string]string{
aci.KubeProxyExtensionSettingClusterCIDR: clusterCIDR,
aci.KubeProxyExtensionSettingKubeVersion: aci.KubeProxyExtensionKubeVersion,
},
ProtectedSettings: map[string]string{
aci.KubeProxyExtensionSettingKubeConfig: base64.StdEncoding.EncodeToString(b.Bytes()),
},
},
}
return &volume, nil
return &extension, nil
}
// CreatePod accepts a Pod definition and creates
@@ -566,7 +535,7 @@ func (p *ACIProvider) CreatePod(pod *v1.Pod) error {
"CreationTimestamp": podCreationTimestamp,
}
p.amendVnetResources(&containerGroup)
p.amendVnetResources(&containerGroup, pod)
_, err = p.aciClient.CreateContainerGroup(
p.resourceGroup,
@@ -577,19 +546,54 @@ func (p *ACIProvider) CreatePod(pod *v1.Pod) error {
return err
}
func containerGroupName(pod *v1.Pod) string {
return fmt.Sprintf("%s-%s", pod.Namespace, pod.Name)
}
func (p *ACIProvider) amendVnetResources(containerGroup *aci.ContainerGroup) {
func (p *ACIProvider) amendVnetResources(containerGroup *aci.ContainerGroup, pod *v1.Pod) {
if p.networkProfile == "" {
return
}
containerGroup.NetworkProfile = &aci.NetworkProfileDefinition{ID: p.networkProfile}
containerGroup.ContainerGroupProperties.Containers = append(containerGroup.ContainerGroupProperties.Containers, *(p.kubeProxyContainer))
containerGroup.ContainerGroupProperties.Volumes = append(containerGroup.ContainerGroupProperties.Volumes, *(p.kubeProxyVolume))
extensions := make([]aci.Extension, 0, 1)
extensions = append(extensions, *p.kubeProxyExtension)
containerGroup.ContainerGroupProperties.Extensions = extensions
containerGroup.ContainerGroupProperties.DNSConfig = p.getDNSConfig(pod.Spec.DNSPolicy, pod.Spec.DNSConfig)
}
func (p *ACIProvider) getDNSConfig(dnsPolicy v1.DNSPolicy, dnsConfig *v1.PodDNSConfig) *aci.DNSConfig {
nameServers := make([]string, 0)
if dnsPolicy == v1.DNSClusterFirst || dnsPolicy == v1.DNSClusterFirstWithHostNet {
nameServers = append(nameServers, p.kubeDNSIP)
}
var searchDomains string
options := []string{}
if dnsConfig != nil {
nameServers = append(nameServers, dnsConfig.Nameservers...)
searchDomains = strings.Join(dnsConfig.Nameservers, " ")
for _, option := range dnsConfig.Options {
op := option.Name
if option.Value != nil && *(option.Value) != "" {
op = op + ":" + *(option.Value)
}
options = append(options, op)
}
}
if len(nameServers) == 0 {
return nil
}
result := aci.DNSConfig{
NameServers: nameServers,
SearchDomains: searchDomains,
Options: strings.Join(options, " "),
}
return &result
}
func containerGroupName(pod *v1.Pod) string {
@@ -1229,10 +1233,6 @@ func containerGroupToPod(cg *aci.ContainerGroup) (*v1.Pod, error) {
containers := make([]v1.Container, 0, len(cg.Containers))
containerStatuses := make([]v1.ContainerStatus, 0, len(cg.Containers))
for _, c := range cg.Containers {
if strings.EqualFold(c.Name, kubeProxyContainerName) {
continue;
}
container := v1.Container{
Name: c.Name,
Image: c.Image,

55
providers/azure/aci_test.go
View File

@@ -652,57 +652,4 @@ func TestCreatePodWithReadinessProbe(t *testing.T) {
if err := provider.CreatePod(pod); err != nil {
t.Fatal("Failed to create pod", err)
}
}
func TestGetKubeProxyContainer(t *testing.T) {
clusterCIDR := "cidr-" + uuid.New().String()
commands := []string{
"/hyperkube",
"proxy",
"--kubeconfig="+kubeConfigDir+"/"+kubeConfigFile,
"--cluster-cidr="+clusterCIDR,
"--feature-gates=ExperimentalCriticalPodAnnotation=true",
}
c := getKubeProxyContainer(clusterCIDR)
assert.NotNil(t, c, "container should not be nil")
assert.Equal(t, c.Name, kubeProxyContainerName, "Container name is not expected")
assert.Equal(t, c.ContainerProperties.Command, commands, "Command doesn't match")
assert.Equal(t, len(c.ContainerProperties.VolumeMounts), 1, "VolumeMounts number should be 1")
assert.Equal(t, c.ContainerProperties.VolumeMounts[0].Name, kubeConfigSecretVolume, "Volume name is not expected")
assert.Equal(t, c.ContainerProperties.VolumeMounts[0].MountPath, kubeConfigDir, "Volume mount path is not expected")
assert.NotNil(t, c.ContainerProperties.Resources.Requests, "Resource request should be specified")
assert.Equal(t, c.ContainerProperties.Resources.Requests.CPU, 0.1, "CPU request should be 0.1")
assert.Equal(t, c.ContainerProperties.Resources.Requests.MemoryInGB, 0.10, "CPU request should be 0.10")
}
func TestGetKubeProxyVolume(t *testing.T) {
ca := "this is a fake ca"
token := "this is a fake token"
masterURI := "this is a fake master URI"
dir, err := ioutil.TempDir("", "serviceaccount")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(dir)
if err := ioutil.WriteFile(filepath.Join(dir, "ca.crt"), []byte(ca), 0666); err != nil {
t.Fatal(err)
}
if err := ioutil.WriteFile(filepath.Join(dir, "token"), []byte(token), 0666); err != nil {
t.Fatal(err)
}
var v *aci.Volume
v, err = getKubeProxyVolume(dir, masterURI)
if err != nil {
t.Fatal(err)
}
assert.NotNil(t, v, "volume should not be nil")
assert.Equal(t, v.Name, kubeConfigSecretVolume, "Volume name is not expected")
assert.NotNil(t, v.Secret, "Secret should not be nil")
assert.NotNil(t, v.Secret[kubeConfigFile], "kubeconfig should not be nil")
}
}

4
providers/azure/client/aci/client.go
View File

@@ -10,8 +10,8 @@ import (
const (
// BaseURI is the default URI used for compute services.
baseURI = "https://management.azure.com"
userAgent = "virtual-kubelet/azure-arm-aci/2018-07-01"
apiVersion = "2018-07-01"
userAgent = "virtual-kubelet/azure-arm-aci/2018-09-01"
apiVersion = "2018-09-01"
containerGroupURLPath = "subscriptions/{{.subscriptionId}}/resourceGroups/{{.resourceGroup}}/providers/Microsoft.ContainerInstance/containerGroups/{{.containerGroupName}}"
containerGroupListURLPath = "subscriptions/{{.subscriptionId}}/providers/Microsoft.ContainerInstance/containerGroups"

2
providers/azure/client/aci/client_test.go
View File

@@ -14,7 +14,7 @@ import (
var (
client *Client
location = "eastus2euap"
location = "westcentralus"
resourceGroup = "virtual-kubelet-tests"
containerGroup = "virtual-kubelet-test-container-group"
subscriptionID string

49
providers/azure/client/aci/types.go
View File

@@ -95,6 +95,8 @@ type ContainerGroupProperties struct {
InstanceView ContainerGroupPropertiesInstanceView `json:"instanceView,omitempty"`
Diagnostics *ContainerGroupDiagnostics `json:"diagnostics,omitempty"`
NetworkProfile *NetworkProfileDefinition `json:"networkProfile,omitempty"`
Extensions []Extension `json:"extension,omitempty"`
DNSConfig *DNSConfig `json:"dnsConfig,omitempty"`
}
// ContainerGroupPropertiesInstanceView is the instance view of the container group. Only valid in response.
@@ -301,7 +303,7 @@ type ExecRequest struct {
TerminalSize TerminalSize `json:"terminalSize,omitempty"`
}
// ExecRequest is a request for Launch Exec API response for ACI.
// ExecResponse is a request for Launch Exec API response for ACI.
type ExecResponse struct {
WebSocketUri string `json:"webSocketUri,omitempty"`
Password string `json:"password,omitempty"`
@@ -423,3 +425,48 @@ const (
AggregationTypeAverage AggregationType = "average"
AggregationTypeTotal AggregationType = "total"
)
// Extension is the container group extension
type Extension struct {
Name string `json:"name"`
Properties *ExtensionProperties `json:"properties"`
}
// ExtensionProperties is the properties for extension
type ExtensionProperties struct {
Type ExtensionType `json:"extensionType"`
Version ExtensionVersion `json:"version"`
Settings map[string]string `json:"settings,omitempty"`
ProtectedSettings map[string]string `json:"protectedSettings,omitempty"`
}
// ExtensionType is an enum type for defining supported extension types
type ExtensionType string
// Supported extension types
const (
ExtensionTypeKubeProxy ExtensionType = "kube-proxy"
)
// ExtensionVersion is an enum type for defining supported extension versions
type ExtensionVersion string
// Supported extension version
const (
ExtensionVersion1_0 ExtensionVersion = "1.0"
)
// Supported kube-proxy extension constants
const (
KubeProxyExtensionSettingClusterCIDR string = "clusterCidr"
KubeProxyExtensionSettingKubeVersion string = "kubeVersion"
KubeProxyExtensionSettingKubeConfig string = "kubeConfig"
KubeProxyExtensionKubeVersion string = "v1.9.10"
)
// DNSConfig is the DNS config for container group
type DNSConfig struct {
NameServers []string `json:"nameServers"`
SearchDomains string `json:"searchDomains,omitempty"`
Options string `json:"options,omitempty"`
}