From b364af21eaff137a0ced57f03839fef057a8d01e Mon Sep 17 00:00:00 2001 From: Robbie Zhang Date: Fri, 3 Aug 2018 11:27:38 -0700 Subject: [PATCH] Add the Kube-Proxy Side Car Container and Volume to the container group (#292) * Add Kube-Proxy Side Car Container * Add the Kube-Proxy Side Car Container * Add test * Bug fix --- providers/azure/aci.go | 176 +++++++++++++++++++++++++----------- providers/azure/aci_test.go | 54 +++++++++++ 2 files changed, 175 insertions(+), 55 deletions(-) diff --git a/providers/azure/aci.go b/providers/azure/aci.go index 9a75a6131..1147d0339 100644 --- a/providers/azure/aci.go +++ b/providers/azure/aci.go @@ -31,6 +31,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + clientcmdv1 "k8s.io/client-go/tools/clientcmd/api/v1" "k8s.io/client-go/tools/remotecommand" stats "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1" ) @@ -74,7 +75,9 @@ type ACIProvider struct { networkProfile string masterURI string clusterCIDR string - + kubeProxyContainer *aci.Container + kubeProxyVolume *aci.Volume + metricsSync sync.Mutex metricsSyncTime time.Time lastMetric *stats.Summary @@ -278,16 +281,23 @@ func NewACIProvider(config string, rm *manager.ResourceManager, nodeName, operat if err := p.setupNetworkProfile(azAuth); err != nil { return nil, fmt.Errorf("error setting up network profile: %v", err) } - } - p.masterURI = "10.0.0.1" - if masterURI := os.Getenv("MASTER_URI"); masterURI != "" { - p.masterURI = masterURI - } + masterURI := os.Getenv("MASTER_URI") + if masterURI == "" { + masterURI = "10.0.0.1" + } - p.clusterCIDR = "10.240.0.0/16" - if clusterCIDR := os.Getenv("CLUSTER_CIDR"); clusterCIDR != "" { - p.clusterCIDR = clusterCIDR + p.kubeProxyVolume, err = getKubeProxyVolume(serviceAccountSecretMountPath, masterURI) + if err != nil { + return nil, fmt.Errorf("error creating kube proxy volume spec: %v", err) + } + + clusterCIDR := os.Getenv("CLUSTER_CIDR") + if clusterCIDR == "" { + clusterCIDR = "10.240.0.0/16" + } + + p.kubeProxyContainer = getKubeProxyContainer(clusterCIDR) } return &p, err @@ -394,6 +404,102 @@ func populateNetworkProfile(p *network.Profile, subnet *network.Subnet) { }) } + +func getKubeProxyContainer(clusterCIDR string) *aci.Container { + container := aci.Container{ + Name: kubeProxyContainerName, + ContainerProperties: aci.ContainerProperties{ + Image: kubeProxyImageName, + Command: []string{ + "/hyperkube", + "proxy", + "--kubeconfig="+kubeConfigDir+"/"+kubeConfigFile, + "--cluster-cidr="+clusterCIDR, + "--feature-gates=ExperimentalCriticalPodAnnotation=true", + }, + }, + } + + container.VolumeMounts = []aci.VolumeMount{ + aci.VolumeMount{ + Name: kubeConfigSecretVolume, + MountPath: kubeConfigDir, + ReadOnly: true, + }, + } + + container.Resources = aci.ResourceRequirements{ + Requests: &aci.ResourceRequests{ + CPU: 0.1, + MemoryInGB: 0.10, + }, + } + + return &container +} + +func getKubeProxyVolume(secretPath, masterURI string) (*aci.Volume, error) { + ca, err := ioutil.ReadFile(secretPath + "/ca.crt") + if err != nil { + return nil, fmt.Errorf("failed to read ca.crt file: %v", err) + } + + var token []byte + token, err = ioutil.ReadFile(secretPath + "/token") + if err != nil { + return nil, fmt.Errorf("failed to read token file: %v", err) + } + + name := "virtual-kubelet" + + config := clientcmdv1.Config{ + APIVersion: "v1", + Kind: "Config", + Clusters: []clientcmdv1.NamedCluster{ + clientcmdv1.NamedCluster{ + Name: name, + Cluster: clientcmdv1.Cluster{ + Server: masterURI, + CertificateAuthorityData: ca, + }, + }, + }, + AuthInfos: []clientcmdv1.NamedAuthInfo{ + clientcmdv1.NamedAuthInfo{ + Name: name, + AuthInfo: clientcmdv1.AuthInfo{ + Token: string(token), + }, + }, + }, + Contexts: []clientcmdv1.NamedContext{ + clientcmdv1.NamedContext{ + Name: name, + Context: clientcmdv1.Context{ + Cluster: name, + AuthInfo: name, + }, + }, + }, + CurrentContext: name, + } + + b := new(bytes.Buffer) + if err := json.NewEncoder(b).Encode(config); err != nil { + return nil, fmt.Errorf("failed to encode the kubeconfig: %v", err) + } + + paths := make(map[string]string) + paths[kubeConfigFile] = base64.StdEncoding.EncodeToString(b.Bytes()) + + volume := aci.Volume{ + Name: kubeConfigSecretVolume, + Secret: paths, + } + + return &volume, nil +} + // CreatePod accepts a Pod definition and creates // an ACI deployment func (p *ACIProvider) CreatePod(pod *v1.Pod) error { @@ -484,52 +590,8 @@ func (p *ACIProvider) amendVnetResources(containerGroup *aci.ContainerGroup) { containerGroup.NetworkProfile = &aci.NetworkProfileDefinition{ID: p.networkProfile} - //containerGroup.ContainerGroupProperties.Containers = append(containerGroup.ContainerGroupProperties.Containers, *(getKubeProxyContainerSpec(p.clusterCIDR))) - //containerGroup.ContainerGroupProperties.Volumes = append(containerGroup.ContainerGroupProperties.Volumes, *(getKubeProxyVolumeSpec(p.masterURI))) -} - -func getKubeProxyContainerSpec(clusterCIDR string) *aci.Container { - container := aci.Container{ - Name: kubeProxyContainerName, - ContainerProperties: aci.ContainerProperties{ - Image: kubeProxyImageName, - Command: []string{ - "/hyperkube", - "proxy", - "--kubeconfig="+kubeConfigDir+"/"+kubeConfigFile, - "--cluster-cidr="+clusterCIDR, - }, - }, - } - - container.VolumeMounts = []aci.VolumeMount{ - aci.VolumeMount{ - Name: kubeConfigSecretVolume, - MountPath: kubeConfigDir, - ReadOnly: true, - }, - } - - container.Resources = aci.ResourceRequirements{ - Requests: &aci.ResourceRequests{ - CPU: 0.1, - MemoryInGB: 0.10, - }, - } - - return &container -} - -func getKubeProxyVolumeSpec(masterURI string) *aci.Volume { - paths := make(map[string]string) - - - volume := aci.Volume{ - Name: kubeConfigSecretVolume, - Secret: paths, - } - - return &volume + containerGroup.ContainerGroupProperties.Containers = append(containerGroup.ContainerGroupProperties.Containers, *(p.kubeProxyContainer)) + containerGroup.ContainerGroupProperties.Volumes = append(containerGroup.ContainerGroupProperties.Volumes, *(p.kubeProxyVolume)) } // UpdatePod is a noop, ACI currently does not support live updates of a pod. @@ -1165,6 +1227,10 @@ func containerGroupToPod(cg *aci.ContainerGroup) (*v1.Pod, error) { containers := make([]v1.Container, 0, len(cg.Containers)) containerStatuses := make([]v1.ContainerStatus, 0, len(cg.Containers)) for _, c := range cg.Containers { + if strings.EqualFold(c.Name, kubeProxyContainerName) { + continue; + } + container := v1.Container{ Name: c.Name, Image: c.Image, diff --git a/providers/azure/aci_test.go b/providers/azure/aci_test.go index 9bcfde51c..74e86c718 100644 --- a/providers/azure/aci_test.go +++ b/providers/azure/aci_test.go @@ -11,6 +11,7 @@ import ( "io/ioutil" "net/http" "os" + "path/filepath" "testing" "github.com/google/uuid" @@ -652,3 +653,56 @@ func TestCreatePodWithReadinessProbe(t *testing.T) { t.Fatal("Failed to create pod", err) } } + +func TestGetKubeProxyContainer(t *testing.T) { + clusterCIDR := "cidr-" + uuid.New().String() + commands := []string{ + "/hyperkube", + "proxy", + "--kubeconfig="+kubeConfigDir+"/"+kubeConfigFile, + "--cluster-cidr="+clusterCIDR, + "--feature-gates=ExperimentalCriticalPodAnnotation=true", + } + + c := getKubeProxyContainer(clusterCIDR) + assert.NotNil(t, c, "container should not be nil") + assert.Equal(t, c.Name, kubeProxyContainerName, "Container name is not expected") + assert.Equal(t, c.ContainerProperties.Command, commands, "Command doesn't match") + assert.Equal(t, len(c.ContainerProperties.VolumeMounts), 1, "VolumeMounts number should be 1") + assert.Equal(t, c.ContainerProperties.VolumeMounts[0].Name, kubeConfigSecretVolume, "Volume name is not expected") + assert.Equal(t, c.ContainerProperties.VolumeMounts[0].MountPath, kubeConfigDir, "Volume mount path is not expected") + assert.NotNil(t, c.ContainerProperties.Resources.Requests, "Resource request should be specified") + assert.Equal(t, c.ContainerProperties.Resources.Requests.CPU, 0.1, "CPU request should be 0.1") + assert.Equal(t, c.ContainerProperties.Resources.Requests.MemoryInGB, 0.10, "CPU request should be 0.10") +} + +func TestGetKubeProxyVolume(t *testing.T) { + ca := "this is a fake ca" + token := "this is a fake token" + masterURI := "this is a fake master URI" + + dir, err := ioutil.TempDir("", "serviceaccount") + if err != nil { + t.Fatal(err) + } + + defer os.RemoveAll(dir) + + if err := ioutil.WriteFile(filepath.Join(dir, "ca.crt"), []byte(ca), 0666); err != nil { + t.Fatal(err) + } + if err := ioutil.WriteFile(filepath.Join(dir, "token"), []byte(token), 0666); err != nil { + t.Fatal(err) + } + + var v *aci.Volume + v, err = getKubeProxyVolume(dir, masterURI) + if err != nil { + t.Fatal(err) + } + + assert.NotNil(t, v, "volume should not be nil") + assert.Equal(t, v.Name, kubeConfigSecretVolume, "Volume name is not expected") + assert.NotNil(t, v.Secret, "Secret should not be nil") + assert.NotNil(t, v.Secret[kubeConfigFile], "kubeconfig should not be nil") +}