d8dfun/virtual-kubelet
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Add the Kube-Proxy Side Car Container and Volume to the container group (#292)

Browse Source 
* Add Kube-Proxy Side Car Container

* Add the Kube-Proxy Side Car Container

* Add test

* Bug fix
This commit is contained in:
Robbie Zhang
2018-08-03 11:27:38 -07:00
parent bac3a585da
commit b364af21ea
2 changed files with 175 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

176
providers/azure/aci.go
View File

@@ -31,6 +31,7 @@ import (
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
clientcmdv1 "k8s.io/client-go/tools/clientcmd/api/v1"
"k8s.io/client-go/tools/remotecommand"
stats "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
)
@@ -74,7 +75,9 @@ type ACIProvider struct {
networkProfile string
masterURI string
clusterCIDR string
kubeProxyContainer *aci.Container
kubeProxyVolume *aci.Volume
metricsSync sync.Mutex
metricsSyncTime time.Time
lastMetric *stats.Summary
@@ -278,16 +281,23 @@ func NewACIProvider(config string, rm *manager.ResourceManager, nodeName, operat
if err := p.setupNetworkProfile(azAuth); err != nil {
return nil, fmt.Errorf("error setting up network profile: %v", err)
}
}
p.masterURI = "10.0.0.1"
if masterURI := os.Getenv("MASTER_URI"); masterURI != "" {
p.masterURI = masterURI
}
masterURI := os.Getenv("MASTER_URI")
if masterURI == "" {
masterURI = "10.0.0.1"
}
p.clusterCIDR = "10.240.0.0/16"
if clusterCIDR := os.Getenv("CLUSTER_CIDR"); clusterCIDR != "" {
p.clusterCIDR = clusterCIDR
p.kubeProxyVolume, err = getKubeProxyVolume(serviceAccountSecretMountPath, masterURI)
if err != nil {
return nil, fmt.Errorf("error creating kube proxy volume spec: %v", err)
}
clusterCIDR := os.Getenv("CLUSTER_CIDR")
if clusterCIDR == "" {
clusterCIDR = "10.240.0.0/16"
}
p.kubeProxyContainer = getKubeProxyContainer(clusterCIDR)
}
return &p, err
@@ -394,6 +404,102 @@ func populateNetworkProfile(p *network.Profile, subnet *network.Subnet) {
})
}
func getKubeProxyContainer(clusterCIDR string) *aci.Container {
container := aci.Container{
Name: kubeProxyContainerName,
ContainerProperties: aci.ContainerProperties{
Image: kubeProxyImageName,
Command: []string{
"/hyperkube",
"proxy",
"--kubeconfig="+kubeConfigDir+"/"+kubeConfigFile,
"--cluster-cidr="+clusterCIDR,
"--feature-gates=ExperimentalCriticalPodAnnotation=true",
},
},
}
container.VolumeMounts = []aci.VolumeMount{
aci.VolumeMount{
Name: kubeConfigSecretVolume,
MountPath: kubeConfigDir,
ReadOnly: true,
},
}
container.Resources = aci.ResourceRequirements{
Requests: &aci.ResourceRequests{
CPU: 0.1,
MemoryInGB: 0.10,
},
}
return &container
}
func getKubeProxyVolume(secretPath, masterURI string) (*aci.Volume, error) {
ca, err := ioutil.ReadFile(secretPath + "/ca.crt")
if err != nil {
return nil, fmt.Errorf("failed to read ca.crt file: %v", err)
}
var token []byte
token, err = ioutil.ReadFile(secretPath + "/token")
if err != nil {
return nil, fmt.Errorf("failed to read token file: %v", err)
}
name := "virtual-kubelet"
config := clientcmdv1.Config{
APIVersion: "v1",
Kind: "Config",
Clusters: []clientcmdv1.NamedCluster{
clientcmdv1.NamedCluster{
Name: name,
Cluster: clientcmdv1.Cluster{
Server: masterURI,
CertificateAuthorityData: ca,
},
},
},
AuthInfos: []clientcmdv1.NamedAuthInfo{
clientcmdv1.NamedAuthInfo{
Name: name,
AuthInfo: clientcmdv1.AuthInfo{
Token: string(token),
},
},
},
Contexts: []clientcmdv1.NamedContext{
clientcmdv1.NamedContext{
Name: name,
Context: clientcmdv1.Context{
Cluster: name,
AuthInfo: name,
},
},
},
CurrentContext: name,
}
b := new(bytes.Buffer)
if err := json.NewEncoder(b).Encode(config); err != nil {
return nil, fmt.Errorf("failed to encode the kubeconfig: %v", err)
}
paths := make(map[string]string)
paths[kubeConfigFile] = base64.StdEncoding.EncodeToString(b.Bytes())
volume := aci.Volume{
Name: kubeConfigSecretVolume,
Secret: paths,
}
return &volume, nil
}
// CreatePod accepts a Pod definition and creates
// an ACI deployment
func (p *ACIProvider) CreatePod(pod *v1.Pod) error {
@@ -484,52 +590,8 @@ func (p *ACIProvider) amendVnetResources(containerGroup *aci.ContainerGroup) {
containerGroup.NetworkProfile = &aci.NetworkProfileDefinition{ID: p.networkProfile}
//containerGroup.ContainerGroupProperties.Containers = append(containerGroup.ContainerGroupProperties.Containers, *(getKubeProxyContainerSpec(p.clusterCIDR)))
//containerGroup.ContainerGroupProperties.Volumes = append(containerGroup.ContainerGroupProperties.Volumes, *(getKubeProxyVolumeSpec(p.masterURI)))
}
func getKubeProxyContainerSpec(clusterCIDR string) *aci.Container {
container := aci.Container{
Name: kubeProxyContainerName,
ContainerProperties: aci.ContainerProperties{
Image: kubeProxyImageName,
Command: []string{
"/hyperkube",
"proxy",
"--kubeconfig="+kubeConfigDir+"/"+kubeConfigFile,
"--cluster-cidr="+clusterCIDR,
},
},
}
container.VolumeMounts = []aci.VolumeMount{
aci.VolumeMount{
Name: kubeConfigSecretVolume,
MountPath: kubeConfigDir,
ReadOnly: true,
},
}
container.Resources = aci.ResourceRequirements{
Requests: &aci.ResourceRequests{
CPU: 0.1,
MemoryInGB: 0.10,
},
}
return &container
}
func getKubeProxyVolumeSpec(masterURI string) *aci.Volume {
paths := make(map[string]string)
volume := aci.Volume{
Name: kubeConfigSecretVolume,
Secret: paths,
}
return &volume
containerGroup.ContainerGroupProperties.Containers = append(containerGroup.ContainerGroupProperties.Containers, *(p.kubeProxyContainer))
containerGroup.ContainerGroupProperties.Volumes = append(containerGroup.ContainerGroupProperties.Volumes, *(p.kubeProxyVolume))
}
// UpdatePod is a noop, ACI currently does not support live updates of a pod.
@@ -1165,6 +1227,10 @@ func containerGroupToPod(cg *aci.ContainerGroup) (*v1.Pod, error) {
containers := make([]v1.Container, 0, len(cg.Containers))
containerStatuses := make([]v1.ContainerStatus, 0, len(cg.Containers))
for _, c := range cg.Containers {
if strings.EqualFold(c.Name, kubeProxyContainerName) {
continue;
}
container := v1.Container{
Name: c.Name,
Image: c.Image,

54
providers/azure/aci_test.go
View File

@@ -11,6 +11,7 @@ import (
"io/ioutil"
"net/http"
"os"
"path/filepath"
"testing"
"github.com/google/uuid"
@@ -652,3 +653,56 @@ func TestCreatePodWithReadinessProbe(t *testing.T) {
t.Fatal("Failed to create pod", err)
}
}
func TestGetKubeProxyContainer(t *testing.T) {
clusterCIDR := "cidr-" + uuid.New().String()
commands := []string{
"/hyperkube",
"proxy",
"--kubeconfig="+kubeConfigDir+"/"+kubeConfigFile,
"--cluster-cidr="+clusterCIDR,
"--feature-gates=ExperimentalCriticalPodAnnotation=true",
}
c := getKubeProxyContainer(clusterCIDR)
assert.NotNil(t, c, "container should not be nil")
assert.Equal(t, c.Name, kubeProxyContainerName, "Container name is not expected")
assert.Equal(t, c.ContainerProperties.Command, commands, "Command doesn't match")
assert.Equal(t, len(c.ContainerProperties.VolumeMounts), 1, "VolumeMounts number should be 1")
assert.Equal(t, c.ContainerProperties.VolumeMounts[0].Name, kubeConfigSecretVolume, "Volume name is not expected")
assert.Equal(t, c.ContainerProperties.VolumeMounts[0].MountPath, kubeConfigDir, "Volume mount path is not expected")
assert.NotNil(t, c.ContainerProperties.Resources.Requests, "Resource request should be specified")
assert.Equal(t, c.ContainerProperties.Resources.Requests.CPU, 0.1, "CPU request should be 0.1")
assert.Equal(t, c.ContainerProperties.Resources.Requests.MemoryInGB, 0.10, "CPU request should be 0.10")
}
func TestGetKubeProxyVolume(t *testing.T) {
ca := "this is a fake ca"
token := "this is a fake token"
masterURI := "this is a fake master URI"
dir, err := ioutil.TempDir("", "serviceaccount")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(dir)
if err := ioutil.WriteFile(filepath.Join(dir, "ca.crt"), []byte(ca), 0666); err != nil {
t.Fatal(err)
}
if err := ioutil.WriteFile(filepath.Join(dir, "token"), []byte(token), 0666); err != nil {
t.Fatal(err)
}
var v *aci.Volume
v, err = getKubeProxyVolume(dir, masterURI)
if err != nil {
t.Fatal(err)
}
assert.NotNil(t, v, "volume should not be nil")
assert.Equal(t, v.Name, kubeConfigSecretVolume, "Volume name is not expected")
assert.NotNil(t, v.Secret, "Secret should not be nil")
assert.NotNil(t, v.Secret[kubeConfigFile], "kubeconfig should not be nil")
}