d8dfun/virtual-kubelet
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Huawei Cloud Provider implementation (#241)

Browse Source 
* add huawei CCI provider

* add readme

* add vender

* add huawei provider mock test
This commit is contained in:
Fei Xu
2018-06-30 01:21:15 +08:00
committed by Robbie Zhang
parent df6a8750bb
commit a30303035f
34 changed files with 3884 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

85
providers/huawei/README.md Normal file
View File

@@ -0,0 +1,85 @@
# Huawei CCI
Huawei CCI [(Cloud Container Instance)](https://www.huaweicloud.com/product/cci.html) service provides serverless container management,
and does not require users to manage the cluster and the server.
Only through simple configuration, users can enjoy the agility and high performance of the container.
CCI supports stateless workloads (Deployment) and stateful workload (StatefulSet).
On the basis of Kubernetes, we have made a series of important enhancements such as secure container,
elastic load balancing, elastic scalability, Blue Green Deployment and so on.
## Huawei CCI Virtual Kubelet Provider
Huawei CCI virtual kubelet provider configures a CCI project as node in any of your Kubernetes cluster,
such as Huawei CCE [(Cloud Container Engine)](https://www.huaweicloud.com/en-us/product/cce.html).
CCE supports native Kubernetes applications and tools as private cluster, allowing you to easily set up a container runtime environment.
Pod which is scheduled to the virtual kubelet provider will run in the CCI, that will makes good use of the high performance of CCI.
The diagram below illustrates how Huawei CCI virtual kubelet provider works.
![diagram](cci-provider.svg)
**NOTE:** The Huawei CCI virtual-kubelet provider is in the early stages of development,
and don't use it in a production environment.
## Prerequisites
You must install the provider in a Kubernetes cluster and connect to the CCI, and also need create an account for CCI.
Once you've created your account, then need to record the aksk, region for the configuration in next step.
## Configuration
Before run CCI Virtual Kubelet Provider, you must do as the following steps.
1. Create a configuration profile.
You need to provide the fields you specify like in the [example fils](cci.toml).
2. Copy your AKSK and save them in environment variable:
```console
export APP_KEY="<AppKey>"
export APP_SECRET="<AppSecret>"
```
## Connect to CCI from your cluster via Virtual Kubelet
On the Kubernetes work node, starting a virtual-kubelet process as follows.
```console
virtual-kubelet --provider huawei --provider-config cci.toml
```
Then run ``kubectl get nodes`` in your cluster to validate the provider has been running as a node.
```console
kubectl get nodes
NAME STATUS AGE
virtual-kubelet Ready 5m
cce-192.168.0.178 Ready 10d
cce-192.168.0.233 Ready 10d
```
If want to stop the virtual kubelet, just stop the virtual kubelet process.
## Schedule pod to CCI via Virtual Kubelet
```console
apiVersion: v1
kind: Pod
metadata:
name: myapp
labels:
app: myapp
spec:
nodeName: virtual-kubelet
containers:
- name: nginx
image: 1and1internet/ubuntu-16-nginx
imagePullPolicy: IfNotPresent
ports:
- containerPort: 8080
tolerations:
- key: huawei.com/cci
effect: NoSchedule
```
Replace the nodeName to the virtual-kubelet nodename and save the configuration to a file ``virtual-kubelet-pod.yaml``.
Then run ``kubectl create -f virtual-kubelet-pod.yaml`` to create the pod. Run ``kubectl get pods -owide`` to get pods.
```console
kubectl get pods -o wide
NAME READY STATUS RESTARTS AGE IP NODE
myapp-7c7877989-vbffm 1/1 Running 0 39s 172.17.0.3 virtual-kubelet
```

309
providers/huawei/auth/auth.go Normal file
View File

@@ -0,0 +1,309 @@
package auth
// HWS API Gateway Signature
// Analog to AWS Signature Version 4, with some HWS specific parameters
// Please refer to: http://docs.aws.amazon.com/general/latest/gr/signature-version-4.html
import (
"bytes"
"crypto/hmac"
"crypto/sha256"
"fmt"
"io/ioutil"
"net/http"
"sort"
"strings"
"time"
)
// BasicDateFormat and BasicDateFormatShort define aws-date format
const (
BasicDateFormat = "20060102T150405Z"
BasicDateFormatShort = "20060102"
TerminationString = "sdk_request"
Algorithm = "SDK-HMAC-SHA256"
PreSKString = "SDK"
HeaderXDate = "x-sdk-date"
HeaderDate = "date"
HeaderHost = "host"
HeaderAuthorization = "Authorization"
HeaderContentSha256 = "x-sdk-content-sha256"
// todo: use the region and service.
DefaultRegion = "default"
DefaultService = "apigateway"
)
func shouldEscape(c byte) bool {
if 'A' <= c && c <= 'Z' || 'a' <= c && c <= 'z' || '0' <= c && c <= '9' || c == '_' || c == '-' || c == '~' || c == '.' {
return false
}
return true
}
func escape(s string) string {
hexCount := 0
for i := 0; i < len(s); i++ {
c := s[i]
if shouldEscape(c) {
hexCount++
}
}
if hexCount == 0 {
return s
}
t := make([]byte, len(s)+2*hexCount)
j := 0
for i := 0; i < len(s); i++ {
switch c := s[i]; {
case shouldEscape(c):
t[j] = '%'
t[j+1] = "0123456789ABCDEF"[c>>4]
t[j+2] = "0123456789ABCDEF"[c&15]
j += 3
default:
t[j] = s[i]
j++
}
}
return string(t)
}
func hmacsha256(key []byte, data string) ([]byte, error) {
h := hmac.New(sha256.New, []byte(key))
if _, err := h.Write([]byte(data)); err != nil {
return nil, err
}
return h.Sum(nil), nil
}
// Build a CanonicalRequest from a regular request string
//
// See http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
// CanonicalRequest =
// HTTPRequestMethod + '\n' +
// CanonicalURI + '\n' +
// CanonicalQueryString + '\n' +
// CanonicalHeaders + '\n' +
// SignedHeaders + '\n' +
// HexEncode(Hash(RequestPayload))
func CanonicalRequest(r *http.Request) (string, error) {
var hexencode string
var err error
if hex := r.Header.Get(HeaderContentSha256); hex != "" {
hexencode = hex
} else {
data, err := RequestPayload(r)
if err != nil {
return "", err
}
hexencode, err = HexEncodeSHA256Hash(data)
}
return fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s", r.Method, CanonicalURI(r), CanonicalQueryString(r), CanonicalHeaders(r), SignedHeaders(r), hexencode), err
}
// CanonicalURI returns request uri
func CanonicalURI(r *http.Request) string {
pattens := strings.Split(r.URL.Path, "/")
var uri []string
for _, v := range pattens {
switch v {
case "":
continue
case ".":
continue
case "..":
if len(uri) > 0 {
uri = uri[:len(uri)-1]
}
default:
uri = append(uri, escape(v))
}
}
urlpath := "/"
if len(uri) > 0 {
urlpath = urlpath + strings.Join(uri, "/") + "/"
}
return urlpath
}
// CanonicalQueryString
func CanonicalQueryString(r *http.Request) string {
var a []string
for key, value := range r.URL.Query() {
k := escape(key)
for _, v := range value {
var kv string
if v == "" {
kv = k
} else {
kv = fmt.Sprintf("%s=%s", k, escape(v))
}
a = append(a, kv)
}
}
sort.Strings(a)
query := strings.Join(a, "&")
r.URL.RawQuery = query
return query
}
// CanonicalHeaders
func CanonicalHeaders(r *http.Request) string {
var a []string
for key, value := range r.Header {
sort.Strings(value)
var q []string
for _, v := range value {
q = append(q, trimString(v))
}
a = append(a, strings.ToLower(key)+":"+strings.Join(q, ","))
}
a = append(a, HeaderHost+":"+r.Host)
sort.Strings(a)
return fmt.Sprintf("%s\n", strings.Join(a, "\n"))
}
// SignedHeaders
func SignedHeaders(r *http.Request) string {
var a []string
for key := range r.Header {
a = append(a, strings.ToLower(key))
}
a = append(a, HeaderHost)
sort.Strings(a)
return fmt.Sprintf("%s", strings.Join(a, ";"))
}
// RequestPayload
func RequestPayload(r *http.Request) ([]byte, error) {
if r.Body == nil {
return []byte(""), nil
}
b, err := ioutil.ReadAll(r.Body)
r.Body = ioutil.NopCloser(bytes.NewBuffer(b))
return b, err
}
// Return the Credential Scope. See http://docs.aws.amazon.com/general/latest/gr/sigv4-create-string-to-sign.html
func CredentialScope(t time.Time, regionName, serviceName string) string {
return fmt.Sprintf("%s/%s/%s/%s", t.UTC().Format(BasicDateFormatShort), regionName, serviceName, TerminationString)
}
// Create a "String to Sign". See http://docs.aws.amazon.com/general/latest/gr/sigv4-create-string-to-sign.html
func StringToSign(canonicalRequest, credentialScope string, t time.Time) string {
hash := sha256.New()
hash.Write([]byte(canonicalRequest))
return fmt.Sprintf("%s\n%s\n%s\n%x",
Algorithm, t.UTC().Format(BasicDateFormat), credentialScope, hash.Sum(nil))
}
// Generate a "signing key" to sign the "String To Sign". See http://docs.aws.amazon.com/general/latest/gr/sigv4-calculate-signature.html
func GenerateSigningKey(secretKey, regionName, serviceName string, t time.Time) ([]byte, error) {
key := []byte(PreSKString + secretKey)
var err error
dateStamp := t.UTC().Format(BasicDateFormatShort)
data := []string{dateStamp, regionName, serviceName, TerminationString}
for _, d := range data {
key, err = hmacsha256(key, d)
if err != nil {
return nil, err
}
}
return key, nil
}
// Create the HWS Signature. See http://docs.aws.amazon.com/general/latest/gr/sigv4-calculate-signature.html
func SignStringToSign(stringToSign string, signingKey []byte) (string, error) {
hm, err := hmacsha256(signingKey, stringToSign)
return fmt.Sprintf("%x", hm), err
}
// HexEncodeSHA256Hash returns hexcode of sha256
func HexEncodeSHA256Hash(body []byte) (string, error) {
hash := sha256.New()
if body == nil {
body = []byte("")
}
_, err := hash.Write(body)
return fmt.Sprintf("%x", hash.Sum(nil)), err
}
// Get the finalized value for the "Authorization" header. The signature parameter is the output from SignStringToSign
func AuthHeaderValue(signature, accessKey, credentialScope, signedHeaders string) string {
return fmt.Sprintf("%s Credential=%s/%s, SignedHeaders=%s, Signature=%s", Algorithm, accessKey, credentialScope, signedHeaders, signature)
}
func trimString(s string) string {
var trimedString []byte
inQuote := false
var lastChar byte
s = strings.TrimSpace(s)
for _, v := range []byte(s) {
if byte(v) == byte('"') {
inQuote = !inQuote
}
if lastChar == byte(' ') && byte(v) == byte(' ') && !inQuote {
continue
}
trimedString = append(trimedString, v)
lastChar = v
}
return string(trimedString)
}
type Signer interface {
Sign(*http.Request) error
}
// Signature HWS meta
type SignerHws struct {
AppKey string
AppSecret string
Region string
Service string
}
// SignRequest set Authorization header
func (s *SignerHws) Sign(r *http.Request) error {
var t time.Time
var err error
var dt string
if dt = r.Header.Get(HeaderXDate); dt != "" {
t, err = time.Parse(BasicDateFormat, dt)
} else if dt = r.Header.Get(HeaderDate); dt != "" {
t, err = time.Parse(time.RFC1123, dt)
}
if err != nil || dt == "" {
r.Header.Del(HeaderDate)
t = time.Now()
r.Header.Set(HeaderXDate, t.UTC().Format(BasicDateFormat))
}
canonicalRequest, err := CanonicalRequest(r)
if err != nil {
return err
}
Region := DefaultRegion
Service := DefaultService
if s.Region != "" {
Region = s.Region
}
if s.Service != "" {
Service = s.Service
}
credentialScope := CredentialScope(t, Region, Service)
stringToSign := StringToSign(canonicalRequest, credentialScope, t)
key, err := GenerateSigningKey(s.AppSecret, Region, Service, t)
if err != nil {
return err
}
signature, err := SignStringToSign(stringToSign, key)
if err != nil {
return err
}
signedHeaders := SignedHeaders(r)
authValue := AuthHeaderValue(signature, s.AppKey, credentialScope, signedHeaders)
r.Header.Set(HeaderAuthorization, authValue)
return nil
}

784
providers/huawei/cci-provider.svg Normal file
View File

@@ -0,0 +1,784 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
<!-- 由 Microsoft Visio, SVG Export 生成 cci-provider.svg Page-1 -->
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:ev="http://www.w3.org/2001/xml-events"
xmlns:v="http://schemas.microsoft.com/visio/2003/SVGExtensions/" width="8.02082in" height="4.62889in"
viewBox="0 0 577.499 333.28" xml:space="preserve" color-interpolation-filters="sRGB" class="st19">
<v:documentProperties v:langID="2052" v:metric="true" v:viewMarkup="false">
<v:userDefs>
<v:ud v:nameU="msvSubprocessMaster" v:prompt="" v:val="VT4(Rectangle)"/>
<v:ud v:nameU="msvNoAutoConnect" v:val="VT0(1):26"/>
</v:userDefs>
</v:documentProperties>
<style type="text/css">
<![CDATA[
.st1 {fill:#ffffff;stroke:#41719c;stroke-width:0.75}
.st2 {visibility:visible}
.st3 {fill:#5b9bd5;fill-opacity:0.22;filter:url(#filter_2);stroke:#5b9bd5;stroke-opacity:0.22}
.st4 {fill:#ffffff;stroke:#5b9bd5;stroke-width:0.25}
.st5 {fill:#e2efd9;stroke:#61973d;stroke-dasharray:5.25,3.75;stroke-width:0.75}
.st6 {fill:none;stroke:none;stroke-width:0.25}
.st7 {fill:#000000;font-family:Calibri;font-size:1.33333em}
.st8 {fill:#000000;font-family:Calibri;font-size:1.00001em}
.st9 {fill:#c5e0b3;stroke:#c7c8c8;stroke-width:0.25}
.st10 {fill:#000000;font-family:Calibri;font-size:1.00001em;font-weight:bold}
.st11 {marker-end:url(#mrkr1-32);stroke:#000000;stroke-linecap:round;stroke-linejoin:round;stroke-width:1}
.st12 {fill:#000000;fill-opacity:1;stroke:#000000;stroke-opacity:1;stroke-width:0.28409090909091}
.st13 {fill:#9cc3e5;stroke:#c7c8c8;stroke-width:0.25}
.st14 {fill:#000000;font-family:Calibri;font-size:1.16666em;font-weight:bold}
.st15 {fill:#000000;font-family:Calibri;font-size:1.33333em;font-weight:bold}
.st16 {fill:#ffffff;stroke:#41719c;stroke-dasharray:3.5,2.5;stroke-width:0.5}
.st17 {fill:#e2efd9;stroke:#61973d;stroke-width:0.75}
.st18 {fill:#000000;font-family:Calibri;font-size:1.16666em}
.st19 {fill:none;fill-rule:evenodd;font-size:12px;overflow:visible;stroke-linecap:square;stroke-miterlimit:3}
]]>
</style>
<defs id="Markers">
<g id="lend1">
<path d="M 1 -1 L 0 0 L 1 1 " style="stroke-linecap:round;stroke-linejoin:round;fill:none"/>
</g>
<marker id="mrkr1-32" class="st12" v:arrowType="1" v:arrowSize="2" orient="auto" markerUnits="strokeWidth"
overflow="visible">
<use xlink:href="#lend1" transform="scale(-3.52,-3.52) "/>
</marker>
</defs>
<defs id="Filters">
<filter id="filter_2">
<feGaussianBlur stdDeviation="2"/>
</filter>
</defs>
<g v:mID="0" v:index="1" v:groupContext="foregroundPage">
<v:userDefs>
<v:ud v:nameU="msvThemeOrder" v:val="VT0(0):26"/>
</v:userDefs>
<title>页-1</title>
<v:pageProperties v:drawingScale="0.0393701" v:pageScale="0.0393701" v:drawingUnits="24" v:shadowOffsetX="8.50394"
v:shadowOffsetY="-8.50394"/>
<g id="shape17-1" v:mID="17" v:groupContext="shape" transform="translate(18.7496,-157.931)">
<title>圆角的矩形.17</title>
<v:userDefs>
<v:ud v:nameU="CTypeTopLeftSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CTypeTopRightSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CTypeBotLeftSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CTypeBotRightSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CornerLockHoriz" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CornerLockVert" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CornerLockDiag" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="TopLeftOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="TopRightOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="BotLeftOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="BotRightOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="visVersion" v:prompt="" v:val="VT0(15):26"/>
<v:ud v:nameU="TopLeftOffset" v:prompt="" v:val="VT0(0.34251968503937):1"/>
<v:ud v:nameU="TopRightOffset" v:prompt="" v:val="VT0(0.34251968503937):1"/>
<v:ud v:nameU="BotLeftOffset" v:prompt="" v:val="VT0(0.34251968503937):1"/>
<v:ud v:nameU="BotRightOffset" v:prompt="" v:val="VT0(0.34251968503937):1"/>
</v:userDefs>
<path d="M24.66 333.28 L221.95 333.28 A24.661 24.661 -180 0 0 246.61 308.62 L246.61 206.51 A24.661 24.661 -180 0 0 221.95
181.85 L24.66 181.85 A24.661 24.661 -180 0 0 -0 206.51 L0 308.62 A24.661 24.661 -180 0 0 24.66 333.28 Z"
class="st1"/>
</g>
<g id="shape85-3" v:mID="85" v:groupContext="shape" transform="translate(33.1466,-173.969)">
<title>圆角的矩形</title>
<v:userDefs>
<v:ud v:nameU="CTypeTopLeftSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CTypeTopRightSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CTypeBotLeftSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CTypeBotRightSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CornerLockHoriz" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CornerLockVert" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CornerLockDiag" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="TopLeftOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="TopRightOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="BotLeftOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="BotRightOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="visVersion" v:prompt="" v:val="VT0(15):26"/>
</v:userDefs>
<g id="shadow85-4" v:groupContext="shadow" v:shadowOffsetX="0.345598" v:shadowOffsetY="-1.97279" v:shadowType="1"
transform="matrix(1,0,0,1,0.345598,1.97279)" class="st2">
<path d="M11.34 333.28 L102.05 333.28 A11.3384 11.3384 -180 0 0 113.39 321.94 L113.39 287.93 A11.3384 11.3384 -180
0 0 102.05 276.59 L11.34 276.59 A11.3384 11.3384 -180 0 0 0 287.93 L0 321.94 A11.3384 11.3384 -180 0
0 11.34 333.28 Z" class="st3"/>
</g>
<path d="M11.34 333.28 L102.05 333.28 A11.3384 11.3384 -180 0 0 113.39 321.94 L113.39 287.93 A11.3384 11.3384 -180 0
0 102.05 276.59 L11.34 276.59 A11.3384 11.3384 -180 0 0 0 287.93 L0 321.94 A11.3384 11.3384 -180 0 0 11.34
333.28 Z" class="st4"/>
</g>
<g id="shape5-8" v:mID="5" v:groupContext="shape" transform="translate(89.6157,-242.747)">
<title>圆角的矩形.5</title>
<v:userDefs>
<v:ud v:nameU="CTypeTopLeftSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CTypeTopRightSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CTypeBotLeftSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CTypeBotRightSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CornerLockHoriz" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CornerLockVert" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CornerLockDiag" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="TopLeftOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="TopRightOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="BotLeftOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="BotRightOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="visVersion" v:prompt="" v:val="VT0(15):26"/>
<v:ud v:nameU="TopLeftOffset" v:prompt="" v:val="VT0(0.11811023622047):1"/>
<v:ud v:nameU="TopRightOffset" v:prompt="" v:val="VT0(0.11811023622047):1"/>
<v:ud v:nameU="BotLeftOffset" v:prompt="" v:val="VT0(0.11811023622047):1"/>
<v:ud v:nameU="BotRightOffset" v:prompt="" v:val="VT0(0.11811023622047):1"/>
</v:userDefs>
<path d="M8.5 333.28 L76.54 333.28 A8.5038 8.5038 -180 0 0 85.04 324.78 L85.04 295.01 A8.5038 8.5038 -180 0 0 76.54 286.51
L8.5 286.51 A8.5038 8.5038 -180 0 0 -0 295.01 L0 324.78 A8.5038 8.5038 -180 0 0 8.5 333.28 Z" class="st5"/>
</g>
<g id="shape7-10" v:mID="7" v:groupContext="shape" transform="translate(140.639,-281.014)">
<title>工作表.7</title>
<desc>CCE-Cluster1</desc>
<v:textBlock v:margins="rect(4,4,4,4)" v:tabSpace="42.5197"/>
<v:textRect cx="62.3622" cy="316.272" width="124.73" height="34.0157"/>
<rect x="0" y="299.264" width="124.724" height="34.0157" class="st6"/>
<text x="20.54" y="321.07" class="st7" v:langID="1033"><v:paragraph v:horizAlign="1"/><v:tabList/>CCE-Cluster1</text> </g>
<g id="shape8-13" v:mID="8" v:groupContext="shape" transform="translate(100.954,-255.503)">
<title>工作表.8</title>
<desc>POD</desc>
<v:textBlock v:margins="rect(4,4,4,4)" v:tabSpace="42.5197"/>
<v:textRect cx="28.3465" cy="316.272" width="56.7" height="34.0157"/>
<rect x="0" y="299.264" width="56.6929" height="34.0157" class="st6"/>
<text x="17.58" y="319.87" class="st8" v:langID="1033"><v:paragraph v:horizAlign="1"/><v:tabList/>POD</text> </g>
<g id="shape9-16" v:mID="9" v:groupContext="shape" transform="translate(75.4425,-238.495)">
<title>工作表.9</title>
<desc>nodeName: VK</desc>
<v:textBlock v:margins="rect(4,4,4,4)" v:tabSpace="42.5197"/>
<v:textRect cx="55.2756" cy="316.272" width="110.56" height="34.0157"/>
<rect x="0" y="299.264" width="110.551" height="34.0157" class="st6"/>
<text x="18.81" y="319.87" class="st8" v:langID="1033"><v:paragraph v:horizAlign="1"/><v:tabList/>nodeName: VK</text> </g>
<g id="shape11-19" v:mID="11" v:groupContext="shape" transform="translate(159.065,-184.636)">
<title>圆角的矩形.11</title>
<v:userDefs>
<v:ud v:nameU="CTypeTopLeftSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CTypeTopRightSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CTypeBotLeftSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CTypeBotRightSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CornerLockHoriz" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CornerLockVert" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CornerLockDiag" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="TopLeftOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="TopRightOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="BotLeftOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="BotRightOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="visVersion" v:prompt="" v:val="VT0(15):26"/>
<v:ud v:nameU="TopLeftOffset" v:prompt="" v:val="VT0(0.13779527559055):1"/>
<v:ud v:nameU="TopRightOffset" v:prompt="" v:val="VT0(0.13779527559055):1"/>
<v:ud v:nameU="BotLeftOffset" v:prompt="" v:val="VT0(0.13779527559055):1"/>
<v:ud v:nameU="BotRightOffset" v:prompt="" v:val="VT0(0.13779527559055):1"/>
</v:userDefs>
<g id="shadow11-20" v:groupContext="shadow" v:shadowOffsetX="0.345598" v:shadowOffsetY="-1.97279" v:shadowType="1"
transform="matrix(1,0,0,1,0.345598,1.97279)" class="st2">
<path d="M9.92 333.28 L89.29 333.28 A9.9211 9.9211 -180 0 0 99.21 323.36 L99.21 303.52 A9.9211 9.9211 -180 0 0 89.29
293.6 L9.92 293.6 A9.9211 9.9211 -180 0 0 0 303.52 L0 323.36 A9.9211 9.9211 -180 0 0 9.92 333.28 Z"
class="st3"/>
</g>
<path d="M9.92 333.28 L89.29 333.28 A9.9211 9.9211 -180 0 0 99.21 323.36 L99.21 303.52 A9.9211 9.9211 -180 0 0 89.29
293.6 L9.92 293.6 A9.9211 9.9211 -180 0 0 0 303.52 L0 323.36 A9.9211 9.9211 -180 0 0 9.92 333.28 Z"
class="st9"/>
</g>
<g id="shape12-24" v:mID="12" v:groupContext="shape" transform="translate(167.568,-187.471)">
<title>工作表.12</title>
<desc>Virtual-Kubelet</desc>
<v:textBlock v:margins="rect(4,4,4,4)" v:tabSpace="42.5197"/>
<v:textRect cx="42.5197" cy="316.272" width="85.04" height="34.0157"/>
<rect x="0" y="299.264" width="85.0394" height="34.0157" class="st6"/>
<text x="4.48" y="319.87" class="st10" v:langID="1033"><v:paragraph v:horizAlign="1"/><v:tabList/>Virtual-Kubelet</text> </g>
<g id="shape14-27" v:mID="14" v:groupContext="shape" transform="translate(433.184,-143.181) rotate(50.8696)">
<title>工作表.14</title>
<path d="M0 333.28 L53.9 333.28" class="st11"/>
</g>
<g id="shape22-33" v:mID="22" v:groupContext="shape" transform="translate(159.065,-184.636)">
<title>圆角的矩形.22</title>
<v:userDefs>
<v:ud v:nameU="CTypeTopLeftSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CTypeTopRightSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CTypeBotLeftSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CTypeBotRightSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CornerLockHoriz" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CornerLockVert" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CornerLockDiag" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="TopLeftOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="TopRightOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="BotLeftOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="BotRightOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="visVersion" v:prompt="" v:val="VT0(15):26"/>
<v:ud v:nameU="TopLeftOffset" v:prompt="" v:val="VT0(0.13779527559055):1"/>
<v:ud v:nameU="TopRightOffset" v:prompt="" v:val="VT0(0.13779527559055):1"/>
<v:ud v:nameU="BotLeftOffset" v:prompt="" v:val="VT0(0.13779527559055):1"/>
<v:ud v:nameU="BotRightOffset" v:prompt="" v:val="VT0(0.13779527559055):1"/>
</v:userDefs>
<g id="shadow22-34" v:groupContext="shadow" v:shadowOffsetX="0.345598" v:shadowOffsetY="-1.97279" v:shadowType="1"
transform="matrix(1,0,0,1,0.345598,1.97279)" class="st2">
<path d="M9.92 333.28 L89.29 333.28 A9.9211 9.9211 -180 0 0 99.21 323.36 L99.21 303.52 A9.9211 9.9211 -180 0 0 89.29
293.6 L9.92 293.6 A9.9211 9.9211 -180 0 0 0 303.52 L0 323.36 A9.9211 9.9211 -180 0 0 9.92 333.28 Z"
class="st3"/>
</g>
<path d="M9.92 333.28 L89.29 333.28 A9.9211 9.9211 -180 0 0 99.21 323.36 L99.21 303.52 A9.9211 9.9211 -180 0 0 89.29
293.6 L9.92 293.6 A9.9211 9.9211 -180 0 0 0 303.52 L0 323.36 A9.9211 9.9211 -180 0 0 9.92 333.28 Z"
class="st9"/>
</g>
<g id="shape23-38" v:mID="23" v:groupContext="shape" transform="translate(167.568,-187.471)">
<title>工作表.23</title>
<desc>Virtual-Kubelet</desc>
<v:textBlock v:margins="rect(4,4,4,4)" v:tabSpace="42.5197"/>
<v:textRect cx="42.5197" cy="316.272" width="85.04" height="34.0157"/>
<rect x="0" y="299.264" width="85.0394" height="34.0157" class="st6"/>
<text x="4.48" y="319.87" class="st10" v:langID="1033"><v:paragraph v:horizAlign="1"/><v:tabList/>Virtual-Kubelet</text> </g>
<g id="shape37-41" v:mID="37" v:groupContext="shape" transform="translate(93.9143,-192.618)">
<title>圆角的矩形.37</title>
<v:userDefs>
<v:ud v:nameU="CTypeTopLeftSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CTypeTopRightSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CTypeBotLeftSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CTypeBotRightSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CornerLockHoriz" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CornerLockVert" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CornerLockDiag" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="TopLeftOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="TopRightOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="BotLeftOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="BotRightOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="visVersion" v:prompt="" v:val="VT0(15):26"/>
<v:ud v:nameU="TopLeftOffset" v:prompt="" v:val="VT0(0.061023622047244):1"/>
<v:ud v:nameU="TopRightOffset" v:prompt="" v:val="VT0(0.061023622047244):1"/>
<v:ud v:nameU="BotLeftOffset" v:prompt="" v:val="VT0(0.061023622047244):1"/>
<v:ud v:nameU="BotRightOffset" v:prompt="" v:val="VT0(0.061023622047244):1"/>
</v:userDefs>
<g id="shadow37-42" v:groupContext="shadow" v:shadowOffsetX="0.345598" v:shadowOffsetY="-1.97279" v:shadowType="1"
transform="matrix(1,0,0,1,0.345598,1.97279)" class="st2">
<path d="M4.39 333.28 L39.54 333.28 A4.39363 4.39363 -180 0 0 43.94 328.89 L43.94 309.33 A4.39363 4.39363 -180 0
0 39.54 304.93 L4.39 304.93 A4.39363 4.39363 -180 0 0 0 309.33 L0 328.89 A4.39363 4.39363 -180 0 0 4.39
333.28 Z" class="st3"/>
</g>
<path d="M4.39 333.28 L39.54 333.28 A4.39363 4.39363 -180 0 0 43.94 328.89 L43.94 309.33 A4.39363 4.39363 -180 0 0 39.54
304.93 L4.39 304.93 A4.39363 4.39363 -180 0 0 0 309.33 L0 328.89 A4.39363 4.39363 -180 0 0 4.39 333.28 Z"
class="st13"/>
</g>
<g id="shape38-46" v:mID="38" v:groupContext="shape" transform="translate(98.1663,-194.602)">
<title>工作表.38</title>
<desc>POD</desc>
<v:textBlock v:margins="rect(4,4,4,4)" v:tabSpace="42.5197"/>
<v:textRect cx="18.4252" cy="320.808" width="36.86" height="24.9449"/>
<rect x="0" y="308.335" width="36.8504" height="24.9449" class="st6"/>
<text x="7.66" y="324.41" class="st8" v:langID="1033"><v:paragraph v:horizAlign="1"/><v:tabList/>POD</text> </g>
<g id="shape39-49" v:mID="39" v:groupContext="shape" transform="translate(42.5364,-192.618)">
<title>圆角的矩形.39</title>
<v:userDefs>
<v:ud v:nameU="CTypeTopLeftSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CTypeTopRightSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CTypeBotLeftSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CTypeBotRightSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CornerLockHoriz" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CornerLockVert" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CornerLockDiag" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="TopLeftOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="TopRightOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="BotLeftOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="BotRightOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="visVersion" v:prompt="" v:val="VT0(15):26"/>
<v:ud v:nameU="TopLeftOffset" v:prompt="" v:val="VT0(0.061023622047244):1"/>
<v:ud v:nameU="TopRightOffset" v:prompt="" v:val="VT0(0.061023622047244):1"/>
<v:ud v:nameU="BotLeftOffset" v:prompt="" v:val="VT0(0.061023622047244):1"/>
<v:ud v:nameU="BotRightOffset" v:prompt="" v:val="VT0(0.061023622047244):1"/>
</v:userDefs>
<g id="shadow39-50" v:groupContext="shadow" v:shadowOffsetX="0.345598" v:shadowOffsetY="-1.97279" v:shadowType="1"
transform="matrix(1,0,0,1,0.345598,1.97279)" class="st2">
<path d="M4.39 333.28 L39.54 333.28 A4.39363 4.39363 -180 0 0 43.94 328.89 L43.94 309.33 A4.39363 4.39363 -180 0
0 39.54 304.93 L4.39 304.93 A4.39363 4.39363 -180 0 0 0 309.33 L0 328.89 A4.39363 4.39363 -180 0 0 4.39
333.28 Z" class="st3"/>
</g>
<path d="M4.39 333.28 L39.54 333.28 A4.39363 4.39363 -180 0 0 43.94 328.89 L43.94 309.33 A4.39363 4.39363 -180 0 0 39.54
304.93 L4.39 304.93 A4.39363 4.39363 -180 0 0 0 309.33 L0 328.89 A4.39363 4.39363 -180 0 0 4.39 333.28 Z"
class="st13"/>
</g>
<g id="shape40-54" v:mID="40" v:groupContext="shape" transform="translate(44.6624,-193.185)">
<title>工作表.40</title>
<desc>POD</desc>
<v:textBlock v:margins="rect(4,4,4,4)" v:tabSpace="42.5197"/>
<v:textRect cx="18.4252" cy="320.808" width="36.86" height="24.9449"/>
<rect x="0" y="308.335" width="36.8504" height="24.9449" class="st6"/>
<text x="7.66" y="324.41" class="st8" v:langID="1033"><v:paragraph v:horizAlign="1"/><v:tabList/>POD</text> </g>
<g id="shape42-57" v:mID="42" v:groupContext="shape" transform="translate(363.159,-240.509)">
<title>圆角的矩形.42</title>
<v:userDefs>
<v:ud v:nameU="CTypeTopLeftSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CTypeTopRightSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CTypeBotLeftSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CTypeBotRightSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CornerLockHoriz" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CornerLockVert" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CornerLockDiag" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="TopLeftOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="TopRightOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="BotLeftOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="BotRightOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="visVersion" v:prompt="" v:val="VT0(15):26"/>
<v:ud v:nameU="TopLeftOffset" v:prompt="" v:val="VT0(0.11811023622047):1"/>
<v:ud v:nameU="TopRightOffset" v:prompt="" v:val="VT0(0.11811023622047):1"/>
<v:ud v:nameU="BotLeftOffset" v:prompt="" v:val="VT0(0.11811023622047):1"/>
<v:ud v:nameU="BotRightOffset" v:prompt="" v:val="VT0(0.11811023622047):1"/>
</v:userDefs>
<path d="M8.5 333.28 L76.54 333.28 A8.5038 8.5038 -180 0 0 85.04 324.78 L85.04 295.01 A8.5038 8.5038 -180 0 0 76.54 286.51
L8.5 286.51 A8.5038 8.5038 -180 0 0 -0 295.01 L0 324.78 A8.5038 8.5038 -180 0 0 8.5 333.28 Z" class="st5"/>
</g>
<g id="shape43-59" v:mID="43" v:groupContext="shape" transform="translate(312.135,-279.131)">
<title>工作表.43</title>
<desc>CCE-Cluster1</desc>
<v:textBlock v:margins="rect(4,4,4,4)" v:tabSpace="42.5197"/>
<v:textRect cx="62.3622" cy="316.272" width="124.73" height="34.0157"/>
<rect x="0" y="299.264" width="124.724" height="34.0157" class="st6"/>
<text x="20.54" y="321.07" class="st7" v:langID="1033"><v:paragraph v:horizAlign="1"/><v:tabList/>CCE-Cluster1</text> </g>
<g id="shape44-62" v:mID="44" v:groupContext="shape" transform="translate(374.498,-253.265)">
<title>工作表.44</title>
<desc>POD</desc>
<v:textBlock v:margins="rect(4,4,4,4)" v:tabSpace="42.5197"/>
<v:textRect cx="28.3465" cy="316.272" width="56.7" height="34.0157"/>
<rect x="0" y="299.264" width="56.6929" height="34.0157" class="st6"/>
<text x="17.58" y="319.87" class="st8" v:langID="1033"><v:paragraph v:horizAlign="1"/><v:tabList/>POD</text> </g>
<g id="shape45-65" v:mID="45" v:groupContext="shape" transform="translate(348.986,-236.257)">
<title>工作表.45</title>
<desc>nodeName: VK</desc>
<v:textBlock v:margins="rect(4,4,4,4)" v:tabSpace="42.5197"/>
<v:textRect cx="55.2756" cy="316.272" width="110.56" height="34.0157"/>
<rect x="0" y="299.264" width="110.551" height="34.0157" class="st6"/>
<text x="18.81" y="319.87" class="st8" v:langID="1033"><v:paragraph v:horizAlign="1"/><v:tabList/>nodeName: VK</text> </g>
<g id="shape46-68" v:mID="46" v:groupContext="shape" transform="translate(432.608,-182.399)">
<title>圆角的矩形.46</title>
<v:userDefs>
<v:ud v:nameU="CTypeTopLeftSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CTypeTopRightSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CTypeBotLeftSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CTypeBotRightSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CornerLockHoriz" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CornerLockVert" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CornerLockDiag" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="TopLeftOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="TopRightOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="BotLeftOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="BotRightOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="visVersion" v:prompt="" v:val="VT0(15):26"/>
<v:ud v:nameU="TopLeftOffset" v:prompt="" v:val="VT0(0.13779527559055):1"/>
<v:ud v:nameU="TopRightOffset" v:prompt="" v:val="VT0(0.13779527559055):1"/>
<v:ud v:nameU="BotLeftOffset" v:prompt="" v:val="VT0(0.13779527559055):1"/>
<v:ud v:nameU="BotRightOffset" v:prompt="" v:val="VT0(0.13779527559055):1"/>
</v:userDefs>
<g id="shadow46-69" v:groupContext="shadow" v:shadowOffsetX="0.345598" v:shadowOffsetY="-1.97279" v:shadowType="1"
transform="matrix(1,0,0,1,0.345598,1.97279)" class="st2">
<path d="M9.92 333.28 L89.29 333.28 A9.9211 9.9211 -180 0 0 99.21 323.36 L99.21 303.52 A9.9211 9.9211 -180 0 0 89.29
293.6 L9.92 293.6 A9.9211 9.9211 -180 0 0 0 303.52 L0 323.36 A9.9211 9.9211 -180 0 0 9.92 333.28 Z"
class="st3"/>
</g>
<path d="M9.92 333.28 L89.29 333.28 A9.9211 9.9211 -180 0 0 99.21 323.36 L99.21 303.52 A9.9211 9.9211 -180 0 0 89.29
293.6 L9.92 293.6 A9.9211 9.9211 -180 0 0 0 303.52 L0 323.36 A9.9211 9.9211 -180 0 0 9.92 333.28 Z"
class="st9"/>
</g>
<g id="shape48-73" v:mID="48" v:groupContext="shape" transform="translate(706.728,-140.943) rotate(50.8696)">
<title>工作表.48</title>
<path d="M0 333.28 L53.9 333.28" class="st11"/>
</g>
<g id="shape50-78" v:mID="50" v:groupContext="shape" transform="translate(312.135,-155.693)">
<title>圆角的矩形.50</title>
<v:userDefs>
<v:ud v:nameU="CTypeTopLeftSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CTypeTopRightSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CTypeBotLeftSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CTypeBotRightSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CornerLockHoriz" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CornerLockVert" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CornerLockDiag" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="TopLeftOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="TopRightOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="BotLeftOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="BotRightOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="visVersion" v:prompt="" v:val="VT0(15):26"/>
<v:ud v:nameU="TopLeftOffset" v:prompt="" v:val="VT0(0.34251968503937):1"/>
<v:ud v:nameU="TopRightOffset" v:prompt="" v:val="VT0(0.34251968503937):1"/>
<v:ud v:nameU="BotLeftOffset" v:prompt="" v:val="VT0(0.34251968503937):1"/>
<v:ud v:nameU="BotRightOffset" v:prompt="" v:val="VT0(0.34251968503937):1"/>
</v:userDefs>
<path d="M24.66 333.28 L221.95 333.28 A24.661 24.661 -180 0 0 246.61 308.62 L246.61 206.51 A24.661 24.661 -180 0 0 221.95
181.85 L24.66 181.85 A24.661 24.661 -180 0 0 -0 206.51 L0 308.62 A24.661 24.661 -180 0 0 24.66 333.28 Z"
class="st1"/>
</g>
<g id="shape51-80" v:mID="51" v:groupContext="shape" transform="translate(400.009,-240.136)">
<title>圆角的矩形.51</title>
<v:userDefs>
<v:ud v:nameU="CTypeTopLeftSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CTypeTopRightSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CTypeBotLeftSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CTypeBotRightSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CornerLockHoriz" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CornerLockVert" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CornerLockDiag" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="TopLeftOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="TopRightOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="BotLeftOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="BotRightOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="visVersion" v:prompt="" v:val="VT0(15):26"/>
<v:ud v:nameU="TopLeftOffset" v:prompt="" v:val="VT0(0.11811023622047):1"/>
<v:ud v:nameU="TopRightOffset" v:prompt="" v:val="VT0(0.11811023622047):1"/>
<v:ud v:nameU="BotLeftOffset" v:prompt="" v:val="VT0(0.11811023622047):1"/>
<v:ud v:nameU="BotRightOffset" v:prompt="" v:val="VT0(0.11811023622047):1"/>
</v:userDefs>
<path d="M8.5 333.28 L76.54 333.28 A8.5038 8.5038 -180 0 0 85.04 324.78 L85.04 295.01 A8.5038 8.5038 -180 0 0 76.54 286.51
L8.5 286.51 A8.5038 8.5038 -180 0 0 -0 295.01 L0 324.78 A8.5038 8.5038 -180 0 0 8.5 333.28 Z" class="st5"/>
</g>
<g id="shape52-82" v:mID="52" v:groupContext="shape" transform="translate(314.97,-278.404)">
<title>工作表.52</title>
<desc>CCE-ClusterN</desc>
<v:textBlock v:margins="rect(4,4,4,4)" v:tabSpace="42.5197"/>
<v:textRect cx="62.3622" cy="316.272" width="124.73" height="34.0157"/>
<rect x="0" y="299.264" width="124.724" height="34.0157" class="st6"/>
<text x="19.43" y="321.07" class="st7" v:langID="1033"><v:paragraph v:horizAlign="1"/><v:tabList/>CCE-ClusterN</text> </g>
<g id="shape53-85" v:mID="53" v:groupContext="shape" transform="translate(411.348,-252.892)">
<title>工作表.53</title>
<desc>POD</desc>
<v:textBlock v:margins="rect(4,4,4,4)" v:tabSpace="42.5197"/>
<v:textRect cx="28.3465" cy="316.272" width="56.7" height="34.0157"/>
<rect x="0" y="299.264" width="56.6929" height="34.0157" class="st6"/>
<text x="17.58" y="319.87" class="st8" v:langID="1033"><v:paragraph v:horizAlign="1"/><v:tabList/>POD</text> </g>
<g id="shape54-88" v:mID="54" v:groupContext="shape" transform="translate(385.836,-235.884)">
<title>工作表.54</title>
<desc>nodeName: VK</desc>
<v:textBlock v:margins="rect(4,4,4,4)" v:tabSpace="42.5197"/>
<v:textRect cx="55.2756" cy="316.272" width="110.56" height="34.0157"/>
<rect x="0" y="299.264" width="110.551" height="34.0157" class="st6"/>
<text x="18.81" y="319.87" class="st8" v:langID="1033"><v:paragraph v:horizAlign="1"/><v:tabList/>nodeName: VK</text> </g>
<g id="shape55-91" v:mID="55" v:groupContext="shape" transform="translate(318.536,-181.69)">
<title>圆角的矩形.55</title>
<v:userDefs>
<v:ud v:nameU="CTypeTopLeftSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CTypeTopRightSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CTypeBotLeftSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CTypeBotRightSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CornerLockHoriz" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CornerLockVert" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CornerLockDiag" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="TopLeftOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="TopRightOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="BotLeftOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="BotRightOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="visVersion" v:prompt="" v:val="VT0(15):26"/>
<v:ud v:nameU="TopLeftOffset" v:prompt="" v:val="VT0(0.13779527559055):1"/>
<v:ud v:nameU="TopRightOffset" v:prompt="" v:val="VT0(0.13779527559055):1"/>
<v:ud v:nameU="BotLeftOffset" v:prompt="" v:val="VT0(0.13779527559055):1"/>
<v:ud v:nameU="BotRightOffset" v:prompt="" v:val="VT0(0.13779527559055):1"/>
</v:userDefs>
<g id="shadow55-92" v:groupContext="shadow" v:shadowOffsetX="0.345598" v:shadowOffsetY="-1.97279" v:shadowType="1"
transform="matrix(1,0,0,1,0.345598,1.97279)" class="st2">
<path d="M9.92 333.28 L89.29 333.28 A9.9211 9.9211 -180 0 0 99.21 323.36 L99.21 303.52 A9.9211 9.9211 -180 0 0 89.29
293.6 L9.92 293.6 A9.9211 9.9211 -180 0 0 0 303.52 L0 323.36 A9.9211 9.9211 -180 0 0 9.92 333.28 Z"
class="st3"/>
</g>
<path d="M9.92 333.28 L89.29 333.28 A9.9211 9.9211 -180 0 0 99.21 323.36 L99.21 303.52 A9.9211 9.9211 -180 0 0 89.29
293.6 L9.92 293.6 A9.9211 9.9211 -180 0 0 0 303.52 L0 323.36 A9.9211 9.9211 -180 0 0 9.92 333.28 Z"
class="st9"/>
</g>
<g id="shape56-96" v:mID="56" v:groupContext="shape" transform="translate(329.28,-185.233)">
<title>工作表.56</title>
<desc>Virtual-Kubelet</desc>
<v:textBlock v:margins="rect(4,4,4,4)" v:tabSpace="42.5197"/>
<v:textRect cx="42.5197" cy="316.272" width="85.04" height="34.0157"/>
<rect x="0" y="299.264" width="85.0394" height="34.0157" class="st6"/>
<text x="4.48" y="319.87" class="st10" v:langID="1033"><v:paragraph v:horizAlign="1"/><v:tabList/>Virtual-Kubelet</text> </g>
<g id="shape65-99" v:mID="65" v:groupContext="shape" transform="translate(164.734,-18.75)">
<title>圆角的矩形.65</title>
<v:userDefs>
<v:ud v:nameU="CTypeTopLeftSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CTypeTopRightSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CTypeBotLeftSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CTypeBotRightSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CornerLockHoriz" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CornerLockVert" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CornerLockDiag" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="TopLeftOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="TopRightOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="BotLeftOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="BotRightOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="visVersion" v:prompt="" v:val="VT0(15):26"/>
<v:ud v:nameU="TopLeftOffset" v:prompt="" v:val="VT0(0.34251968503937):1"/>
<v:ud v:nameU="TopRightOffset" v:prompt="" v:val="VT0(0.34251968503937):1"/>
<v:ud v:nameU="BotLeftOffset" v:prompt="" v:val="VT0(0.34251968503937):1"/>
<v:ud v:nameU="BotRightOffset" v:prompt="" v:val="VT0(0.34251968503937):1"/>
</v:userDefs>
<path d="M24.66 333.28 L221.95 333.28 A24.661 24.661 -180 0 0 246.61 308.62 L246.61 235.48 A24.661 24.661 -180 0 0 221.95
210.82 L24.66 210.82 A24.661 24.661 -180 0 0 -0 235.48 L0 308.62 A24.661 24.661 -180 0 0 24.66 333.28 Z"
class="st1"/>
</g>
<g id="shape66-101" v:mID="66" v:groupContext="shape" transform="translate(275.285,-220.17)">
<title>工作表.66</title>
<desc>. . .</desc>
<v:textBlock v:margins="rect(4,4,4,4)" v:tabSpace="42.5197"/>
<v:textRect cx="18.4252" cy="320.808" width="36.86" height="24.9449"/>
<rect x="0" y="308.335" width="36.8504" height="24.9449" class="st6"/>
<text x="9.65" y="325.01" class="st14" v:langID="1033"><v:paragraph v:horizAlign="1"/><v:tabList/>. . .</text> </g>
<g id="shape67-104" v:mID="67" v:groupContext="shape" transform="translate(269.616,-113.386)">
<title>工作表.67</title>
<desc>CCI</desc>
<v:textBlock v:margins="rect(4,4,4,4)" v:tabSpace="42.5197"/>
<v:textRect cx="18.4252" cy="320.808" width="36.86" height="24.9449"/>
<rect x="0" y="308.335" width="36.8504" height="24.9449" class="st6"/>
<text x="7.82" y="325.61" class="st15" v:langID="1033"><v:paragraph v:horizAlign="1"/><v:tabList/>CCI</text> </g>
<g id="shape68-107" v:mID="68" v:groupContext="shape" transform="translate(665.854,270.762) rotate(127.093)">
<title>工作表.68</title>
<path d="M0 333.28 L52.84 333.28" class="st11"/>
</g>
<g id="shape69-112" v:mID="69" v:groupContext="shape" transform="translate(181.417,-34.9075)">
<title>圆角的矩形.69</title>
<v:userDefs>
<v:ud v:nameU="CTypeTopLeftSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CTypeTopRightSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CTypeBotLeftSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CTypeBotRightSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CornerLockHoriz" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CornerLockVert" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CornerLockDiag" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="TopLeftOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="TopRightOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="BotLeftOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="BotRightOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="visVersion" v:prompt="" v:val="VT0(15):26"/>
<v:ud v:nameU="TopLeftOffset" v:prompt="" v:val="VT0(0.11417322834646):1"/>
<v:ud v:nameU="TopRightOffset" v:prompt="" v:val="VT0(0.11417322834646):1"/>
<v:ud v:nameU="BotLeftOffset" v:prompt="" v:val="VT0(0.11417322834646):1"/>
<v:ud v:nameU="BotRightOffset" v:prompt="" v:val="VT0(0.11417322834646):1"/>
</v:userDefs>
<path d="M8.22 333.28 L73.98 333.28 A8.22034 8.22034 -180 0 0 82.2 325.06 L82.2 263.55 A8.22034 8.22034 -180 0 0 73.98
255.33 L8.22 255.33 A8.22034 8.22034 -180 0 0 0 263.55 L0 325.06 A8.22034 8.22034 -180 0 0 8.22 333.28 Z"
class="st16"/>
</g>
<g id="shape70-114" v:mID="70" v:groupContext="shape" transform="translate(185.315,-34.9075)">
<title>工作表.70</title>
<desc>Project-VK1</desc>
<v:textBlock v:margins="rect(4,4,4,4)" v:tabSpace="42.5197"/>
<v:textRect cx="36.4961" cy="319.107" width="73" height="28.3465"/>
<rect x="0" y="304.934" width="72.9921" height="28.3465" class="st6"/>
<text x="7.77" y="322.71" class="st8" v:langID="1033"><v:paragraph v:horizAlign="1"/><v:tabList/>Project-VK1</text> </g>
<g id="shape71-117" v:mID="71" v:groupContext="shape" transform="translate(197.008,-60.4193)">
<title>圆角的矩形.71</title>
<v:userDefs>
<v:ud v:nameU="CTypeTopLeftSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CTypeTopRightSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CTypeBotLeftSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CTypeBotRightSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CornerLockHoriz" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CornerLockVert" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CornerLockDiag" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="TopLeftOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="TopRightOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="BotLeftOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="BotRightOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="visVersion" v:prompt="" v:val="VT0(15):26"/>
<v:ud v:nameU="TopLeftOffset" v:prompt="" v:val="VT0(0.07007239014478):1"/>
<v:ud v:nameU="TopRightOffset" v:prompt="" v:val="VT0(0.07007239014478):1"/>
<v:ud v:nameU="BotLeftOffset" v:prompt="" v:val="VT0(0.07007239014478):1"/>
<v:ud v:nameU="BotRightOffset" v:prompt="" v:val="VT0(0.07007239014478):1"/>
</v:userDefs>
<path d="M5.05 333.28 L45.41 333.28 A5.04513 5.04513 -180 0 0 50.45 328.23 L50.45 291.55 A5.04513 5.04513 -180 0 0 45.41
286.51 L5.05 286.51 A5.04513 5.04513 -180 0 0 0 291.55 L0 328.23 A5.04513 5.04513 -180 0 0 5.05 333.28 Z"
class="st17"/>
</g>
<g id="shape76-119" v:mID="76" v:groupContext="shape" transform="translate(202.814,-71.3327)">
<title>工作表.76</title>
<desc>POD</desc>
<v:textBlock v:margins="rect(4,4,4,4)" v:tabSpace="42.5197"/>
<v:textRect cx="18.4252" cy="320.808" width="36.86" height="24.9449"/>
<rect x="0" y="308.335" width="36.8504" height="24.9449" class="st6"/>
<text x="7.66" y="324.41" class="st8" v:langID="1033"><v:paragraph v:horizAlign="1"/><v:tabList/>POD</text> </g>
<g id="shape77-122" v:mID="77" v:groupContext="shape" transform="translate(536.955,91.1513) rotate(80.0665)">
<title>工作表.77</title>
<path d="M0 333.28 L78.62 333.28" class="st11"/>
</g>
<g id="shape78-127" v:mID="78" v:groupContext="shape" transform="translate(311.528,-34.9075)">
<title>圆角的矩形.78</title>
<v:userDefs>
<v:ud v:nameU="CTypeTopLeftSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CTypeTopRightSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CTypeBotLeftSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CTypeBotRightSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CornerLockHoriz" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CornerLockVert" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CornerLockDiag" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="TopLeftOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="TopRightOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="BotLeftOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="BotRightOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="visVersion" v:prompt="" v:val="VT0(15):26"/>
<v:ud v:nameU="TopLeftOffset" v:prompt="" v:val="VT0(0.11417322834646):1"/>
<v:ud v:nameU="TopRightOffset" v:prompt="" v:val="VT0(0.11417322834646):1"/>
<v:ud v:nameU="BotLeftOffset" v:prompt="" v:val="VT0(0.11417322834646):1"/>
<v:ud v:nameU="BotRightOffset" v:prompt="" v:val="VT0(0.11417322834646):1"/>
</v:userDefs>
<path d="M8.22 333.28 L73.98 333.28 A8.22034 8.22034 -180 0 0 82.2 325.06 L82.2 263.55 A8.22034 8.22034 -180 0 0 73.98
255.33 L8.22 255.33 A8.22034 8.22034 -180 0 0 0 263.55 L0 325.06 A8.22034 8.22034 -180 0 0 8.22 333.28 Z"
class="st16"/>
</g>
<g id="shape79-129" v:mID="79" v:groupContext="shape" transform="translate(315.779,-34.9075)">
<title>工作表.79</title>
<desc>Project-VKN</desc>
<v:textBlock v:margins="rect(4,4,4,4)" v:tabSpace="42.5197"/>
<v:textRect cx="36.4961" cy="320.808" width="73" height="24.9449"/>
<rect x="0" y="308.335" width="72.9921" height="24.9449" class="st6"/>
<text x="6.94" y="324.41" class="st8" v:langID="1033"><v:paragraph v:horizAlign="1"/><v:tabList/>Project-VKN</text> </g>
<g id="shape80-132" v:mID="80" v:groupContext="shape" transform="translate(328.535,-60.4193)">
<title>圆角的矩形.80</title>
<v:userDefs>
<v:ud v:nameU="CTypeTopLeftSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CTypeTopRightSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CTypeBotLeftSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CTypeBotRightSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CornerLockHoriz" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CornerLockVert" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CornerLockDiag" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="TopLeftOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="TopRightOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="BotLeftOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="BotRightOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="visVersion" v:prompt="" v:val="VT0(15):26"/>
<v:ud v:nameU="TopLeftOffset" v:prompt="" v:val="VT0(0.07007239014478):1"/>
<v:ud v:nameU="TopRightOffset" v:prompt="" v:val="VT0(0.07007239014478):1"/>
<v:ud v:nameU="BotLeftOffset" v:prompt="" v:val="VT0(0.07007239014478):1"/>
<v:ud v:nameU="BotRightOffset" v:prompt="" v:val="VT0(0.07007239014478):1"/>
</v:userDefs>
<path d="M5.05 333.28 L45.41 333.28 A5.04513 5.04513 -180 0 0 50.45 328.23 L50.45 291.55 A5.04513 5.04513 -180 0 0 45.41
286.51 L5.05 286.51 A5.04513 5.04513 -180 0 0 0 291.55 L0 328.23 A5.04513 5.04513 -180 0 0 5.05 333.28 Z"
class="st17"/>
</g>
<g id="shape81-134" v:mID="81" v:groupContext="shape" transform="translate(334.342,-71.3327)">
<title>工作表.81</title>
<desc>POD</desc>
<v:textBlock v:margins="rect(4,4,4,4)" v:tabSpace="42.5197"/>
<v:textRect cx="18.4252" cy="320.808" width="36.86" height="24.9449"/>
<rect x="0" y="308.335" width="36.8504" height="24.9449" class="st6"/>
<text x="7.66" y="324.41" class="st8" v:langID="1033"><v:paragraph v:horizAlign="1"/><v:tabList/>POD</text> </g>
<g id="shape82-137" v:mID="82" v:groupContext="shape" transform="translate(695.381,214.76) rotate(100.926)">
<title>工作表.82</title>
<path d="M0 333.28 L75.87 333.28" class="st11"/>
</g>
<g id="shape83-142" v:mID="83" v:groupContext="shape" transform="translate(269.616,-68.5984)">
<title>工作表.83</title>
<desc>. . .</desc>
<v:textBlock v:margins="rect(4,4,4,4)" v:tabSpace="42.5197"/>
<v:textRect cx="18.4252" cy="320.808" width="36.86" height="24.9449"/>
<rect x="0" y="308.335" width="36.8504" height="24.9449" class="st6"/>
<text x="9.65" y="325.01" class="st14" v:langID="1033"><v:paragraph v:horizAlign="1"/><v:tabList/>. . .</text> </g>
<g id="shape86-145" v:mID="86" v:groupContext="shape" transform="translate(64.6135,-169.493)">
<title>工作表.86</title>
<desc>node</desc>
<v:textBlock v:margins="rect(4,4,4,4)" v:tabSpace="42.5197"/>
<v:textRect cx="25.2261" cy="320.808" width="50.46" height="24.9449"/>
<rect x="0" y="308.335" width="50.4521" height="24.9449" class="st6"/>
<text x="10.7" y="325.01" class="st18" v:langID="1033"><v:paragraph v:horizAlign="1"/><v:tabList/>node</text> </g>
<g id="shape87-148" v:mID="87" v:groupContext="shape" transform="translate(431.191,-170.09)">
<title>圆角的矩形.87</title>
<v:userDefs>
<v:ud v:nameU="CTypeTopLeftSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CTypeTopRightSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CTypeBotLeftSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CTypeBotRightSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CornerLockHoriz" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CornerLockVert" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CornerLockDiag" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="TopLeftOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="TopRightOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="BotLeftOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="BotRightOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="visVersion" v:prompt="" v:val="VT0(15):26"/>
</v:userDefs>
<g id="shadow87-149" v:groupContext="shadow" v:shadowOffsetX="0.345598" v:shadowOffsetY="-1.97279" v:shadowType="1"
transform="matrix(1,0,0,1,0.345598,1.97279)" class="st2">
<path d="M11.34 333.28 L102.05 333.28 A11.3384 11.3384 -180 0 0 113.39 321.94 L113.39 287.93 A11.3384 11.3384 -180
0 0 102.05 276.59 L11.34 276.59 A11.3384 11.3384 -180 0 0 0 287.93 L0 321.94 A11.3384 11.3384 -180 0
0 11.34 333.28 Z" class="st3"/>
</g>
<path d="M11.34 333.28 L102.05 333.28 A11.3384 11.3384 -180 0 0 113.39 321.94 L113.39 287.93 A11.3384 11.3384 -180 0
0 102.05 276.59 L11.34 276.59 A11.3384 11.3384 -180 0 0 0 287.93 L0 321.94 A11.3384 11.3384 -180 0 0 11.34
333.28 Z" class="st4"/>
</g>
<g id="shape88-153" v:mID="88" v:groupContext="shape" transform="translate(491.958,-188.739)">
<title>圆角的矩形.88</title>
<v:userDefs>
<v:ud v:nameU="CTypeTopLeftSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CTypeTopRightSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CTypeBotLeftSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CTypeBotRightSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CornerLockHoriz" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CornerLockVert" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CornerLockDiag" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="TopLeftOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="TopRightOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="BotLeftOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="BotRightOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="visVersion" v:prompt="" v:val="VT0(15):26"/>
<v:ud v:nameU="TopLeftOffset" v:prompt="" v:val="VT0(0.061023622047244):1"/>
<v:ud v:nameU="TopRightOffset" v:prompt="" v:val="VT0(0.061023622047244):1"/>
<v:ud v:nameU="BotLeftOffset" v:prompt="" v:val="VT0(0.061023622047244):1"/>
<v:ud v:nameU="BotRightOffset" v:prompt="" v:val="VT0(0.061023622047244):1"/>
</v:userDefs>
<g id="shadow88-154" v:groupContext="shadow" v:shadowOffsetX="0.345598" v:shadowOffsetY="-1.97279" v:shadowType="1"
transform="matrix(1,0,0,1,0.345598,1.97279)" class="st2">
<path d="M4.39 333.28 L39.54 333.28 A4.39363 4.39363 -180 0 0 43.94 328.89 L43.94 309.33 A4.39363 4.39363 -180 0
0 39.54 304.93 L4.39 304.93 A4.39363 4.39363 -180 0 0 0 309.33 L0 328.89 A4.39363 4.39363 -180 0 0 4.39
333.28 Z" class="st3"/>
</g>
<path d="M4.39 333.28 L39.54 333.28 A4.39363 4.39363 -180 0 0 43.94 328.89 L43.94 309.33 A4.39363 4.39363 -180 0 0 39.54
304.93 L4.39 304.93 A4.39363 4.39363 -180 0 0 0 309.33 L0 328.89 A4.39363 4.39363 -180 0 0 4.39 333.28 Z"
class="st13"/>
</g>
<g id="shape89-158" v:mID="89" v:groupContext="shape" transform="translate(496.21,-190.723)">
<title>工作表.89</title>
<desc>POD</desc>
<v:textBlock v:margins="rect(4,4,4,4)" v:tabSpace="42.5197"/>
<v:textRect cx="18.4252" cy="320.808" width="36.86" height="24.9449"/>
<rect x="0" y="308.335" width="36.8504" height="24.9449" class="st6"/>
<text x="7.66" y="324.41" class="st8" v:langID="1033"><v:paragraph v:horizAlign="1"/><v:tabList/>POD</text> </g>
<g id="shape90-161" v:mID="90" v:groupContext="shape" transform="translate(440.58,-188.739)">
<title>圆角的矩形.90</title>
<v:userDefs>
<v:ud v:nameU="CTypeTopLeftSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CTypeTopRightSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CTypeBotLeftSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CTypeBotRightSnip" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CornerLockHoriz" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CornerLockVert" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="CornerLockDiag" v:prompt="" v:val="VT0(0):5"/>
<v:ud v:nameU="TopLeftOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="TopRightOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="BotLeftOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="BotRightOffset" v:prompt="" v:val="VT0(0.15748031496063):24"/>
<v:ud v:nameU="visVersion" v:prompt="" v:val="VT0(15):26"/>
<v:ud v:nameU="TopLeftOffset" v:prompt="" v:val="VT0(0.061023622047244):1"/>
<v:ud v:nameU="TopRightOffset" v:prompt="" v:val="VT0(0.061023622047244):1"/>
<v:ud v:nameU="BotLeftOffset" v:prompt="" v:val="VT0(0.061023622047244):1"/>
<v:ud v:nameU="BotRightOffset" v:prompt="" v:val="VT0(0.061023622047244):1"/>
</v:userDefs>
<g id="shadow90-162" v:groupContext="shadow" v:shadowOffsetX="0.345598" v:shadowOffsetY="-1.97279" v:shadowType="1"
transform="matrix(1,0,0,1,0.345598,1.97279)" class="st2">
<path d="M4.39 333.28 L39.54 333.28 A4.39363 4.39363 -180 0 0 43.94 328.89 L43.94 309.33 A4.39363 4.39363 -180 0
0 39.54 304.93 L4.39 304.93 A4.39363 4.39363 -180 0 0 0 309.33 L0 328.89 A4.39363 4.39363 -180 0 0 4.39
333.28 Z" class="st3"/>
</g>
<path d="M4.39 333.28 L39.54 333.28 A4.39363 4.39363 -180 0 0 43.94 328.89 L43.94 309.33 A4.39363 4.39363 -180 0 0 39.54
304.93 L4.39 304.93 A4.39363 4.39363 -180 0 0 0 309.33 L0 328.89 A4.39363 4.39363 -180 0 0 4.39 333.28 Z"
class="st13"/>
</g>
<g id="shape91-166" v:mID="91" v:groupContext="shape" transform="translate(442.706,-189.306)">
<title>工作表.91</title>
<desc>POD</desc>
<v:textBlock v:margins="rect(4,4,4,4)" v:tabSpace="42.5197"/>
<v:textRect cx="18.4252" cy="320.808" width="36.86" height="24.9449"/>
<rect x="0" y="308.335" width="36.8504" height="24.9449" class="st6"/>
<text x="7.66" y="324.41" class="st8" v:langID="1033"><v:paragraph v:horizAlign="1"/><v:tabList/>POD</text> </g>
<g id="shape92-169" v:mID="92" v:groupContext="shape" transform="translate(462.657,-165.614)">
<title>工作表.92</title>
<desc>node</desc>
<v:textBlock v:margins="rect(4,4,4,4)" v:tabSpace="42.5197"/>
<v:textRect cx="25.2261" cy="320.808" width="50.46" height="24.9449"/>
<rect x="0" y="308.335" width="50.4521" height="24.9449" class="st6"/>
<text x="10.7" y="325.01" class="st18" v:langID="1033"><v:paragraph v:horizAlign="1"/><v:tabList/>node</text> </g>
</g>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 51 KiB

412
providers/huawei/cci.go Normal file
View File

@@ -0,0 +1,412 @@
package huawei
import (
"bytes"
"crypto/tls"
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"net/http"
"os"
"time"
"github.com/virtual-kubelet/virtual-kubelet/manager"
"github.com/virtual-kubelet/virtual-kubelet/providers/huawei/auth"
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
)
const (
podAnnotationNamespaceKey = "virtual-kubelet-namespace"
podAnnotationPodNameKey = "virtual-kubelet-podname"
podAnnotationClusterNameKey = "virtual-kubelet-clustername"
podAnnotationUIDkey = "virtual-kubelet-uid"
podAnnotationNodeName = "virtual-kubelet-nodename"
podAnnotationCreationTimestamp = "virtual-kubelet-creationtimestamp"
)
var defaultApiEndpoint string = "https://cciback.cn-north-1.huaweicloud.com"
// CCIProvider implements the virtual-kubelet provider interface and communicates with Huawei's CCI APIs.
type CCIProvider struct {
appKey string
appSecret string
apiEndpoint string
region string
service string
project string
internalIP string
daemonEndpointPort int32
nodeName string
operatingSystem string
client *Client
resourceManager *manager.ResourceManager
cpu string
memory string
pods string
}
// Client represents the client config for Huawei.
type Client struct {
Signer auth.Signer
HTTPClient http.Client
}
// NewCCIProvider creates a new CCI provider.
func NewCCIProvider(config string, rm *manager.ResourceManager, nodeName, operatingSystem string, internalIP string, daemonEndpointPort int32) (*CCIProvider, error) {
p := CCIProvider{}
if config != "" {
f, err := os.Open(config)
if err != nil {
return nil, err
}
defer f.Close()
if err := p.loadConfig(f); err != nil {
return nil, err
}
}
if appKey := os.Getenv("CCI_APP_KEP"); appKey != "" {
p.appKey = appKey
}
if p.appKey == "" {
return nil, errors.New("AppKey can not be empty please set CCI_APP_KEP")
}
if appSecret := os.Getenv("CCI_APP_SECRET"); appSecret != "" {
p.appSecret = appSecret
}
if p.appSecret == "" {
return nil, errors.New("AppSecret can not be empty please set CCI_APP_SECRET")
}
p.client = new(Client)
p.client.Signer = &auth.SignerHws{
AppKey: p.appKey,
AppSecret: p.appSecret,
Region: p.region,
Service: p.service,
}
tr := &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}
p.client.HTTPClient = http.Client{
Transport: tr,
}
p.resourceManager = rm
p.apiEndpoint = defaultApiEndpoint
p.nodeName = nodeName
p.operatingSystem = operatingSystem
p.internalIP = internalIP
p.daemonEndpointPort = daemonEndpointPort
if err := p.createProject(); err != nil {
return nil, err
}
return &p, nil
}
func (p *CCIProvider) createProject() error {
// Create the createProject request url
uri := p.apiEndpoint + "/api/v1/namespaces"
// build the request
project := &v1.Namespace{
TypeMeta: metav1.TypeMeta{
Kind: "Namespace",
APIVersion: "v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: p.project,
},
}
var bodyReader io.Reader
body, err := json.Marshal(project)
if err != nil {
return err
}
if body != nil {
bodyReader = bytes.NewReader(body)
}
r, err := http.NewRequest("POST", uri, bodyReader)
if err != nil {
return err
}
if err = p.signRequest(r); err != nil {
return fmt.Errorf("Sign the request failed: %v", err)
}
_, err = p.client.HTTPClient.Do(r)
return err
}
func (p *CCIProvider) signRequest(r *http.Request) error {
r.Header.Add("content-type", "application/json; charset=utf-8")
if err := p.client.Signer.Sign(r); err != nil {
return fmt.Errorf("Sign the request failed: %v", err)
}
return nil
}
func (p *CCIProvider) setPodAnnotations(pod *v1.Pod) {
metav1.SetMetaDataAnnotation(&pod.ObjectMeta, podAnnotationNamespaceKey, pod.Namespace)
metav1.SetMetaDataAnnotation(&pod.ObjectMeta, podAnnotationClusterNameKey, pod.ClusterName)
metav1.SetMetaDataAnnotation(&pod.ObjectMeta, podAnnotationPodNameKey, pod.Name)
metav1.SetMetaDataAnnotation(&pod.ObjectMeta, podAnnotationUIDkey, string(pod.UID))
metav1.SetMetaDataAnnotation(&pod.ObjectMeta, podAnnotationNodeName, pod.Spec.NodeName)
metav1.SetMetaDataAnnotation(&pod.ObjectMeta, podAnnotationCreationTimestamp, pod.CreationTimestamp.String())
pod.Namespace = p.project
pod.Name = pod.Namespace + "-" + pod.Name
pod.UID = ""
pod.Spec.NodeName = ""
pod.CreationTimestamp = metav1.Time{}
}
func (p *CCIProvider) deletePodAnnotations(pod *v1.Pod) error {
pod.Name = pod.Annotations[podAnnotationPodNameKey]
pod.Namespace = pod.Annotations[podAnnotationNamespaceKey]
pod.UID = types.UID(pod.Annotations[podAnnotationUIDkey])
pod.ClusterName = pod.Annotations[podAnnotationClusterNameKey]
pod.Spec.NodeName = pod.Annotations[podAnnotationNodeName]
if pod.Annotations[podAnnotationCreationTimestamp] != "" {
t, err := time.Parse("2006-01-02 15:04:05.999999999 -0700 MST", pod.Annotations[podAnnotationCreationTimestamp])
if err != nil {
return err
}
podCreationTimestamp := metav1.NewTime(t)
pod.CreationTimestamp = podCreationTimestamp
}
delete(pod.Annotations, podAnnotationPodNameKey)
delete(pod.Annotations, podAnnotationNamespaceKey)
delete(pod.Annotations, podAnnotationUIDkey)
delete(pod.Annotations, podAnnotationClusterNameKey)
delete(pod.Annotations, podAnnotationNodeName)
delete(pod.Annotations, podAnnotationCreationTimestamp)
pod.Annotations = nil
return nil
}
// CreatePod takes a Kubernetes Pod and deploys it within the huawei CCI provider.
func (p *CCIProvider) CreatePod(pod *v1.Pod) error {
// Create the createPod request url
p.setPodAnnotations(pod)
uri := p.apiEndpoint + "/api/v1/namespaces/" + p.project + "/pods"
// build the request
var bodyReader io.Reader
body, err := json.Marshal(pod)
if err != nil {
return err
}
if body != nil {
bodyReader = bytes.NewReader(body)
}
r, err := http.NewRequest("POST", uri, bodyReader)
if err != nil {
return err
}
if err = p.signRequest(r); err != nil {
return fmt.Errorf("Sign the request failed: %v", err)
}
_, err = p.client.HTTPClient.Do(r)
return err
}
// UpdatePod takes a Kubernetes Pod and updates it within the huawei CCI provider.
func (p *CCIProvider) UpdatePod(pod *v1.Pod) error {
return nil
}
// DeletePod takes a Kubernetes Pod and deletes it from the huawei CCI provider.
func (p *CCIProvider) DeletePod(pod *v1.Pod) error {
// Create the deletePod request url
podName := pod.Namespace + "-" + pod.Name
uri := p.apiEndpoint + "/api/v1/namespaces/" + p.project + "/pods/" + podName
// build the request
r, err := http.NewRequest("DELETE", uri, nil)
if err != nil {
return err
}
if err = p.signRequest(r); err != nil {
return fmt.Errorf("Sign the request failed: %v", err)
}
_, err = p.client.HTTPClient.Do(r)
return err
}
// GetPod retrieves a pod by name from the huawei CCI provider.
func (p *CCIProvider) GetPod(namespace, name string) (*v1.Pod, error) {
// Create the getPod request url
podName := namespace + "-" + name
uri := p.apiEndpoint + "/api/v1/namespaces/" + p.project + "/pods/" + podName
r, err := http.NewRequest("GET", uri, nil)
if err != nil {
return nil, fmt.Errorf("Create get POD request failed: %v", err)
}
if err = p.signRequest(r); err != nil {
return nil, fmt.Errorf("Sign the request failed: %v", err)
}
resp, err := p.client.HTTPClient.Do(r)
if err != nil {
return nil, err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
var pod v1.Pod
if err = json.Unmarshal(body, &pod); err != nil {
return nil, err
}
if err := p.deletePodAnnotations(&pod); err != nil {
return nil, err
}
return &pod, nil
}
// GetContainerLogs retrieves the logs of a container by name from the huawei CCI provider.
func (p *CCIProvider) GetContainerLogs(namespace, podName, containerName string, tail int) (string, error) {
return "", nil
}
// GetPodStatus retrieves the status of a pod by name from the huawei CCI provider.
func (p *CCIProvider) GetPodStatus(namespace, name string) (*v1.PodStatus, error) {
pod, err := p.GetPod(namespace, name)
if err != nil {
return nil, err
}
if pod == nil {
return nil, nil
}
return &pod.Status, nil
}
// GetPods retrieves a list of all pods running on the huawei CCI provider.
func (p *CCIProvider) GetPods() ([]*v1.Pod, error) {
// Create the getPod request url
uri := p.apiEndpoint + "/api/v1/namespaces/" + p.project + "/pods"
r, err := http.NewRequest("GET", uri, nil)
if err != nil {
return nil, fmt.Errorf("Create get POD request failed: %v", err)
}
if err = p.signRequest(r); err != nil {
return nil, fmt.Errorf("Sign the request failed: %v", err)
}
resp, err := p.client.HTTPClient.Do(r)
if err != nil {
return nil, err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
var pods []*v1.Pod
if err = json.Unmarshal(body, &pods); err != nil {
return nil, err
}
for _, pod := range pods {
if err := p.deletePodAnnotations(pod); err != nil {
return nil, err
}
}
return pods, nil
}
// Capacity returns a resource list with the capacity constraints of the huawei CCI provider.
func (p *CCIProvider) Capacity() v1.ResourceList {
return v1.ResourceList{
"cpu": resource.MustParse(p.cpu),
"memory": resource.MustParse(p.memory),
"pods": resource.MustParse(p.pods),
}
}
// NodeConditions returns a list of conditions (Ready, OutOfDisk, etc), which is
// polled periodically to update the node status within Kubernetes.
func (p *CCIProvider) NodeConditions() []v1.NodeCondition {
// TODO: Make these dynamic and augment with custom CCI specific conditions of interest
return []v1.NodeCondition{
{
Type: "Ready",
Status: v1.ConditionTrue,
LastHeartbeatTime: metav1.Now(),
LastTransitionTime: metav1.Now(),
Reason: "KubeletReady",
Message: "kubelet is ready.",
},
{
Type: "OutOfDisk",
Status: v1.ConditionFalse,
LastHeartbeatTime: metav1.Now(),
LastTransitionTime: metav1.Now(),
Reason: "KubeletHasSufficientDisk",
Message: "kubelet has sufficient disk space available",
},
{
Type: "MemoryPressure",
Status: v1.ConditionFalse,
LastHeartbeatTime: metav1.Now(),
LastTransitionTime: metav1.Now(),
Reason: "KubeletHasSufficientMemory",
Message: "kubelet has sufficient memory available",
},
{
Type: "DiskPressure",
Status: v1.ConditionFalse,
LastHeartbeatTime: metav1.Now(),
LastTransitionTime: metav1.Now(),
Reason: "KubeletHasNoDiskPressure",
Message: "kubelet has no disk pressure",
},
{
Type: "NetworkUnavailable",
Status: v1.ConditionFalse,
LastHeartbeatTime: metav1.Now(),
LastTransitionTime: metav1.Now(),
Reason: "RouteCreated",
Message: "RouteController created a route",
},
}
}
// NodeAddresses returns a list of addresses for the node status
// within Kubernetes.
func (p *CCIProvider) NodeAddresses() []v1.NodeAddress {
// TODO: Make these dynamic and augment with custom CCI specific conditions of interest
return []v1.NodeAddress{
{
Type: "InternalIP",
Address: p.internalIP,
},
}
}
// NodeDaemonEndpoints returns NodeDaemonEndpoints for the node status
// within Kubernetes.
func (p *CCIProvider) NodeDaemonEndpoints() *v1.NodeDaemonEndpoints {
return &v1.NodeDaemonEndpoints{
KubeletEndpoint: v1.DaemonEndpoint{
Port: p.daemonEndpointPort,
},
}
}
// OperatingSystem returns the operating system the huawei CCI provider is for.
func (p *CCIProvider) OperatingSystem() string {
return p.operatingSystem
}

9
providers/huawei/cci.toml Normal file
View File

@@ -0,0 +1,9 @@
# example configuration file for Huawei CCI virtual-kubelet provider.
Project = "default"
Region = "southchina"
Service = "CCI"
OperatingSystem = "Linux"
CPU = "20"
Memory = "100Gi"
Pods = "20"

151
providers/huawei/cciMock.go Normal file
View File

@@ -0,0 +1,151 @@
package huawei
import (
"bytes"
"encoding/json"
"net/http"
"net/http/httptest"
"github.com/gorilla/mux"
"k8s.io/api/core/v1"
)
// CCIMock implements a CCI service mock server.
type CCIMock struct {
server *httptest.Server
OnCreateProject func(*v1.Namespace) (int, interface{})
OnCreatePod func(*v1.Pod) (int, interface{})
OnGetPods func() (int, interface{})
OnGetPod func(string, string) (int, interface{})
}
// fakeSigner signature HWS meta
type fakeSigner struct {
AppKey string
AppSecret string
Region string
Service string
}
// Sign set Authorization header
func (s *fakeSigner) Sign(r *http.Request) error {
return nil
}
const (
cciProjectRoute = "/api/v1/namespaces"
cciPodsRoute = cciProjectRoute + "/{namespaceID}/pods"
cciPodRoute = cciPodsRoute + "/{podID}"
)
// NewCCIMock creates a CCI service mock server.
func NewCCIMock() *CCIMock {
mock := new(CCIMock)
mock.start()
return mock
}
// Start the CCI service mock service.
func (mock *CCIMock) start() {
if mock.server != nil {
return
}
router := mux.NewRouter()
router.HandleFunc(
cciProjectRoute,
func(w http.ResponseWriter, r *http.Request) {
var ns v1.Namespace
if err := json.NewDecoder(r.Body).Decode(&ns); err != nil {
panic(err)
}
if mock.OnCreatePod != nil {
statusCode, response := mock.OnCreateProject(&ns)
w.WriteHeader(statusCode)
b := new(bytes.Buffer)
json.NewEncoder(b).Encode(response)
w.Write(b.Bytes())
return
}
w.WriteHeader(http.StatusNotImplemented)
}).Methods("PUT")
router.HandleFunc(
cciPodsRoute,
func(w http.ResponseWriter, r *http.Request) {
var pod v1.Pod
if err := json.NewDecoder(r.Body).Decode(&pod); err != nil {
panic(err)
}
if mock.OnCreatePod != nil {
statusCode, response := mock.OnCreatePod(&pod)
w.WriteHeader(statusCode)
b := new(bytes.Buffer)
json.NewEncoder(b).Encode(response)
w.Write(b.Bytes())
return
}
w.WriteHeader(http.StatusNotImplemented)
}).Methods("PUT")
router.HandleFunc(
cciPodRoute,
func(w http.ResponseWriter, r *http.Request) {
namespace, _ := mux.Vars(r)["namespaceID"]
podname, _ := mux.Vars(r)["podID"]
if mock.OnGetPod != nil {
statusCode, response := mock.OnGetPod(namespace, podname)
w.WriteHeader(statusCode)
b := new(bytes.Buffer)
json.NewEncoder(b).Encode(response)
w.Write(b.Bytes())
return
}
w.WriteHeader(http.StatusNotImplemented)
}).Methods("GET")
router.HandleFunc(
cciPodsRoute,
func(w http.ResponseWriter, r *http.Request) {
if mock.OnGetPods != nil {
statusCode, response := mock.OnGetPods()
w.WriteHeader(statusCode)
b := new(bytes.Buffer)
json.NewEncoder(b).Encode(response)
w.Write(b.Bytes())
return
}
w.WriteHeader(http.StatusNotImplemented)
}).Methods("GET")
mock.server = httptest.NewServer(router)
}
// GetServerURL returns the mock server URL.
func (mock *CCIMock) GetServerURL() string {
if mock.server != nil {
return mock.server.URL
}
panic("Mock server is not initialized.")
}
// Close terminates the CCI mock server.
func (mock *CCIMock) Close() {
if mock.server != nil {
mock.server.Close()
mock.server = nil
}
}

212
providers/huawei/cci_test.go Normal file
View File

@@ -0,0 +1,212 @@
package huawei
import (
"net/http"
"os"
"testing"
"github.com/stretchr/testify/assert"
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/uuid"
)
const (
fakeAppKey = "Whj8f5RAHsvQahveqCdo"
fakeAppSecret = "ymW5JgrdwrIvRS76YxyIqHNXe9s5ocIhaWWvPUhx"
fakeRegion = "southchina"
fakeService = "default"
fakeProject = "vk-project"
fakeNodeName = "vk"
)
// TestCreateProject test create project.
func TestCreateProject(t *testing.T) {
cciServerMocker, provider, err := prepareMocks()
if err != nil {
t.Fatal("Unable to prepare the mocks", err)
}
cciServerMocker.OnCreateProject = func(ns *v1.Namespace) (int, interface{}) {
assert.NotNil(t, ns, "Project is nil")
assert.Equal(t, fakeProject, ns.Name, "pod.Annotations[\"virtual-kubelet-podname\"] is not expected")
return http.StatusOK, &v1.Namespace{
TypeMeta: metav1.TypeMeta{
Kind: "Namespace",
APIVersion: "v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: fakeProject,
},
}
}
if err := provider.createProject(); err != nil {
t.Fatal("Failed to create project", err)
}
}
// TestCreatePod test create pod.
func TestCreatePod(t *testing.T) {
cciServerMocker, provider, err := prepareMocks()
if err != nil {
t.Fatal("Unable to prepare the mocks", err)
}
podName := "pod-" + string(uuid.NewUUID())
podNamespace := "ns-" + string(uuid.NewUUID())
cciServerMocker.OnCreatePod = func(pod *v1.Pod) (int, interface{}) {
assert.NotNil(t, pod, "Pod is nil")
assert.NotNil(t, pod.Annotations, "pod.Annotations is expected")
assert.Equal(t, podName, pod.Annotations[podAnnotationPodNameKey], "pod.Annotations[\"virtual-kubelet-podname\"] is not expected")
assert.Equal(t, podNamespace, pod.Annotations[podAnnotationNamespaceKey], "pod.Annotations[\"virtual-kubelet-namespace\"] is not expected")
assert.Equal(t, 1, len(pod.Spec.Containers), "1 Container is expected")
assert.Equal(t, "nginx", pod.Spec.Containers[0].Name, "Container nginx is expected")
return http.StatusOK, pod
}
pod := &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: podName,
Namespace: podNamespace,
},
Spec: v1.PodSpec{
Containers: []v1.Container{
v1.Container{
Name: "nginx",
},
},
},
}
if err := provider.CreatePod(pod); err != nil {
t.Fatal("Failed to create pod", err)
}
}
// Tests get pod.
func TestGetPod(t *testing.T) {
cciServerMocker, provider, err := prepareMocks()
if err != nil {
t.Fatal("Unable to prepare the mocks", err)
}
podName := "pod-" + string(uuid.NewUUID())
podNamespace := "ns-" + string(uuid.NewUUID())
cciServerMocker.OnGetPod = func(namespace, name string) (int, interface{}) {
annotations := map[string]string{
podAnnotationPodNameKey: "podname",
podAnnotationNamespaceKey: "podnamespaces",
podAnnotationUIDkey: "poduid",
podAnnotationClusterNameKey: "podclustername",
podAnnotationNodeName: "podnodename",
}
return http.StatusOK, &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: podName,
Namespace: podNamespace,
Annotations: annotations,
},
Spec: v1.PodSpec{
Containers: []v1.Container{
v1.Container{
Name: "nginx",
},
},
},
}
}
pod, err := provider.GetPod(podNamespace, podName)
if err != nil {
t.Fatal("Failed to get pod", err)
}
assert.NotNil(t, pod, "Response pod should not be nil")
assert.NotNil(t, pod.Spec.Containers, "Containers should not be nil")
assert.Equal(t, pod.Name, "podname", "Pod name is not expected")
assert.Equal(t, pod.Namespace, "podnamespaces", "Pod namespace is not expected")
assert.Nil(t, pod.Annotations, "Pod Annotations should be nil")
assert.Equal(t, string(pod.UID), "poduid", "Pod UID is not expected")
assert.Equal(t, pod.ClusterName, "podclustername", "Pod clustername is not expected")
assert.Equal(t, pod.Spec.NodeName, "podnodename", "Pod node name is not expected")
}
// Tests get pod.
func TestGetPods(t *testing.T) {
cciServerMocker, provider, err := prepareMocks()
if err != nil {
t.Fatal("Unable to prepare the mocks", err)
}
podName := "pod-" + string(uuid.NewUUID())
podNamespace := "ns-" + string(uuid.NewUUID())
cciServerMocker.OnGetPods = func() (int, interface{}) {
annotations := map[string]string{
podAnnotationPodNameKey: "podname",
podAnnotationNamespaceKey: "podnamespaces",
podAnnotationUIDkey: "poduid",
podAnnotationClusterNameKey: "podclustername",
podAnnotationNodeName: "podnodename",
}
pod := v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: podName,
Namespace: podNamespace,
Annotations: annotations,
},
Spec: v1.PodSpec{
Containers: []v1.Container{
v1.Container{
Name: "nginx",
},
},
},
}
return http.StatusOK, []v1.Pod{pod}
}
pods, err := provider.GetPods()
if err != nil {
t.Fatal("Failed to get pods", err)
}
pod := pods[0]
assert.NotNil(t, pod, "Response pod should not be nil")
assert.NotNil(t, pod.Spec.Containers, "Containers should not be nil")
assert.Equal(t, pod.Name, "podname", "Pod name is not expected")
assert.Equal(t, pod.Namespace, "podnamespaces", "Pod namespace is not expected")
assert.Nil(t, pod.Annotations, "Pod Annotations should be nil")
assert.Equal(t, string(pod.UID), "poduid", "Pod UID is not expected")
assert.Equal(t, pod.ClusterName, "podclustername", "Pod clustername is not expected")
assert.Equal(t, pod.Spec.NodeName, "podnodename", "Pod node name is not expected")
}
func prepareMocks() (*CCIMock, *CCIProvider, error) {
cciServerMocker := NewCCIMock()
os.Setenv("CCI_APP_KEP", fakeAppKey)
os.Setenv("CCI_APP_SECRET", fakeAppSecret)
defaultApiEndpoint = cciServerMocker.GetServerURL()
provider, err := NewCCIProvider("cci.toml", nil, fakeNodeName, "Linux", "0.0.0.0", 10250)
if err != nil {
return nil, nil, err
}
provider.project = fakeProject
provider.client.Signer = &fakeSigner{
AppKey: fakeAppKey,
AppSecret: fakeAppSecret,
Region: fakeRegion,
Service: fakeService,
}
return cciServerMocker, provider, nil
}

54
providers/huawei/config.go Normal file
View File

@@ -0,0 +1,54 @@
package huawei
import (
"io"
"github.com/BurntSushi/toml"
"github.com/virtual-kubelet/virtual-kubelet/providers"
"k8s.io/apimachinery/pkg/util/uuid"
)
type providerConfig struct {
Project string
Region string
Service string
OperatingSystem string
CPU string
Memory string
Pods string
}
func (p *CCIProvider) loadConfig(r io.Reader) error {
var config providerConfig
if _, err := toml.DecodeReader(r, &config); err != nil {
return err
}
p.apiEndpoint = defaultApiEndpoint
p.service = "CCI"
p.region = config.Region
if p.region == "" {
p.region = "southchina"
}
p.cpu = config.CPU
if p.cpu == "" {
p.cpu = "20"
}
p.memory = config.Memory
if p.memory == "" {
p.memory = "100Gi"
}
p.pods = config.Pods
if p.pods == "" {
p.pods = "20"
}
p.project = config.Project
if p.project == "" {
p.project = string(uuid.NewUUID())
}
p.operatingSystem = config.OperatingSystem
if p.operatingSystem == "" {
p.operatingSystem = providers.OperatingSystemLinux
}
return nil
}