From fb34a8c990ac0c1938cc4bbf5dcadc5f09b003ea Mon Sep 17 00:00:00 2001 From: Jimmy Xu Date: Wed, 20 Dec 2017 17:36:50 +0800 Subject: [PATCH] [hyper-provider] 1.add doc 2.support read hyper config file --- providers/hypersh/README.md | 124 +++++++++++++++++++++++++++ providers/hypersh/config.go | 64 -------------- providers/hypersh/example.toml | 8 -- providers/hypersh/hypersh.go | 149 ++++++++++++++++++++++++++------- 4 files changed, 245 insertions(+), 100 deletions(-) create mode 100644 providers/hypersh/README.md delete mode 100644 providers/hypersh/config.go delete mode 100644 providers/hypersh/example.toml diff --git a/providers/hypersh/README.md b/providers/hypersh/README.md new file mode 100644 index 000000000..8a871517a --- /dev/null +++ b/providers/hypersh/README.md @@ -0,0 +1,124 @@ +hyper.sh provider for virtual-kubelet +===================================== + +# Configure for hyper.sh + +## Use environment variable + +- necessary + - HYPER_ACCESS_KEY + - HYPER_SECRET_KEY +- optional + - HYPER_INSTANCE_TYPE: default s4 + - HYPER_DEFAULT_REGION: default us-west-1 + - HYPER_HOST: tcp://${HYPER_DEFAULT_REGION}.hyper.sh:443 + +> You can use You can use either HYPER_HOST or HYPER_DEFAULT_REGION + + +## Use config file + +> default config file for hyper.sh is ~/.hyper/config.json + +``` +//example configuration file for Hyper.sh +{ + "auths": { + "https://index.docker.io/v1/": { + "auth": "xxxxxx", + "email": "xxxxxx" + }, + }, + "clouds": { + "tcp://*.hyper.sh:443": { + "accesskey": "xxxxxx", + "secretkey": "xxxxxx", + "region": "us-west-1" + } + } +} +``` + +# Usage of virtual-kubelet cli + +``` +// example 1 : use environment variable +export HYPER_ACCESS_KEY=xxxxxx +export HYPER_SECRET_KEY=xxxxxx +export HYPER_DEFAULT_REGION=eu-central-1 +export HYPER_INSTANCE_TYPE=s4 +./virtual-kubelet --provider=hyper + + +// example 2 : use default config file(~/.hyper/config.json) +unset HYPER_ACCESS_KEY +unset HYPER_SECRET_KEY +export HYPER_DEFAULT_REGION=eu-central-1 +./virtual-kubelet --provider=hyper + + +// example 3 : use custom config file, eg: ~/.hyper2/config.json +$ ./virtual-kubelet --provider=hyper --provider-config=$HOME/.hyper2 +``` + + +# Quick Start + +## create pod yaml + +``` +$ cat pod-nginx +apiVersion: v1 +kind: Pod +metadata: + name: nginx +spec: + nodeName: virtual-kubelet + containers: + - name: nginx + image: nginx:latest + ports: + - containerPort: 80 +``` + +## create pod + +``` +$ kubectl create -f pod-nginx +``` + +## list container on hyper.sh + +``` +$ hyper ps +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES PUBLIC IP +a0ae3d4112d5 nginx:latest "nginx -g 'daemon off" 9 seconds ago Up 4 seconds 0.0.0.0:80->80/tcp pod-nginx-nginx +``` + +## server log + +``` +$ export HYPER_DEFAULT_REGION=eu-central-1 +$ ./virtual-kubelet --provider=hyper --provider-config=$HOME/.hyper3 +/home/xjimmy/.kube/config +2017/12/20 17:30:30 config file under "/home/xjimmy/.hyper3" was loaded +2017/12/20 17:30:30 + Host: tcp://eu-central-1.hyper.sh:443 + AccessKey: K********** + SecretKey: 4********** + InstanceType: s4 +2017/12/20 17:30:31 Node 'virtual-kubelet' with OS type 'Linux' registered +2017/12/20 17:30:31 receive GetPods +2017/12/20 17:30:32 found 0 pods +2017/12/20 17:30:37 receive GetPods +2017/12/20 17:30:37 found 0 pods +2017/12/20 17:30:38 Error retrieving pod 'nginx' from provider: Error: No such container: pod-nginx-nginx +2017/12/20 17:30:38 receive CreatePod "nginx" +2017/12/20 17:30:38 container "a0ae3d4112d53023b5972906f2f15c0d34360c132b3c273b286473afad613b63" for pod "nginx" was created +2017/12/20 17:30:43 container "a0ae3d4112d53023b5972906f2f15c0d34360c132b3c273b286473afad613b63" for pod "nginx" was started +2017/12/20 17:30:43 Pod 'nginx' created. +2017/12/20 17:30:43 receive GetPods +2017/12/20 17:30:43 found 1 pods +2017/12/20 17:30:47 receive GetPods +2017/12/20 17:30:47 found 1 pods +``` diff --git a/providers/hypersh/config.go b/providers/hypersh/config.go deleted file mode 100644 index 52f61b075..000000000 --- a/providers/hypersh/config.go +++ /dev/null @@ -1,64 +0,0 @@ -package hypersh - -import ( - "fmt" - "io" - - "github.com/BurntSushi/toml" - "github.com/virtual-kubelet/virtual-kubelet/providers" -) - -type providerConfig struct { - Region string - AccessKey string - SecretKey string - OperatingSystem string - CPU string - Memory string - InstanceType string - Pods string -} - -func (p *HyperProvider) loadConfig(r io.Reader) error { - var config providerConfig - if _, err := toml.DecodeReader(r, &config); err != nil { - return err - } - p.region = config.Region - p.accessKey = config.AccessKey - p.secretKey = config.SecretKey - - p.instanceType = "s4" - if config.InstanceType != "" { - p.instanceType = config.InstanceType - } - - // Default to 20 mcpu - p.cpu = "20" - if config.CPU != "" { - p.cpu = config.CPU - } - // Default to 100Gi - p.memory = "100Gi" - if config.Memory != "" { - p.memory = config.Memory - } - // Default to 20 pods - p.pods = "20" - if config.Pods != "" { - p.pods = config.Pods - } - - // Default to Linux if the operating system was not defined in the config. - if config.OperatingSystem == "" { - config.OperatingSystem = providers.OperatingSystemLinux - } - - // Validate operating system from config. - if config.OperatingSystem != providers.OperatingSystemLinux { - return fmt.Errorf("%q is not a valid operating system, only %s is valid", config.OperatingSystem, providers.OperatingSystemLinux) - } - - p.operatingSystem = config.OperatingSystem - return nil -} diff --git a/providers/hypersh/example.toml b/providers/hypersh/example.toml deleted file mode 100644 index cf439affc..000000000 --- a/providers/hypersh/example.toml +++ /dev/null @@ -1,8 +0,0 @@ -# example configuration file for Hyper.sh virtual-kubelet - -AccessKey = "" -SecretKey = "" -Region = "us-west-1" -CPU = 100 -Memory = 100Gi -Pods = 50 diff --git a/providers/hypersh/hypersh.go b/providers/hypersh/hypersh.go index 654816edd..ae70e0819 100755 --- a/providers/hypersh/hypersh.go +++ b/providers/hypersh/hypersh.go @@ -5,6 +5,7 @@ import ( "fmt" "log" "net/http" + "net/url" "os" "runtime" "strings" @@ -18,6 +19,8 @@ import ( "github.com/hyperhq/hyper-api/types/container" "github.com/hyperhq/hyper-api/types/filters" "github.com/hyperhq/hyper-api/types/network" + "github.com/hyperhq/hypercli/cliconfig" + "github.com/hyperhq/hypercli/opts" "github.com/virtual-kubelet/virtual-kubelet/manager" "github.com/virtual-kubelet/virtual-kubelet/providers" "k8s.io/api/core/v1" @@ -37,10 +40,12 @@ const ( // HyperProvider implements the virtual-kubelet provider interface and communicates with hyper.sh APIs. type HyperProvider struct { hyperClient *hyper.Client + configFile *cliconfig.ConfigFile resourceManager *manager.ResourceManager nodeName string operatingSystem string region string + host string accessKey string secretKey string cpu string @@ -51,41 +56,102 @@ type HyperProvider struct { // NewHyperProvider creates a new HyperProvider func NewHyperProvider(config string, rm *manager.ResourceManager, nodeName, operatingSystem string) (*HyperProvider, error) { - var p HyperProvider - var err error + var ( + p HyperProvider + err error + host string + dft bool + tlsOptions = &tlsconfig.Options{InsecureSkipVerify: false} + ) p.resourceManager = rm - if config != "" { - f, err := os.Open(config) - if err != nil { - return nil, err - } - defer f.Close() - - if err := p.loadConfig(f); err != nil { - return nil, err - } + // Get config from environment variable + if h := os.Getenv("HYPER_HOST"); h != "" { + p.host = h } - - if ak := os.Getenv("HYPERSH_ACCESS_KEY"); ak != "" { + if ak := os.Getenv("HYPER_ACCESS_KEY"); ak != "" { p.accessKey = ak } - - if sk := os.Getenv("HYPERSH_SECRET_KEY"); sk != "" { + if sk := os.Getenv("HYPER_SECRET_KEY"); sk != "" { p.secretKey = sk } - - if r := os.Getenv("HYPERSH_REGION"); r != "" { - p.region = r + if p.host == "" { + // ignore HYPER_DEFAULT_REGION when HYPER_HOST was specified + if r := os.Getenv("HYPER_DEFAULT_REGION"); r != "" { + p.region = r + } } - - if it := os.Getenv("HYPERSH_INSTANCE_TYPE"); it != "" { + if it := os.Getenv("HYPER_INSTANCE_TYPE"); it != "" { p.instanceType = it + } else { + p.instanceType = "s4" } - host = fmt.Sprintf("tcp://%s.hyper.sh:443", p.region) - httpClient, err := newHTTPClient(host, &tlsconfig.Options{InsecureSkipVerify: false}) + if p.accessKey != "" || p.secretKey != "" { + //use environment variable + if p.accessKey == "" || p.secretKey == "" { + return nil, fmt.Errorf("WARNING: Need to specify HYPER_ACCESS_KEY and HYPER_SECRET_KEY at the same time.") + } + log.Printf("Use AccessKey and SecretKey from HYPER_ACCESS_KEY and HYPER_SECRET_KEY") + if p.region == "" { + p.region = cliconfig.DefaultHyperRegion + } + if p.host == "" { + host, _, err = p.getServerHost(p.region, tlsOptions) + if err != nil { + return nil, err + } + p.host = host + } + } else { + // use config file, default path is ~/.hyper + if config == "" { + config = cliconfig.ConfigDir() + } + configFile, err := cliconfig.Load(config) + if err != nil { + return nil, fmt.Errorf("WARNING: Error loading config file %q: %v\n", config, err) + } + p.configFile = configFile + log.Printf("config file under %q was loaded\n", config) + + if p.host == "" { + host, dft, err = p.getServerHost(p.region, tlsOptions) + if err != nil { + return nil, err + } + p.host = host + } + // Get Region, AccessKey and SecretKey from config file + cc, ok := configFile.CloudConfig[p.host] + if !ok { + cc, ok = configFile.CloudConfig[cliconfig.DefaultHyperFormat] + } + if ok { + p.accessKey = cc.AccessKey + p.secretKey = cc.SecretKey + + if p.region == "" && dft { + if p.region = cc.Region; p.region == "" { + p.region = p.getDefaultRegion() + } + } + if !dft { + if p.region = cc.Region; p.region == "" { + p.region = cliconfig.DefaultHyperRegion + } + } + } else { + return nil, fmt.Errorf("WARNING: can not find entrypoint %q in config file", cliconfig.DefaultHyperFormat) + } + if p.accessKey == "" || p.secretKey == "" { + return nil, fmt.Errorf("WARNING: AccessKey or SecretKey is empty in config %q", config) + } + } + + log.Printf("\n Host: %s\n AccessKey: %s**********\n SecretKey: %s**********\n InstanceType: %s\n", p.host, p.accessKey[0:1], p.secretKey[0:1], p.instanceType) + httpClient, err := newHTTPClient(p.host, tlsOptions) customHeaders := map[string]string{} ver := "0.1" @@ -94,11 +160,15 @@ func NewHyperProvider(config string, rm *manager.ResourceManager, nodeName, oper p.operatingSystem = operatingSystem p.nodeName = nodeName - p.hyperClient, err = hyper.NewClient(host, verStr, httpClient, customHeaders, p.accessKey, p.secretKey, p.region) + p.hyperClient, err = hyper.NewClient(p.host, verStr, httpClient, customHeaders, p.accessKey, p.secretKey, p.region) + if err != nil { + return nil, err + } + //test connect to hyper.sh + _, err = p.hyperClient.Info(context.Background()) if err != nil { return nil, err } - return &p, nil } @@ -201,7 +271,7 @@ func (p *HyperProvider) DeletePod(pod *v1.Pod) (err error) { if v, ok := container.Config.Labels[containerLabel]; ok { // Check value of label if v != pod.Name { - return fmt.Errorf("the label %q of hyper container %q should be %q, but it's %q currently", pod.Name, containerLabel, container.Name, pod.Name, v) + return fmt.Errorf("the label %q of hyper container %q should be %q, but it's %q currently", containerLabel, container.Name, pod.Name, v) } rmOptions := types.ContainerRemoveOptions{ RemoveVolumes: true, @@ -218,7 +288,7 @@ func (p *HyperProvider) DeletePod(pod *v1.Pod) (err error) { } log.Printf("container %q for pod %q was deleted\n", container.ID, pod.Name) } else { - return fmt.Errorf("hyper container %q has no label %q", pod.Name, container.Name, containerLabel) + return fmt.Errorf("hyper container %q has no label %q", container.Name, containerLabel) } return nil } @@ -275,7 +345,7 @@ func (p *HyperProvider) GetPods() ([]*v1.Pod, error) { for _, container := range containers { pod, err := containerToPod(&container) if err != nil { - fmt.Errorf("convert container %q to pod error: %v\n", container.ID, err) + log.Printf("WARNING: convert container %q to pod error: %v\n", container.ID, err) continue } pods = append(pods, pod) @@ -541,3 +611,26 @@ func hyperStateToPodPhase(state string) v1.PodPhase { } return v1.PodUnknown } + +func (p *HyperProvider) getServerHost(region string, tlsOptions *tlsconfig.Options) (host string, dft bool, err error) { + dft = false + host = region + if host == "" { + host = os.Getenv("HYPER_DEFAULT_REGION") + region = p.getDefaultRegion() + } + if _, err := url.ParseRequestURI(host); err != nil { + host = "tcp://" + region + "." + cliconfig.DefaultHyperEndpoint + dft = true + } + host, err = opts.ParseHost(tlsOptions != nil, host) + return +} + +func (p *HyperProvider) getDefaultRegion() string { + cc, ok := p.configFile.CloudConfig[cliconfig.DefaultHyperFormat] + if ok && cc.Region != "" { + return cc.Region + } + return cliconfig.DefaultHyperRegion +}