Merge branch 'master' into maintainers

This commit is contained in:
Brian Goff
2019-06-18 15:26:21 -07:00
committed by GitHub
2339 changed files with 19796 additions and 618260 deletions

View File

@@ -2,7 +2,7 @@ version: 2
jobs:
validate:
docker:
- image: circleci/golang:1.10
- image: circleci/golang:1.12
working_directory: /go/src/github.com/virtual-kubelet/virtual-kubelet
steps:
- checkout
@@ -21,20 +21,25 @@ jobs:
test:
docker:
- image: circleci/golang:1.10
- image: circleci/golang:1.12
working_directory: /go/src/github.com/virtual-kubelet/virtual-kubelet
steps:
- checkout
- run:
name: Create the credentials file
command: sh scripts/createCredentials.sh
- run: |
echo 'export AZURE_AUTH_LOCATION=${outputPathCredsfile}' >> $BASH_ENV
- run: |
echo 'export KUBECONFIG=${outputPathKubeConfigFile}' >> $BASH_ENV
- run:
name: Build
command: V=1 make build
- run:
name: Install Nomad
command: |
curl \
--silent \
--location \
--output nomad.zip \
https://releases.hashicorp.com/nomad/0.8.6/nomad_0.8.6_linux_amd64.zip && \
unzip nomad.zip && \
chmod +x nomad && \
mv nomad /go/bin/nomad && \
rm nomad.zip
- run:
name: Tests
command: V=1 CI=1 SKIP_AWS_E2E=1 make test
@@ -86,10 +91,28 @@ jobs:
until kubectl get nodes -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do
sleep 1;
done
- run:
name: Watch pods
command: kubectl get pods -o json --watch
background: true
- run:
name: Watch nodes
command: kubectl get nodes -o json --watch
background: true
- run:
name: Run the end-to-end test suite
command: |
mkdir $HOME/.go
export PATH=$HOME/.go/bin:${PATH}
curl -fsSL -o "/tmp/go.tar.gz" "https://dl.google.com/go/go1.12.5.linux-amd64.tar.gz"
tar -C $HOME/.go --strip-components=1 -xzf "/tmp/go.tar.gz"
go version
make e2e
- run:
name: Collect logs on failure from vkubelet-mock-0
command: |
kubectl logs vkubelet-mock-0
when: on_fail
workflows:
version: 2
@@ -97,4 +120,7 @@ workflows:
jobs:
- validate
- test
- e2e
- e2e:
requires:
- validate
- test

3
.gitignore vendored
View File

@@ -33,6 +33,9 @@ loganalytics.json
# VS Code files
.vscode/
# IntelliJ Goland files
.idea
# Terraform ignores
**/.terraform/**
**/terraform-provider-kubernetes

3
ADOPTERS.md Normal file
View File

@@ -0,0 +1,3 @@
## Virtual Kubelet adopters
Are you currently using Virtual Kubelet in production? Please let us know by adding your company name and a description of your use case to this document!

View File

@@ -9,4 +9,5 @@ Rita Zhang <rita.z.zhang@gmail.com>
Robbie Zhang <junjiez@microsoft.com>
Ben Corrie <bcorrie@vmware.com>
Fei Xu <xufei40@huawei.com>
Eric Jadi <erjadi@microsoft.com>
Eric Jadi <erjadi@microsoft.com>
Anubhav Mishra <mishra@hashicorp.com>

View File

@@ -1,25 +1,12 @@
FROM golang:alpine as builder
FROM golang:1.12 as builder
ENV PATH /go/bin:/usr/local/go/bin:$PATH
ENV GOPATH /go
RUN apk add --no-cache \
ca-certificates \
--virtual .build-deps \
git \
gcc \
libc-dev \
libgcc \
make \
bash
COPY . /go/src/github.com/virtual-kubelet/virtual-kubelet
WORKDIR /go/src/github.com/virtual-kubelet/virtual-kubelet
ARG BUILD_TAGS="netgo osusergo"
ARG BUILD_TAGS=""
RUN make VK_BUILD_TAGS="${BUILD_TAGS}" build
RUN cp bin/virtual-kubelet /usr/bin/virtual-kubelet
FROM scratch
COPY --from=builder /usr/bin/virtual-kubelet /usr/bin/virtual-kubelet
COPY --from=builder /etc/ssl/certs/ /etc/ssl/certs

1013
Gopkg.lock generated

File diff suppressed because it is too large Load Diff

View File

@@ -33,40 +33,10 @@
name = "github.com/spf13/cobra"
version = "0.0.1"
[[constraint]]
name = "github.com/spf13/viper"
version = "1.0.0"
[[constraint]]
name = "github.com/google/uuid"
version = "0.2.0"
[[constraint]]
name = "github.com/hyperhq/hyper-api"
revision = "18c77d3f9fe0abebb41b45c12f383ecac46f4ff1"
[[constraint]]
name = "github.com/gorilla/mux"
version = "1.6.0"
[[constraint]]
name = "github.com/hyperhq/hypercli"
revision = "29217d318cab52815518a1126d57ca010de83e4d"
[[constraint]]
name = "github.com/Azure/azure-sdk-for-go"
version = "21.1.0"
[[constraint]]
name = "github.com/lawrencegripper/pod2docker"
version = "0.5.1"
[[constraint]]
name = "github.com/aws/aws-sdk-go"
version = "1.13.38"
[[constraint]]
name = "k8s.io/api"
version = "kubernetes-1.13.1"
@@ -80,70 +50,5 @@
version = "kubernetes-1.13.1"
[[constraint]]
name = "github.com/cpuguy83/strongerrors"
version = "0.2.1"
# These are required for HyperHQ
[[override]]
name = "github.com/xeipuuv/gojsonschema"
revision = "0c8571ac0ce161a5feb57375a9cdf148c98c0f70"
[[override]]
name = "github.com/docker/libcompose"
version = "0.2.0"
[[override]]
name = "github.com/hyperhq/libcompose"
revision = "15d3a105140f968f5d4f62d2f44afd22a24a98fb"
[[constraint]]
branch = "feature/wolfpack"
name = "github.com/vmware/vic"
[[override]]
name = "github.com/docker/docker"
revision = "49bf474f9ed7ce7143a59d1964ff7b7fd9b52178"
[[override]]
name = "github.com/vishvananda/netlink"
revision = "482f7a52b758233521878cb6c5904b6bd63f3457"
[[override]]
name = "github.com/opencontainers/runtime-spec"
revision = "1c7c27d043c2a5e513a44084d2b10d77d1402b8c"
[[override]]
name = "github.com/go-openapi/analysis"
revision = "d5a75b7d751ca3f11ad5d93cfe97405f2c3f6a47"
[[override]]
name = "github.com/go-openapi/errors"
revision = "fc3f73a224499b047eda7191e5d22e1e9631e86f"
[[override]]
name = "github.com/go-openapi/loads"
revision = "6bb6486231e079ea125c0f39994ed3d0c53399ed"
[[override]]
name = "github.com/go-openapi/runtime"
revision = "3b13ebb46790d871d74a6c2450fa4b1280f90854"
[[override]]
name = "github.com/go-openapi/strfmt"
revision = "0cb3db44c13bad3b3f567b762a66751972a310cc"
[[override]]
name = "github.com/go-openapi/validate"
revision = "035dcd74f1f61e83debe1c22950dc53556e7e4b2"
[[constraint]]
name = "github.com/Azure/go-autorest"
version = "10.8.1"
[[constraint]]
name = "github.com/aliyun/alibaba-cloud-sdk-go"
version = "1.27.7"
[[override]]
name = "github.com/json-iterator/go"
version = "v1.1.5"
name = "gotest.tools"
version = "2.3.0"

View File

@@ -3,7 +3,7 @@
### Environment summary
Provider (e.g. ACI, AWS Fargate, Hyper)
Provider (e.g. ACI, AWS Fargate)
Version (e.g. 0.1, 0.2-beta)

120
Makefile
View File

@@ -1,13 +1,23 @@
SHELL := /bin/bash
IMPORT_PATH := github.com/virtual-kubelet/virtual-kubelet
DOCKER_IMAGE := virtual-kubelet
exec := $(DOCKER_IMAGE)
github_repo := virtual-kubelet/virtual-kubelet
binary := virtual-kubelet
build_tags := "netgo osusergo $(VK_BUILD_TAGS)"
include Makefile.e2e
# Currently this looks for a globally installed gobin. When we move to modules,
# should consider installing it locally
# Also, we will want to lock our tool versions using go mod:
# https://github.com/golang/go/wiki/Modules#how-can-i-track-tool-dependencies-for-a-module
gobin_tool ?= $(shell which gobin || echo $(GOPATH)/bin/gobin)
goimports := golang.org/x/tools/cmd/goimports@release-branch.go1.12
gocovmerge := github.com/wadey/gocovmerge@b5bfa59ec0adc420475f97f89b58045c721d761c
goreleaser := github.com/goreleaser/goreleaser@v0.82.2
gox := github.com/mitchellh/gox@v1.0.1
# comment this line out for quieter things
#V := 1 # When V is set, print commands and build progress.
# V := 1 # When V is set, print commands and build progress.
# Space separated patterns of packages to skip in list, test, format.
IGNORED_PACKAGES := /vendor/
@@ -19,12 +29,14 @@ all: test build
# safebuild builds inside a docker container with no clingons from your $GOPATH
safebuild:
@echo "Building..."
$Q docker build --build-arg BUILD_TAGS=$(build_tags) -t $(DOCKER_IMAGE):$(VERSION) .
$Q docker build --build-arg BUILD_TAGS="$(VK_BUILD_TAGS)" -t $(DOCKER_IMAGE):$(VERSION) .
.PHONY: build
build: build_tags := netgo osusergo
build: OUTPUT_DIR ?= bin
build: authors
@echo "Building..."
$Q CGO_ENABLED=0 go build -a --tags $(build_tags) -ldflags '-extldflags "-static"' -o bin/$(binary) $(if $V,-v) $(VERSION_FLAGS) $(IMPORT_PATH)
$Q CGO_ENABLED=0 go build -ldflags '-extldflags "-static"' -o $(OUTPUT_DIR)/$(binary) $(if $V,-v) $(VERSION_FLAGS) ./cmd/$(binary)
.PHONY: tags
tags:
@@ -32,16 +44,8 @@ tags:
$Q @git tag
.PHONY: release
release: build $(GOPATH)/bin/goreleaser
goreleaser
### Code not in the repository root? Another binary? Add to the path like this.
# .PHONY: otherbin
# otherbin: .GOPATH/.ok
# $Q go install $(if $V,-v) $(VERSION_FLAGS) $(IMPORT_PATH)/cmd/otherbin
##### ^^^^^^ EDIT ABOVE ^^^^^^ #####
release: build goreleaser
$(gobin_tool) -run $(goreleaser)
##### =====> Utility targets <===== #####
@@ -89,7 +93,7 @@ list:
@echo "List..."
@echo $(allpackages)
cover: $(GOPATH)/bin/gocovmerge
cover: gocovmerge
@echo "Coverage Report..."
@echo "NOTE: make cover does not exit 1 on failure, don't use it to check for tests success!"
$Q rm -f .GOPATH/cover/*.out cover/all.merged
@@ -99,7 +103,7 @@ cover: $(GOPATH)/bin/gocovmerge
-coverprofile=cover/unit-`echo $$MOD|tr "/" "_"`.out \
$$MOD 2>&1 | grep -v "no packages being tested depend on"; \
done
$Q gocovmerge cover/*.out > cover/all.merged
$Q $(gobin_tool) -run $(gocovmerge) cover/*.out > cover/all.merged
ifndef CI
@echo "Coverage Report..."
$Q go tool cover -html .GOPATH/cover/all.merged
@@ -112,54 +116,18 @@ endif
@echo ""
$Q go tool cover -func .GOPATH/cover/all.merged
format: $(GOPATH)/bin/goimports
format: goimports
@echo "Formatting..."
$Q find . -iname \*.go | grep -v \
-e "^$$" $(addprefix -e ,$(IGNORED_PACKAGES)) | xargs goimports -w
-e "^$$" $(addprefix -e ,$(IGNORED_PACKAGES)) | xargs $(gobin_tool) -run $(goimports) -w
# skaffold deploys the virtual-kubelet to the Kubernetes cluster targeted by the current kubeconfig using skaffold.
# The current context (as indicated by "kubectl config current-context") must be one of "minikube" or "docker-for-desktop".
# MODE must be set to one of "dev" (default), "delete" or "run", and is used as the skaffold command to be run.
.PHONY: skaffold
skaffold: MODE ?= dev
skaffold: PROFILE := local
skaffold: VK_BUILD_TAGS ?= no_alicloud_provider no_aws_provider no_azure_provider no_azurebatch_provider no_cri_provider no_huawei_provider no_hyper_provider no_vic_provider no_web_provider
skaffold:
@if [[ ! "minikube,docker-for-desktop" =~ .*"$(kubectl_context)".* ]]; then \
echo current-context is [$(kubectl_context)]. Must be one of [minikube,docker-for-desktop]; false; \
fi
@if [[ ! "$(MODE)" == "delete" ]]; then \
GOOS=linux GOARCH=amd64 VK_BUILD_TAGS="$(VK_BUILD_TAGS)" $(MAKE) build; \
fi
@skaffold $(MODE) \
-f $(PWD)/hack/skaffold/virtual-kubelet/skaffold.yml \
-p $(PROFILE)
# e2e runs the end-to-end test suite against the Kubernetes cluster targeted by the current kubeconfig.
# It automatically deploys the virtual-kubelet with the mock provider by running "make skaffold MODE=run".
# It is the caller's responsibility to cleanup the deployment after running this target (e.g. by running "make skaffold MODE=delete").
.PHONY: e2e
e2e: KUBECONFIG ?= $(HOME)/.kube/config
e2e: NAMESPACE := default
e2e: NODE_NAME := vkubelet-mock-0
e2e: TAINT_KEY := virtual-kubelet.io/provider
e2e: TAINT_VALUE := mock
e2e: TAINT_EFFECT := NoSchedule
e2e:
@$(MAKE) skaffold MODE=delete && kubectl delete --ignore-not-found node $(NODE_NAME)
@$(MAKE) skaffold MODE=run
@cd $(PWD)/test/e2e && go test -v -tags e2e ./... \
-kubeconfig=$(KUBECONFIG) \
-namespace=$(NAMESPACE) \
-node-name=$(NODE_NAME) \
-taint-key=$(TAINT_KEY) \
-taint-value=$(TAINT_VALUE) \
-taint-effect=$(TAINT_EFFECT)
##### =====> Internals <===== #####
.PHONY: setup
setup: clean
setup: goimports gocovmerge goreleaser gox clean
env
@echo "Setup..."
if ! grep "/bin" .gitignore > /dev/null 2>&1; then \
echo "/bin" >> .gitignore; \
@@ -171,14 +139,10 @@ setup: clean
mkdir -p bin
mkdir -p test
go get -u github.com/golang/dep/cmd/dep
go get github.com/wadey/gocovmerge
go get golang.org/x/tools/cmd/goimports
go get github.com/mitchellh/gox
go get github.com/goreleaser/goreleaser
VERSION := $(shell git describe --tags --always --dirty="-dev")
DATE := $(shell date -u '+%Y-%m-%d-%H:%M UTC')
VERSION_FLAGS := -ldflags='-X "github.com/virtual-kubelet/virtual-kubelet/version.Version=$(VERSION)" -X "github.com/virtual-kubelet/virtual-kubelet/version.BuildTime=$(DATE)"'
VERSION_FLAGS := -ldflags='-X "main.buildVersion=$(VERSION)" -X "main.buildTime=$(DATE)"'
# assuming go 1.9 here!!
_allpackages = $(shell go list ./...)
@@ -186,23 +150,27 @@ _allpackages = $(shell go list ./...)
# memoize allpackages, so that it's executed only once and only if used
allpackages = $(if $(__allpackages),,$(eval __allpackages := $$(_allpackages)))$(__allpackages)
.PHONY: goimports
goimports: $(gobin_tool)
$(gobin_tool) -d $(goimports)
.PHONY: gocovmerge
gocovmerge: $(gobin_tool)
$(gobin_tool) -d $(gocovmerge)
.PHONY: goreleaser
goreleaser: $(gobin_tool)
$(gobin_tool) -d $(goreleaser)
.PHONY: gox
gox: $(gobin_tool)
# We make gox globally available, for people to use by hand
$(gobin_tool) $(gox)
Q := $(if $V,,@)
$(GOPATH)/bin/gocovmerge:
@echo "Checking Coverage Tool Installation..."
@test -d $(GOPATH)/src/github.com/wadey/gocovmerge || \
{ echo "Vendored gocovmerge not found, try running 'make setup'..."; exit 1; }
$Q go install github.com/wadey/gocovmerge
$(GOPATH)/bin/goimports:
@echo "Checking Import Tool Installation..."
@test -d $(GOPATH)/src/golang.org/x/tools/cmd/goimports || \
{ echo "Vendored goimports not found, try running 'make setup'..."; exit 1; }
$Q go install golang.org/x/tools/cmd/goimports
$(GOPATH)/bin/goreleaser:
go get -u github.com/goreleaser/goreleaser
$(gobin_tool):
GO111MODULE=off go get -u github.com/myitcv/gobin
authors:
$Q git log --all --format='%aN <%cE>' | sort -u | sed -n '/github/!p' > GITAUTHORS

48
Makefile.e2e Normal file
View File

@@ -0,0 +1,48 @@
.PHONY: skaffold.validate
skaffold.validate: kubectl_context := $(shell kubectl config current-context)
skaffold.validate:
if [[ ! "minikube,docker-for-desktop,docker-desktop" =~ .*"$(kubectl_context)".* ]]; then \
echo current-context is [$(kubectl_context)]. Must be one of [minikube,docker-for-desktop,docker-desktop]; \
false; \
fi
# skaffold deploys the virtual-kubelet to the Kubernetes cluster targeted by the current kubeconfig using skaffold.
# The current context (as indicated by "kubectl config current-context") must be one of "minikube" or "docker-for-desktop".
# MODE must be set to one of "dev" (default), "delete" or "run", and is used as the skaffold command to be run.
.PHONY: skaffold
skaffold: MODE ?= dev
skaffold: skaffold/$(MODE)
.PHONY: skaffold/%
skaffold/%: PROFILE := local
skaffold/%: skaffold.validate
skaffold $(*) \
-f $(PWD)/hack/skaffold/virtual-kubelet/skaffold.yml \
-p $(PROFILE)
bin/e2e:
@mkdir -p bin/e2e
bin/e2e/virtual-kubelet: bin/e2e
GOOS=linux GOARCH=amd64 $(MAKE) OUTPUT_DIR=$(@D) build
# e2e runs the end-to-end test suite against the Kubernetes cluster targeted by the current kubeconfig.
# It automatically deploys the virtual-kubelet with the mock provider by running "make skaffold MODE=run".
# It is the caller's responsibility to cleanup the deployment after running this target (e.g. by running "make skaffold MODE=delete").
.PHONY: e2e
e2e: KUBECONFIG ?= $(HOME)/.kube/config
e2e: NAMESPACE := default
e2e: NODE_NAME := vkubelet-mock-0
e2e: export VK_BUILD_TAGS += mock_provider
e2e: e2e.clean bin/e2e/virtual-kubelet skaffold/run
@echo Running tests...
cd $(PWD)/internal/test/e2e && go test -v -timeout 5m -tags e2e ./... \
-kubeconfig=$(KUBECONFIG) \
-namespace=$(NAMESPACE) \
-node-name=$(NODE_NAME) \
.PHONY: e2e.clean
e2e.clean: NODE_NAME ?= vkubelet-mock-0
e2e.clean: skaffold/delete
kubectl delete --ignore-not-found node $(NODE_NAME); \
if [ -f bin/e2e/virtual-kubelet ]; then rm bin/e2e/virtual-kubelet; fi

View File

@@ -2,7 +2,7 @@
Virtual Kubelet is an open source [Kubernetes kubelet](https://kubernetes.io/docs/reference/generated/kubelet/)
implementation that masquerades as a kubelet for the purposes of connecting Kubernetes to other APIs.
This allows the nodes to be backed by other services like ACI, AWS Fargate, Hyper.sh, [IoT Edge](https://github.com/Azure/iot-edge-virtual-kubelet-provider) etc. The primary scenario for VK is enabling the extension of the Kubernetes API into serverless container platforms like ACI, Fargate, and Hyper.sh, though we are open to others. However, it should be noted that VK is explicitly not intended to be an alternative to Kubernetes federation.
This allows the nodes to be backed by other services like ACI, AWS Fargate, [IoT Edge](https://github.com/Azure/iot-edge-virtual-kubelet-provider) etc. The primary scenario for VK is enabling the extension of the Kubernetes API into serverless container platforms like ACI and Fargate, though we are open to others. However, it should be noted that VK is explicitly not intended to be an alternative to Kubernetes federation.
Virtual Kubelet features a pluggable architecture and direct use of Kubernetes primitives, making it much easier to build on.
@@ -23,8 +23,8 @@ The best description is "Kubernetes API on top, programmable back."
+ [Azure Container Instances Provider](#azure-container-instances-provider)
+ [Azure Batch GPU Provider](./providers/azurebatch/README.md)
+ [AWS Fargate Provider](#aws-fargate-provider)
+ [Hyper.sh Provider](#hypersh-provider)
+ [Service Fabric Mesh Provider](#service-fabric-mesh-provider)
+ [HashiCorp Nomad](#hashicorp-nomad-provider)
+ [OpenStack Zun](#openstack-zun-provider)
+ [Adding a New Provider via the Provider Interface](#adding-a-new-provider-via-the-provider-interface)
* [Testing](#testing)
+ [Unit tests](#unit-tests)
@@ -37,7 +37,7 @@ The best description is "Kubernetes API on top, programmable back."
The diagram below illustrates how Virtual-Kubelet works.
![diagram](diagram.svg)
![diagram](website/static/img/diagram.svg)
## Usage
@@ -139,13 +139,13 @@ Providers must provide the following functionality to be considered a supported
Alibaba Cloud ECI(Elastic Container Instance) is a service that allow you run containers without having to manage servers or clusters.
You can find more details in the [Alibaba Cloud ECI provider documentation](./providers/alicloud/README.md).
You can find more details in the [Alibaba Cloud ECI provider documentation](./providers/alibabacloud/README.md).
#### Configuration File
The alibaba ECI provider will read configuration file specified by the `--provider-config` flag.
The example configure file is `providers/alicloud/eci.toml`.
The example configure file is `providers/alibabacloud/eci.toml`.
### Azure Container Instances Provider
@@ -177,28 +177,34 @@ co-exist with pods on regular worker nodes in the same Kubernetes cluster.
Easy instructions and a sample configuration file is available in the [AWS Fargate provider documentation](providers/aws/README.md).
### Hyper.sh Provider
### HashiCorp Nomad Provider
The Hyper.sh Provider allows Kubernetes clusters to deploy Hyper.sh containers
and manage both typical pods on VMs and Hyper.sh containers in the same
Kubernetes cluster.
HashiCorp [Nomad](https://nomadproject.io) provider for Virtual Kubelet connects your Kubernetes cluster
with Nomad cluster by exposing the Nomad cluster as a node in Kubernetes. By
using the provider, pods that are scheduled on the virtual Nomad node
registered on Kubernetes will run as jobs on Nomad clients as they
would on a Kubernetes node.
```bash
./bin/virtual-kubelet --provider hyper
./bin/virtual-kubelet --provider="nomad"
```
### Service Fabric Mesh Provider
For detailed instructions, follow the guide [here](providers/nomad/README.md).
The Service Fabric Mesh Provider allows you to deploy pods to Azure [Service Fabric Mesh](https://docs.microsoft.com/en-us/azure/service-fabric-mesh/service-fabric-mesh-overview).
### OpenStack Zun Provider
Service Fabric Mesh is a fully managed service that lets developers deploy microservices without managing the underlying infrastructure.
Pods deployed to Service Fabric Mesh will be assigned Public IPs from the Service Fabric Mesh network.
OpenStack [Zun](https://docs.openstack.org/zun/latest/) provider for Virtual Kubelet connects
your Kubernetes cluster with OpenStack in order to run Kubernetes pods on OpenStack Cloud.
Your pods on OpenStack have access to OpenStack tenant networks because they have Neutron ports
in your subnets. Each pod will have private IP addresses to connect to other OpenStack resources
(i.e. VMs) within your tenant, optionally have floating IP addresses to connect to the internet,
and bind-mount Cinder volumes into a path inside a pod's container.
```
./bin/virtual-kubelet --provider sfmesh --taint azure.com/sfmesh
```bash
./bin/virtual-kubelet --provider="openstack"
```
More detailed instructions can be found [here](providers/sfmesh/README.md).
For detailed instructions, follow the guide [here](providers/openstack/README.md).
### Adding a New Provider via the Provider Interface
@@ -209,8 +215,8 @@ Create a new directory for your provider under `providers` and implement the
following interface. Then add register your provider in
`providers/register/<provider_name>_provider.go`. Make sure to add a build tag so that
your provider can be excluded from being built. The format for this build tag
should be `no_<provider_name>_provider`. Also make sure your provdider has all
neccessary platform build tags, e.g. "linux" if your provider only compiles on Linux.
should be `no_<provider_name>_provider`. Also make sure your provider has all
necessary platform build tags, e.g. "linux" if your provider only compiles on Linux.
```go
// Provider contains the methods required to implement a virtual-kubelet provider.
@@ -339,7 +345,7 @@ Enable the ServiceNodeExclusion flag, by modifying the Controller Manager manife
Virtual Kubelet follows the [CNCF Code of Conduct](https://github.com/cncf/foundation/blob/master/code-of-conduct.md).
Sign the [CNCF CLA](https://github.com/kubernetes/community/blob/master/CLA.md) to be able to make Pull Requests to this repo.
Bi-weekly Virtual Kubelet Architecture meetings are held at 11am PST in this [zoom meeting room](https://zoom.us/j/245165908). Our virtual kubelet google calander has the architecture meetings listed and Tuesday & Thursday scrums for anyone interested. Check out the calander [here](https://calendar.google.com/calendar?cid=bjRtbGMxYWNtNXR0NXQ1a2hqZmRkNTRncGNAZ3JvdXAuY2FsZW5kYXIuZ29vZ2xlLmNvbQ).
Bi-weekly Virtual Kubelet Architecture meetings are held at 11am PST in this [zoom meeting room](https://zoom.us/j/245165908). Check out the calendar [here](https://calendar.google.com/calendar?cid=bjRtbGMxYWNtNXR0NXQ1a2hqZmRkNTRncGNAZ3JvdXAuY2FsZW5kYXIuZ29vZ2xlLmNvbQ).
Our google drive with design specifications and meeting notes are [here](https://drive.google.com/drive/folders/19Ndu11WBCCBDowo9CrrGUHoIfd2L8Ueg?usp=sharing).

14
charts/README.md Normal file
View File

@@ -0,0 +1,14 @@
# The Virtual Kubelet Helm chart
Each version of Virtual Kubelet has a dedicated [Helm](https://helm.sh) chart. Those charts are served as static assets directly from GitHub.
## The `index.yaml` file
This subdirectory has an `index.yaml` file, which is necessary for it to act as a Helm chart repository. To re-generate the `index.yaml` file (assuming that you have Helm installed):
```shell
cd /path/to/virtual-kubelet
helm repo index charts
```
The `index.yaml` then needs to be committed to Git and merged to `master`.

231
charts/index.yaml Normal file
View File

@@ -0,0 +1,231 @@
apiVersion: v1
entries:
virtual-kubelet:
- appVersion: "0.5"
created: 2019-01-28T11:18:23.622097-08:00
description: A Helm chart to install virtual kubelet inside a Kubernetes cluster.
digest: b4af3cc07e1a914b862ebcd05facbf155b16e098a138fcd7f5dc8ac715aabfa2
icon: https://avatars2.githubusercontent.com/u/34250142
maintainers:
- email: junjiez@microsoft.com
name: Robbie Zhang
name: virtual-kubelet
sources:
- https://github.com/virtual-kubelet/virtual-kubelet
urls:
- virtual-kubelet-0.5.0.tgz
version: 0.5.0
- appVersion: "0.5"
created: 2019-01-28T11:18:23.62762-08:00
description: A Helm chart to install virtual kubelet inside a Kubernetes cluster.
digest: b4af3cc07e1a914b862ebcd05facbf155b16e098a138fcd7f5dc8ac715aabfa2
icon: https://avatars2.githubusercontent.com/u/34250142
maintainers:
- email: junjiez@microsoft.com
name: Robbie Zhang
name: virtual-kubelet
sources:
- https://github.com/virtual-kubelet/virtual-kubelet
urls:
- virtual-kubelet-latest.tgz
version: 0.5.0
- appVersion: "0.4"
created: 2019-01-28T11:18:23.62156-08:00
description: A Helm chart to install virtual kubelet inside a Kubernetes cluster.
digest: ae4b8be9d69129f1002ea2228848ec790ea1280d9ff0f7dd99d0d6e3e13922f2
icon: https://avatars2.githubusercontent.com/u/34250142
maintainers:
- email: junjiez@microsoft.com
name: Robbie Zhang
name: virtual-kubelet
sources:
- https://github.com/virtual-kubelet/virtual-kubelet
urls:
- virtual-kubelet-0.4.0.tgz
version: 0.4.0
- appVersion: "0.3"
created: 2019-01-28T11:18:23.620905-08:00
description: A Helm chart to install virtual kubelet inside a Kubernetes cluster.
digest: f472f181c0724420d4f12218f8fc7f65d09fa02a8292f418d1537bf71231d643
icon: https://avatars2.githubusercontent.com/u/34250142
maintainers:
- email: junjiez@microsoft.com
name: Robbie Zhang
name: virtual-kubelet
sources:
- https://github.com/virtual-kubelet/virtual-kubelet
urls:
- virtual-kubelet-0.3.0.tgz
version: 0.3.0
- appVersion: "0.3"
created: 2019-01-28T11:18:23.6202-08:00
description: A Helm chart to install virtual kubelet inside a Kubernetes cluster.
digest: 5a2d0a269620ffe49498686161e853f49761ca4f905577fe1b8e552ab85fcaca
icon: https://avatars2.githubusercontent.com/u/34250142
maintainers:
- email: junjiez@microsoft.com
name: Robbie Zhang
name: virtual-kubelet
sources:
- https://github.com/virtual-kubelet/virtual-kubelet
urls:
- virtual-kubelet-0.2.0.tgz
version: 0.2.0
- created: 2019-01-28T11:18:23.619614-08:00
description: a Helm chart to install virtual kubelet inside a Kubernetes cluster.
digest: be2778949548cfb0cebe638cbe044d89045eb34ffc8d62180b86ff2f0f30b323
maintainers:
- email: junjiez@microsoft.com
name: Robbie Zhang
name: virtual-kubelet
sources:
- https://github.com/virtual-kubelet/virtual-kubelet
urls:
- virtual-kubelet-0.1.3.tgz
version: 0.1.3
- created: 2019-01-28T11:18:23.618974-08:00
description: a Helm chart to install virtual kubelet inside a Kubernetes cluster.
digest: b8519c66766d06a68671a2b8940cf44cccdf9321f144dead543f62d16325331e
maintainers:
- email: junjiez@microsoft.com
name: Robbie Zhang
name: virtual-kubelet
sources:
- https://github.com/virtual-kubelet/virtual-kubelet
urls:
- virtual-kubelet-0.1.2.tgz
version: 0.1.2
- created: 2019-01-28T11:18:23.618406-08:00
description: a Helm chart to install virtual kubelet inside a Kubernetes cluster.
digest: b15b3d5acde2cc264b5e8624c3bafcc249440c662ae353ef7012493eb0b6abf6
maintainers:
- email: junjiez@microsoft.com
name: Robbie Zhang
name: virtual-kubelet
sources:
- https://github.com/virtual-kubelet/virtual-kubelet
urls:
- virtual-kubelet-0.1.1.tgz
version: 0.1.1
- created: 2019-01-28T11:18:23.617865-08:00
description: a Helm chart to install virtual kubelet inside a Kubernetes cluster.
digest: 22c60ad2f7ea71abf58d65e52b91c2b9feca1ad6a15091321f7f12e5c43ecf81
maintainers:
- email: junjiez@microsoft.com
name: Robbie Zhang
name: virtual-kubelet
sources:
- https://github.com/virtual-kubelet/virtual-kubelet
urls:
- virtual-kubelet-0.1.0.tgz
version: 0.1.0
virtual-kubelet-for-aks:
- created: 2019-01-28T11:18:23.622632-08:00
description: a Helm chart to install virtual kubelet in an AKS or ACS cluster.
digest: 345cda5aeb537129e1ecc3d23c0cf47651454f672694a4402a7bbb5d3f3a91ba
maintainers:
- email: junjiez@microsoft.com
name: Robbie Zhang
name: virtual-kubelet-for-aks
sources:
- https://github.com/virtual-kubelet/virtual-kubelet
urls:
- virtual-kubelet-for-aks-0.1.10.tgz
version: 0.1.10
- created: 2019-01-28T11:18:23.627017-08:00
description: a Helm chart to install virtual kubelet in an AKS or ACS cluster.
digest: 345cda5aeb537129e1ecc3d23c0cf47651454f672694a4402a7bbb5d3f3a91ba
maintainers:
- email: junjiez@microsoft.com
name: Robbie Zhang
name: virtual-kubelet-for-aks
sources:
- https://github.com/virtual-kubelet/virtual-kubelet
urls:
- virtual-kubelet-for-aks-latest.tgz
version: 0.1.10
- created: 2019-01-28T11:18:23.626524-08:00
description: a Helm chart to install virtual kubelet in an AKS or ACS cluster.
digest: 4a30821657ff4ef4522060c9905e5659656c035126a7a9e650dd9bb54621b9eb
maintainers:
- email: junjiez@microsoft.com
name: Robbie Zhang
name: virtual-kubelet-for-aks
sources:
- https://github.com/virtual-kubelet/virtual-kubelet
urls:
- virtual-kubelet-for-aks-0.1.9.tgz
version: 0.1.9
- created: 2019-01-28T11:18:23.626081-08:00
description: a Helm chart to install virtual kubelet in an AKS or ACS cluster.
digest: 18d988bfb4d24b3674c6c690c0445b85cf3969471cfce74f7c49e87483cb497d
maintainers:
- email: junjiez@microsoft.com
name: Robbie Zhang
name: virtual-kubelet-for-aks
sources:
- https://github.com/virtual-kubelet/virtual-kubelet
urls:
- virtual-kubelet-for-aks-0.1.8.tgz
version: 0.1.8
- created: 2019-01-28T11:18:23.625442-08:00
description: a Helm chart to install virtual kubelet in an AKS or ACS cluster.
digest: 21c0719fe330981a170810ee4c3e84375106c176de08894827c8208a66f52112
maintainers:
- email: junjiez@microsoft.com
name: Robbie Zhang
name: virtual-kubelet-for-aks
sources:
- https://github.com/virtual-kubelet/virtual-kubelet
urls:
- virtual-kubelet-for-aks-0.1.7.tgz
version: 0.1.7
- created: 2019-01-28T11:18:23.624524-08:00
description: a Helm chart to install virtual kubelet in an AKS or ACS cluster.
digest: b5e72d03f04113a46350fa800135a67f5af9b4ffe3d6650bdeebd271b885e5aa
maintainers:
- email: junjiez@microsoft.com
name: Robbie Zhang
name: virtual-kubelet-for-aks
sources:
- https://github.com/virtual-kubelet/virtual-kubelet
urls:
- virtual-kubelet-for-aks-0.1.6.tgz
version: 0.1.6
- created: 2019-01-28T11:18:23.624071-08:00
description: a Helm chart to install virtual kubelet in an AKS or ACS cluster.
digest: 5b04abcc63dcc71b5ad25c8368ad1b114accf721e9c475c5a7f962a48fb0ed65
maintainers:
- email: junjiez@microsoft.com
name: Robbie Zhang
name: virtual-kubelet-for-aks
sources:
- https://github.com/virtual-kubelet/virtual-kubelet
urls:
- virtual-kubelet-for-aks-0.1.5.tgz
version: 0.1.5
- created: 2019-01-28T11:18:23.623547-08:00
description: a Helm chart to install virtual kubelet in an AKS or ACS cluster.
digest: b639126041dc4f3d0307f6a678d22021ba149f28612eb0bfc3903c75cf1ea414
maintainers:
- email: junjiez@microsoft.com
name: Robbie Zhang
name: virtual-kubelet-for-aks
sources:
- https://github.com/virtual-kubelet/virtual-kubelet
urls:
- virtual-kubelet-for-aks-0.1.4.tgz
version: 0.1.4
- created: 2019-01-28T11:18:23.623097-08:00
description: a Helm chart to install virtual kubelet in an AKS or ACS cluster.
digest: 1b428fd99c667482681c83b61753e7327b85ec2644ef4fa0e9366492a0e73cd7
maintainers:
- email: junjiez@microsoft.com
name: Robbie Zhang
name: virtual-kubelet-for-aks
sources:
- https://github.com/virtual-kubelet/virtual-kubelet
urls:
- virtual-kubelet-for-aks-0.1.3.tgz
version: 0.1.3
generated: 2019-01-28T11:18:23.615432-08:00

View File

@@ -14,6 +14,17 @@ spec:
checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }}
spec:
containers:
{{- if eq .Values.trace.exporter "jaeger" }}
{{- with .Values.traceExporters.jaeger }}
{{- if eq .endpoint "" }}
- name: {{ tpl .name $ }}
image: {{ .image.repository}}:{{.image.tag}}
imagePullPolicy: {{ .image.pullPolicy }}
ports:
- containerPort: 16686
{{- end }}
{{- end }}
{{- end }}
- name: {{ template "vk.fullname" . }}
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
imagePullPolicy: {{ .Values.image.pullPolicy }}
@@ -34,7 +45,7 @@ spec:
value: {{ tpl .Values.taint.value $ }}
- name: VKUBELET_TAINT_EFFECT
value: {{ .Values.taint.effect }}
{{- if eq .Values.provider "azure" }}
{{- if eq (required "You must specify a Virtual Kubelet provider" .Values.provider) "azure" }}
{{- with .Values.providers.azure }}
{{- if .loganalytics.enabled }}
- name: LOG_ANALYTICS_AUTH_LOCATION
@@ -88,11 +99,21 @@ spec:
value: {{ .masterUri }}
{{- end }}
{{- end }}
{{- end }}
{{- if eq .Values.trace.exporter "jaeger" }}
- name: JAEGER_ENDPOINT
{{- with .Values.traceExporters.jaeger }}
{{- if eq .endpoint "" }}
value: "http://127.0.0.1:14268"
{{- else }}
value: {{.endpoint}}
{{- end }}
{{- end }}
{{- end }}
volumeMounts:
- name: credentials
mountPath: "/etc/virtual-kubelet"
{{- if eq .Values.provider "azure" }}
{{- if eq (required "You must specify a Virtual Kubelet provider" .Values.provider) "azure" }}
{{- if .Values.providers.azure.targetAKS }}
- name: acs-credential
mountPath: "/etc/acs/azure.json"
@@ -103,19 +124,29 @@ spec:
{{- if not .Values.taint.enabled }}
"--disable-taint", "true",
{{- end }}
"--provider", "{{ required "provider is required" .Values.provider }}",
"--provider", "{{ required "You must specify a Virtual Kubelet provider" .Values.provider }}",
"--namespace", "{{ .Values.monitoredNamespace }}",
"--nodename", "{{ required "nodeName is required" .Values.nodeName }}",
{{- if .Values.logLevel }}
"--log-level", "{{.Values.logLevel}}",
{{- end }}
{{- if ne .Values.trace.exporter "" }}
"--trace-exporter", "{{ .Values.trace.exporter }}",
{{- if gt .Values.trace.sampleRate 0.0 }}
"--trace-sample-rate", "{{ .Values.trace.sampleRate }}",
{{- end }}
{{- $serviceName := tpl .Values.trace.serviceName $ }}
{{- if ne $serviceName "" }}
"--trace-service-name", "{{ $serviceName }}",
{{- end}}
{{- end}}
"--os", "{{ .Values.nodeOsType }}"
]
volumes:
- name: credentials
secret:
secretName: {{ template "vk.fullname" . }}
{{- if eq .Values.provider "azure" }}
{{- if eq (required "You must specify a Virtual Kubelet provider" .Values.provider) "azure" }}
{{- if .Values.providers.azure.targetAKS }}
- name: acs-credential
hostPath:

View File

@@ -17,7 +17,7 @@ data:
cert.pem: {{ quote .Values.apiserverCert }}
key.pem: {{ quote .Values.apiserverKey }}
{{- end }}
{{- if eq .Values.provider "azure" }}
{{- if eq (required "You must specify a Virtual Kubelet provider" .Values.provider) "azure" }}
{{- with .Values.providers.azure }}
{{- if .loganalytics.enabled }}
loganalytics.json: {{ printf "{\"workspaceID\": \"%s\",\"workspaceKey\": \"%s\"}" (required "workspaceId is required for loganalytics" .loganalytics.workspaceId ) (required "workspaceKey is required for loganalytics" .loganalytics.workspaceKey ) | b64enc | quote }}

View File

@@ -3,8 +3,6 @@ image:
tag: latest
pullPolicy: Always
## `provider` should be one of aws, azure, azurebatch, etc...
provider:
nodeName: "virtual-kubelet"
nodeOsType: "Linux"
monitoredNamespace: ""
@@ -19,6 +17,20 @@ taint:
## `effect` must be `NoSchedule`, `PreferNoSchedule` or `NoExecute`.
effect: NoSchedule
trace:
exporter: ""
serviceName: "{{ .Values.nodeName }}"
sampleRate: 0
traceExporters:
jaeger:
name: "{{ .Values.trace.exporter }}"
endpoint: ""
image:
repository: jaegertracing/all-in-one
tag: 1.8
pullPolicy: Always
providers:
azure:
## Set to true if deploying to Azure Kubernetes Service (AKS), otherwise false

View File

@@ -1,37 +0,0 @@
// +build !no_ocagent_exporter
package cmd
import (
"os"
"contrib.go.opencensus.io/exporter/ocagent"
"github.com/cpuguy83/strongerrors"
"github.com/pkg/errors"
"go.opencensus.io/trace"
)
func init() {
RegisterTracingExporter("ocagent", NewOCAgentExporter)
}
// NewOCAgentExporter creates a new opencensus tracing exporter using the opencensus agent forwarder.
func NewOCAgentExporter(opts TracingExporterOptions) (trace.Exporter, error) {
agentOpts := append([]ocagent.ExporterOption{}, ocagent.WithServiceName(opts.ServiceName))
if endpoint := os.Getenv("OCAGENT_ENDPOINT"); endpoint != "" {
agentOpts = append(agentOpts, ocagent.WithAddress(endpoint))
} else {
return nil, strongerrors.InvalidArgument(errors.New("must set endpoint address in OCAGENT_ENDPOINT"))
}
switch os.Getenv("OCAGENT_INSECURE") {
case "0", "no", "n", "off", "":
case "1", "yes", "y", "on":
agentOpts = append(agentOpts, ocagent.WithInsecure())
default:
return nil, strongerrors.InvalidArgument(errors.New("invalid value for OCAGENT_INSECURE"))
}
return ocagent.NewExporter(agentOpts...)
}

View File

@@ -1,35 +0,0 @@
package cmd
import (
"os"
"github.com/pkg/errors"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
)
func newClient(configPath string) (*kubernetes.Clientset, error) {
var config *rest.Config
// Check if the kubeConfig file exists.
if _, err := os.Stat(configPath); !os.IsNotExist(err) {
// Get the kubeconfig from the filepath.
config, err = clientcmd.BuildConfigFromFlags("", configPath)
if err != nil {
return nil, errors.Wrap(err, "error building client config")
}
} else {
// Set to in-cluster config.
config, err = rest.InClusterConfig()
if err != nil {
return nil, errors.Wrap(err, "error building in cluster config")
}
}
if masterURI := os.Getenv("MASTER_URI"); masterURI != "" {
config.Host = masterURI
}
return kubernetes.NewForConfig(config)
}

View File

@@ -1,386 +0,0 @@
// Copyright © 2017 The virtual-kubelet authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cmd
import (
"context"
"fmt"
"os"
"os/signal"
"path/filepath"
"strconv"
"strings"
"syscall"
"time"
"github.com/Sirupsen/logrus"
"github.com/cpuguy83/strongerrors"
"github.com/mitchellh/go-homedir"
"github.com/pkg/errors"
"github.com/spf13/cobra"
"github.com/spf13/viper"
"go.opencensus.io/trace"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
kubeinformers "k8s.io/client-go/informers"
corev1informers "k8s.io/client-go/informers/core/v1"
"k8s.io/client-go/kubernetes"
"github.com/virtual-kubelet/virtual-kubelet/log"
"github.com/virtual-kubelet/virtual-kubelet/manager"
"github.com/virtual-kubelet/virtual-kubelet/providers"
"github.com/virtual-kubelet/virtual-kubelet/providers/register"
"github.com/virtual-kubelet/virtual-kubelet/vkubelet"
)
const (
defaultDaemonPort = "10250"
// kubeSharedInformerFactoryDefaultResync is the default resync period used by the shared informer factories for Kubernetes resources.
// It is set to the same value used by the Kubelet, and can be overridden via the "--full-resync-period" flag.
// https://github.com/kubernetes/kubernetes/blob/v1.12.2/pkg/kubelet/apis/config/v1beta1/defaults.go#L51
kubeSharedInformerFactoryDefaultResync = 1 * time.Minute
)
var kubeletConfig string
var kubeConfig string
var kubeNamespace string
var nodeName string
var operatingSystem string
var provider string
var providerConfig string
var taintKey string
var disableTaint bool
var logLevel string
var metricsAddr string
var taint *corev1.Taint
var k8sClient *kubernetes.Clientset
var p providers.Provider
var rm *manager.ResourceManager
var apiConfig vkubelet.APIConfig
var podInformer corev1informers.PodInformer
var kubeSharedInformerFactoryResync time.Duration
var podSyncWorkers int
var userTraceExporters []string
var userTraceConfig = TracingExporterOptions{Tags: make(map[string]string)}
var traceSampler string
// Create a root context to be used by the pod controller and by the shared informer factories.
var rootContext, rootContextCancel = context.WithCancel(context.Background())
// RootCmd represents the base command when called without any subcommands
var RootCmd = &cobra.Command{
Use: "virtual-kubelet",
Short: "virtual-kubelet provides a virtual kubelet interface for your kubernetes cluster.",
Long: `virtual-kubelet implements the Kubelet interface with a pluggable
backend implementation allowing users to create kubernetes nodes without running the kubelet.
This allows users to schedule kubernetes workloads on nodes that aren't running Kubernetes.`,
Run: func(cmd *cobra.Command, args []string) {
defer rootContextCancel()
f, err := vkubelet.New(rootContext, vkubelet.Config{
Client: k8sClient,
Namespace: kubeNamespace,
NodeName: nodeName,
Taint: taint,
MetricsAddr: metricsAddr,
Provider: p,
ResourceManager: rm,
APIConfig: apiConfig,
PodSyncWorkers: podSyncWorkers,
PodInformer: podInformer,
})
if err != nil {
log.L.WithError(err).Fatal("Error initializing virtual kubelet")
}
sig := make(chan os.Signal, 1)
signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM)
go func() {
<-sig
rootContextCancel()
}()
if err := f.Run(rootContext); err != nil && errors.Cause(err) != context.Canceled {
log.L.Fatal(err)
}
},
}
// Execute adds all child commands to the root command and sets flags appropriately.
// This is called by main.main(). It only needs to happen once to the rootCmd.
func Execute() {
if err := RootCmd.Execute(); err != nil {
log.GetLogger(context.TODO()).WithError(err).Fatal("Error executing root command")
}
}
type mapVar map[string]string
func (mv mapVar) String() string {
var s string
for k, v := range mv {
if s == "" {
s = fmt.Sprintf("%s=%v", k, v)
} else {
s += fmt.Sprintf(", %s=%v", k, v)
}
}
return s
}
func (mv mapVar) Set(s string) error {
split := strings.SplitN(s, "=", 2)
if len(split) != 2 {
return errors.Errorf("invalid format, must be `key=value`: %s", s)
}
_, ok := mv[split[0]]
if ok {
return errors.Errorf("duplicate key: %s", split[0])
}
mv[split[0]] = split[1]
return nil
}
func (mv mapVar) Type() string {
return "map"
}
func init() {
cobra.OnInitialize(initConfig)
// read default node name from environment variable.
// it can be overwritten by cli flags if specified.
defaultNodeName := os.Getenv("DEFAULT_NODE_NAME")
if defaultNodeName == "" {
defaultNodeName = "virtual-kubelet"
}
// Here you will define your flags and configuration settings.
// Cobra supports persistent flags, which, if defined here,
// will be global for your application.
//RootCmd.PersistentFlags().StringVar(&kubeletConfig, "config", "", "config file (default is $HOME/.virtual-kubelet.yaml)")
RootCmd.PersistentFlags().StringVar(&kubeConfig, "kubeconfig", "", "config file (default is $HOME/.kube/config)")
RootCmd.PersistentFlags().StringVar(&kubeNamespace, "namespace", "", "kubernetes namespace (default is 'all')")
RootCmd.PersistentFlags().StringVar(&nodeName, "nodename", defaultNodeName, "kubernetes node name")
RootCmd.PersistentFlags().StringVar(&operatingSystem, "os", "Linux", "Operating System (Linux/Windows)")
RootCmd.PersistentFlags().StringVar(&provider, "provider", "", "cloud provider")
RootCmd.PersistentFlags().BoolVar(&disableTaint, "disable-taint", false, "disable the virtual-kubelet node taint")
RootCmd.PersistentFlags().StringVar(&providerConfig, "provider-config", "", "cloud provider configuration file")
RootCmd.PersistentFlags().StringVar(&metricsAddr, "metrics-addr", ":10255", "address to listen for metrics/stats requests")
RootCmd.PersistentFlags().StringVar(&taintKey, "taint", "", "Set node taint key")
RootCmd.PersistentFlags().MarkDeprecated("taint", "Taint key should now be configured using the VK_TAINT_KEY environment variable")
RootCmd.PersistentFlags().StringVar(&logLevel, "log-level", "info", `set the log level, e.g. "trace", debug", "info", "warn", "error"`)
RootCmd.PersistentFlags().IntVar(&podSyncWorkers, "pod-sync-workers", 10, `set the number of pod synchronization workers`)
RootCmd.PersistentFlags().StringSliceVar(&userTraceExporters, "trace-exporter", nil, fmt.Sprintf("sets the tracing exporter to use, available exporters: %s", AvailableTraceExporters()))
RootCmd.PersistentFlags().StringVar(&userTraceConfig.ServiceName, "trace-service-name", "virtual-kubelet", "sets the name of the service used to register with the trace exporter")
RootCmd.PersistentFlags().Var(mapVar(userTraceConfig.Tags), "trace-tag", "add tags to include with traces in key=value form")
RootCmd.PersistentFlags().StringVar(&traceSampler, "trace-sample-rate", "", "set probability of tracing samples")
RootCmd.PersistentFlags().DurationVar(&kubeSharedInformerFactoryResync, "full-resync-period", kubeSharedInformerFactoryDefaultResync, "how often to perform a full resync of pods between kubernetes and the provider")
// Cobra also supports local flags, which will only run
// when this action is called directly.
// RootCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
}
// initConfig reads in config file and ENV variables if set.
func initConfig() {
if provider == "" {
log.G(context.TODO()).Fatal("You must supply a cloud provider option: use --provider")
}
// Find home directory.
home, err := homedir.Dir()
if err != nil {
log.G(context.TODO()).WithError(err).Fatal("Error reading homedir")
}
if kubeletConfig != "" {
// Use config file from the flag.
viper.SetConfigFile(kubeletConfig)
} else {
// Search config in home directory with name ".virtual-kubelet" (without extension).
viper.AddConfigPath(home)
viper.SetConfigName(".virtual-kubelet")
}
viper.AutomaticEnv() // read in environment variables that match
// If a config file is found, read it in.
if err := viper.ReadInConfig(); err == nil {
log.G(context.TODO()).Debugf("Using config file %s", viper.ConfigFileUsed())
}
if kubeConfig == "" {
kubeConfig = filepath.Join(home, ".kube", "config")
}
if kubeNamespace == "" {
kubeNamespace = corev1.NamespaceAll
}
// Validate operating system.
ok, _ := providers.ValidOperatingSystems[operatingSystem]
if !ok {
log.G(context.TODO()).WithField("OperatingSystem", operatingSystem).Fatalf("Operating system not supported. Valid options are: %s", strings.Join(providers.ValidOperatingSystems.Names(), " | "))
}
level, err := log.ParseLevel(logLevel)
if err != nil {
log.G(context.TODO()).WithField("logLevel", logLevel).Fatal("log level is not supported")
}
logrus.SetLevel(level)
logger := log.L.WithFields(logrus.Fields{
"provider": provider,
"operatingSystem": operatingSystem,
"node": nodeName,
"namespace": kubeNamespace,
})
log.L = logger
if !disableTaint {
taint, err = getTaint(taintKey, provider)
if err != nil {
logger.WithError(err).Fatal("Error setting up desired kubernetes node taint")
}
}
k8sClient, err = newClient(kubeConfig)
if err != nil {
logger.WithError(err).Fatal("Error creating kubernetes client")
}
// Create a shared informer factory for Kubernetes pods in the current namespace (if specified) and scheduled to the current node.
podInformerFactory := kubeinformers.NewSharedInformerFactoryWithOptions(k8sClient, kubeSharedInformerFactoryResync, kubeinformers.WithNamespace(kubeNamespace), kubeinformers.WithTweakListOptions(func(options *metav1.ListOptions) {
options.FieldSelector = fields.OneTermEqualSelector("spec.nodeName", nodeName).String()
}))
// Create a pod informer so we can pass its lister to the resource manager.
podInformer = podInformerFactory.Core().V1().Pods()
// Create another shared informer factory for Kubernetes secrets and configmaps (not subject to any selectors).
scmInformerFactory := kubeinformers.NewSharedInformerFactoryWithOptions(k8sClient, kubeSharedInformerFactoryResync)
// Create a secret informer and a config map informer so we can pass their listers to the resource manager.
secretInformer := scmInformerFactory.Core().V1().Secrets()
configMapInformer := scmInformerFactory.Core().V1().ConfigMaps()
// Create a new instance of the resource manager that uses the listers above for pods, secrets and config maps.
rm, err = manager.NewResourceManager(podInformer.Lister(), secretInformer.Lister(), configMapInformer.Lister())
if err != nil {
logger.WithError(err).Fatal("Error initializing resource manager")
}
// Start the shared informer factory for pods.
go podInformerFactory.Start(rootContext.Done())
// Start the shared informer factory for secrets and configmaps.
go scmInformerFactory.Start(rootContext.Done())
daemonPortEnv := getEnv("KUBELET_PORT", defaultDaemonPort)
daemonPort, err := strconv.ParseInt(daemonPortEnv, 10, 32)
if err != nil {
logger.WithError(err).WithField("value", daemonPortEnv).Fatal("Invalid value from KUBELET_PORT in environment")
}
initConfig := register.InitConfig{
ConfigPath: providerConfig,
NodeName: nodeName,
OperatingSystem: operatingSystem,
ResourceManager: rm,
DaemonPort: int32(daemonPort),
InternalIP: os.Getenv("VKUBELET_POD_IP"),
}
p, err = register.GetProvider(provider, initConfig)
if err != nil {
logger.WithError(err).Fatal("Error initializing provider")
}
apiConfig, err = getAPIConfig()
if err != nil {
logger.WithError(err).Fatal("Error reading API config")
}
if podSyncWorkers <= 0 {
logger.Fatal("The number of pod synchronization workers should not be negative")
}
for k := range userTraceConfig.Tags {
if reservedTagNames[k] {
logger.WithField("tag", k).Fatal("must not use a reserved tag key")
}
}
userTraceConfig.Tags["operatingSystem"] = operatingSystem
userTraceConfig.Tags["provider"] = provider
userTraceConfig.Tags["nodeName"] = nodeName
for _, e := range userTraceExporters {
if e == "zpages" {
go setupZpages()
continue
}
exporter, err := GetTracingExporter(e, userTraceConfig)
if err != nil {
log.L.WithError(err).WithField("exporter", e).Fatal("Cannot initialize exporter")
}
trace.RegisterExporter(exporter)
}
if len(userTraceExporters) > 0 {
var s trace.Sampler
switch strings.ToLower(traceSampler) {
case "":
case "always":
s = trace.AlwaysSample()
case "never":
s = trace.NeverSample()
default:
rate, err := strconv.Atoi(traceSampler)
if err != nil {
logger.WithError(err).WithField("rate", traceSampler).Fatal("unsupported trace sample rate, supported values: always, never, or number 0-100")
}
if rate < 0 || rate > 100 {
logger.WithField("rate", traceSampler).Fatal("trace sample rate must not be less than zero or greater than 100")
}
s = trace.ProbabilitySampler(float64(rate) / 100)
}
if s != nil {
trace.ApplyConfig(
trace.Config{
DefaultSampler: s,
},
)
}
}
}
func getAPIConfig() (vkubelet.APIConfig, error) {
config := vkubelet.APIConfig{
CertPath: os.Getenv("APISERVER_CERT_LOCATION"),
KeyPath: os.Getenv("APISERVER_KEY_LOCATION"),
}
port, err := strconv.Atoi(os.Getenv("KUBELET_PORT"))
if err != nil {
return vkubelet.APIConfig{}, strongerrors.InvalidArgument(errors.Wrap(err, "error parsing KUBELET_PORT variable"))
}
config.Addr = fmt.Sprintf(":%d", port)
return config, nil
}

View File

@@ -1,56 +0,0 @@
package cmd
import (
"os"
"github.com/cpuguy83/strongerrors"
"github.com/pkg/errors"
corev1 "k8s.io/api/core/v1"
)
// Default taint values
const (
DefaultTaintEffect = corev1.TaintEffectNoSchedule
DefaultTaintKey = "virtual-kubelet.io/provider"
)
func getEnv(key, defaultValue string) string {
value, found := os.LookupEnv(key)
if found {
return value
}
return defaultValue
}
// getTaint creates a taint using the provided key/value.
// Taint effect is read from the environment
// The taint key/value may be overwritten by the environment.
func getTaint(key, value string) (*corev1.Taint, error) {
if key == "" {
key = DefaultTaintKey
value = provider
}
key = getEnv("VKUBELET_TAINT_KEY", key)
value = getEnv("VKUBELET_TAINT_VALUE", value)
effectEnv := getEnv("VKUBELET_TAINT_EFFECT", string(DefaultTaintEffect))
var effect corev1.TaintEffect
switch effectEnv {
case "NoSchedule":
effect = corev1.TaintEffectNoSchedule
case "NoExecute":
effect = corev1.TaintEffectNoExecute
case "PreferNoSchedule":
effect = corev1.TaintEffectPreferNoSchedule
default:
return nil, strongerrors.InvalidArgument(errors.Errorf("taint effect %q is not supported", effectEnv))
}
return &corev1.Taint{
Key: key,
Value: value,
Effect: effect,
}, nil
}

View File

@@ -1,46 +0,0 @@
// Copyright © 2017 NAME HERE <EMAIL ADDRESS>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cmd
import (
"fmt"
"github.com/spf13/cobra"
"github.com/virtual-kubelet/virtual-kubelet/version"
)
// versionCmd represents the version command
var versionCmd = &cobra.Command{
Use: "version",
Short: "Show the version of the program",
Long: `Show the version of the program`,
Run: func(cmd *cobra.Command, args []string) {
fmt.Printf("Version: %s, Built: %s", version.Version, version.BuildTime)
},
}
func init() {
RootCmd.AddCommand(versionCmd)
// Here you will define your flags and configuration settings.
// Cobra supports Persistent Flags which will work for this command
// and all subcommands, e.g.:
// versionCmd.PersistentFlags().String("foo", "", "A help for foo")
// Cobra supports local flags which will only run when this command
// is called directly, e.g.:
// versionCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
}

View File

@@ -0,0 +1,53 @@
// Copyright © 2017 The virtual-kubelet authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package providers
import (
"fmt"
"os"
"github.com/spf13/cobra"
"github.com/virtual-kubelet/virtual-kubelet/providers"
)
// NewCommand creates a new providers subcommand
// This subcommand is used to determine which providers are registered.
func NewCommand(s *providers.Store) *cobra.Command {
return &cobra.Command{
Use: "providers",
Short: "Show the list of supported providers",
Long: "Show the list of supported providers",
Args: cobra.MaximumNArgs(2),
Run: func(cmd *cobra.Command, args []string) {
switch len(args) {
case 0:
for _, p := range s.List() {
fmt.Fprintln(cmd.OutOrStdout(), p)
}
case 1:
if !s.Exists(args[0]) {
fmt.Fprintln(cmd.OutOrStderr(), "no such provider", args[0])
// TODO(@cpuuy83): would be nice to not short-circuit the exit here
// But at the momemt this seems to be the only way to exit non-zero and
// handle our own error output
os.Exit(1)
}
fmt.Fprintln(cmd.OutOrStdout(), args[0])
}
return
},
}
}

View File

@@ -0,0 +1,98 @@
// Copyright © 2017 The virtual-kubelet authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package root
import (
"flag"
"fmt"
"os"
"strings"
"github.com/pkg/errors"
"github.com/spf13/pflag"
"k8s.io/klog"
)
type mapVar map[string]string
func (mv mapVar) String() string {
var s string
for k, v := range mv {
if s == "" {
s = fmt.Sprintf("%s=%v", k, v)
} else {
s += fmt.Sprintf(", %s=%v", k, v)
}
}
return s
}
func (mv mapVar) Set(s string) error {
split := strings.SplitN(s, "=", 2)
if len(split) != 2 {
return errors.Errorf("invalid format, must be `key=value`: %s", s)
}
_, ok := mv[split[0]]
if ok {
return errors.Errorf("duplicate key: %s", split[0])
}
mv[split[0]] = split[1]
return nil
}
func (mv mapVar) Type() string {
return "map"
}
func installFlags(flags *pflag.FlagSet, c *Opts) {
flags.StringVar(&c.KubeConfigPath, "kubeconfig", c.KubeConfigPath, "kube config file to use for connecting to the Kubernetes API server")
flags.StringVar(&c.KubeNamespace, "namespace", c.KubeNamespace, "kubernetes namespace (default is 'all')")
flags.StringVar(&c.NodeName, "nodename", c.NodeName, "kubernetes node name")
flags.StringVar(&c.OperatingSystem, "os", c.OperatingSystem, "Operating System (Linux/Windows)")
flags.StringVar(&c.Provider, "provider", c.Provider, "cloud provider")
flags.StringVar(&c.ProviderConfigPath, "provider-config", c.ProviderConfigPath, "cloud provider configuration file")
flags.StringVar(&c.MetricsAddr, "metrics-addr", c.MetricsAddr, "address to listen for metrics/stats requests")
flags.StringVar(&c.TaintKey, "taint", c.TaintKey, "Set node taint key")
flags.BoolVar(&c.DisableTaint, "disable-taint", c.DisableTaint, "disable the virtual-kubelet node taint")
flags.MarkDeprecated("taint", "Taint key should now be configured using the VK_TAINT_KEY environment variable")
flags.IntVar(&c.PodSyncWorkers, "pod-sync-workers", c.PodSyncWorkers, `set the number of pod synchronization workers`)
flags.BoolVar(&c.EnableNodeLease, "enable-node-lease", c.EnableNodeLease, `use node leases (1.13) for node heartbeats`)
flags.StringSliceVar(&c.TraceExporters, "trace-exporter", c.TraceExporters, fmt.Sprintf("sets the tracing exporter to use, available exporters: %s", AvailableTraceExporters()))
flags.StringVar(&c.TraceConfig.ServiceName, "trace-service-name", c.TraceConfig.ServiceName, "sets the name of the service used to register with the trace exporter")
flags.Var(mapVar(c.TraceConfig.Tags), "trace-tag", "add tags to include with traces in key=value form")
flags.StringVar(&c.TraceSampleRate, "trace-sample-rate", c.TraceSampleRate, "set probability of tracing samples")
flags.DurationVar(&c.InformerResyncPeriod, "full-resync-period", c.InformerResyncPeriod, "how often to perform a full resync of pods between kubernetes and the provider")
flags.DurationVar(&c.StartupTimeout, "startup-timeout", c.StartupTimeout, "How long to wait for the virtual-kubelet to start")
flagset := flag.NewFlagSet("klog", flag.PanicOnError)
klog.InitFlags(flagset)
flagset.VisitAll(func(f *flag.Flag) {
f.Name = "klog." + f.Name
flags.AddGoFlag(f)
})
}
func getEnv(key, defaultValue string) string {
value, found := os.LookupEnv(key)
if found {
return value
}
return defaultValue
}

View File

@@ -0,0 +1,161 @@
// Copyright © 2017 The virtual-kubelet authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package root
import (
"context"
"crypto/tls"
"fmt"
"io"
"net"
"net/http"
"os"
"github.com/pkg/errors"
"github.com/virtual-kubelet/virtual-kubelet/log"
"github.com/virtual-kubelet/virtual-kubelet/node/api"
"github.com/virtual-kubelet/virtual-kubelet/providers"
)
// AcceptedCiphers is the list of accepted TLS ciphers, with known weak ciphers elided
// Note this list should be a moving target.
var AcceptedCiphers = []uint16{
tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
}
func loadTLSConfig(certPath, keyPath string) (*tls.Config, error) {
cert, err := tls.LoadX509KeyPair(certPath, keyPath)
if err != nil {
return nil, errors.Wrap(err, "error loading tls certs")
}
return &tls.Config{
Certificates: []tls.Certificate{cert},
MinVersion: tls.VersionTLS12,
PreferServerCipherSuites: true,
CipherSuites: AcceptedCiphers,
}, nil
}
func setupHTTPServer(ctx context.Context, p providers.Provider, cfg *apiServerConfig) (_ func(), retErr error) {
var closers []io.Closer
cancel := func() {
for _, c := range closers {
c.Close()
}
}
defer func() {
if retErr != nil {
cancel()
}
}()
if cfg.CertPath == "" || cfg.KeyPath == "" {
log.G(ctx).
WithField("certPath", cfg.CertPath).
WithField("keyPath", cfg.KeyPath).
Error("TLS certificates not provided, not setting up pod http server")
} else {
tlsCfg, err := loadTLSConfig(cfg.CertPath, cfg.KeyPath)
if err != nil {
return nil, err
}
l, err := tls.Listen("tcp", cfg.Addr, tlsCfg)
if err != nil {
return nil, errors.Wrap(err, "error setting up listener for pod http server")
}
mux := http.NewServeMux()
podRoutes := api.PodHandlerConfig{
RunInContainer: p.RunInContainer,
GetContainerLogs: p.GetContainerLogs,
GetPods: p.GetPods,
}
api.AttachPodRoutes(podRoutes, mux, true)
s := &http.Server{
Handler: mux,
TLSConfig: tlsCfg,
}
go serveHTTP(ctx, s, l, "pods")
closers = append(closers, s)
}
if cfg.MetricsAddr == "" {
log.G(ctx).Info("Pod metrics server not setup due to empty metrics address")
} else {
l, err := net.Listen("tcp", cfg.MetricsAddr)
if err != nil {
return nil, errors.Wrap(err, "could not setup listener for pod metrics http server")
}
mux := http.NewServeMux()
var summaryHandlerFunc api.PodStatsSummaryHandlerFunc
if mp, ok := p.(providers.PodMetricsProvider); ok {
summaryHandlerFunc = mp.GetStatsSummary
}
podMetricsRoutes := api.PodMetricsConfig{
GetStatsSummary: summaryHandlerFunc,
}
api.AttachPodMetricsRoutes(podMetricsRoutes, mux)
s := &http.Server{
Handler: mux,
}
go serveHTTP(ctx, s, l, "pod metrics")
closers = append(closers, s)
}
return cancel, nil
}
func serveHTTP(ctx context.Context, s *http.Server, l net.Listener, name string) {
if err := s.Serve(l); err != nil {
select {
case <-ctx.Done():
default:
log.G(ctx).WithError(err).Errorf("Error setting up %s http server", name)
}
}
l.Close()
}
type apiServerConfig struct {
CertPath string
KeyPath string
Addr string
MetricsAddr string
}
func getAPIConfig(c Opts) (*apiServerConfig, error) {
config := apiServerConfig{
CertPath: os.Getenv("APISERVER_CERT_LOCATION"),
KeyPath: os.Getenv("APISERVER_KEY_LOCATION"),
}
config.Addr = fmt.Sprintf(":%d", c.ListenPort)
config.MetricsAddr = c.MetricsAddr
return &config, nil
}

View File

@@ -0,0 +1,103 @@
// Copyright © 2017 The virtual-kubelet authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package root
import (
"context"
"strings"
"github.com/virtual-kubelet/virtual-kubelet/errdefs"
"github.com/virtual-kubelet/virtual-kubelet/providers"
corev1 "k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
// NodeFromProvider builds a kubernetes node object from a provider
// This is a temporary solution until node stuff actually split off from the provider interface itself.
func NodeFromProvider(ctx context.Context, name string, taint *v1.Taint, p providers.Provider, version string) *v1.Node {
taints := make([]v1.Taint, 0)
if taint != nil {
taints = append(taints, *taint)
}
node := &v1.Node{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Labels: map[string]string{
"type": "virtual-kubelet",
"kubernetes.io/role": "agent",
"beta.kubernetes.io/os": strings.ToLower(p.OperatingSystem()),
"kubernetes.io/hostname": name,
"alpha.service-controller.kubernetes.io/exclude-balancer": "true",
},
},
Spec: v1.NodeSpec{
Taints: taints,
},
Status: v1.NodeStatus{
NodeInfo: v1.NodeSystemInfo{
OperatingSystem: p.OperatingSystem(),
Architecture: "amd64",
KubeletVersion: version,
},
Capacity: p.Capacity(ctx),
Allocatable: p.Capacity(ctx),
Conditions: p.NodeConditions(ctx),
Addresses: p.NodeAddresses(ctx),
DaemonEndpoints: *p.NodeDaemonEndpoints(ctx),
},
}
return node
}
// getTaint creates a taint using the provided key/value.
// Taint effect is read from the environment
// The taint key/value may be overwritten by the environment.
func getTaint(c Opts) (*corev1.Taint, error) {
value := c.Provider
key := c.TaintKey
if key == "" {
key = DefaultTaintKey
}
if c.TaintEffect == "" {
c.TaintEffect = DefaultTaintEffect
}
key = getEnv("VKUBELET_TAINT_KEY", key)
value = getEnv("VKUBELET_TAINT_VALUE", value)
effectEnv := getEnv("VKUBELET_TAINT_EFFECT", string(c.TaintEffect))
var effect corev1.TaintEffect
switch effectEnv {
case "NoSchedule":
effect = corev1.TaintEffectNoSchedule
case "NoExecute":
effect = corev1.TaintEffectNoExecute
case "PreferNoSchedule":
effect = corev1.TaintEffectPreferNoSchedule
default:
return nil, errdefs.InvalidInputf("taint effect %q is not supported", effectEnv)
}
return &corev1.Taint{
Key: key,
Value: value,
Effect: effect,
}, nil
}

View File

@@ -0,0 +1,148 @@
// Copyright © 2017 The virtual-kubelet authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package root
import (
"os"
"path/filepath"
"strconv"
"time"
"github.com/mitchellh/go-homedir"
"github.com/pkg/errors"
corev1 "k8s.io/api/core/v1"
)
// Defaults for root command options
const (
DefaultNodeName = "virtual-kubelet"
DefaultOperatingSystem = "Linux"
DefaultInformerResyncPeriod = 1 * time.Minute
DefaultMetricsAddr = ":10255"
DefaultListenPort = 10250 // TODO(cpuguy83)(VK1.0): Change this to an addr instead of just a port.. we should not be listening on all interfaces.
DefaultPodSyncWorkers = 10
DefaultKubeNamespace = corev1.NamespaceAll
DefaultTaintEffect = string(corev1.TaintEffectNoSchedule)
DefaultTaintKey = "virtual-kubelet.io/provider"
)
// Opts stores all the options for configuring the root virtual-kubelet command.
// It is used for setting flag values.
//
// You can set the default options by creating a new `Opts` struct and passing
// it into `SetDefaultOpts`
type Opts struct {
// Path to the kubeconfig to use to connect to the Kubernetes API server.
KubeConfigPath string
// Namespace to watch for pods and other resources
KubeNamespace string
// Sets the port to listen for requests from the Kubernetes API server
ListenPort int32
// Node name to use when creating a node in Kubernetes
NodeName string
// Operating system to run pods for
OperatingSystem string
Provider string
ProviderConfigPath string
TaintKey string
TaintEffect string
DisableTaint bool
MetricsAddr string
// Number of workers to use to handle pod notifications
PodSyncWorkers int
InformerResyncPeriod time.Duration
// Use node leases when supported by Kubernetes (instead of node status updates)
EnableNodeLease bool
TraceExporters []string
TraceSampleRate string
TraceConfig TracingExporterOptions
// Startup Timeout is how long to wait for the kubelet to start
StartupTimeout time.Duration
Version string
}
// SetDefaultOpts sets default options for unset values on the passed in option struct.
// Fields tht are already set will not be modified.
func SetDefaultOpts(c *Opts) error {
if c.OperatingSystem == "" {
c.OperatingSystem = DefaultOperatingSystem
}
if c.NodeName == "" {
c.NodeName = getEnv("DEFAULT_NODE_NAME", DefaultNodeName)
}
if c.InformerResyncPeriod == 0 {
c.InformerResyncPeriod = DefaultInformerResyncPeriod
}
if c.MetricsAddr == "" {
c.MetricsAddr = DefaultMetricsAddr
}
if c.PodSyncWorkers == 0 {
c.PodSyncWorkers = DefaultPodSyncWorkers
}
if c.TraceConfig.ServiceName == "" {
c.TraceConfig.ServiceName = DefaultNodeName
}
if c.ListenPort == 0 {
if kp := os.Getenv("KUBELET_PORT"); kp != "" {
p, err := strconv.Atoi(kp)
if err != nil {
return errors.Wrap(err, "error parsing KUBELET_PORT environment variable")
}
c.ListenPort = int32(p)
} else {
c.ListenPort = DefaultListenPort
}
}
if c.KubeNamespace == "" {
c.KubeNamespace = DefaultKubeNamespace
}
if c.TaintKey == "" {
c.TaintKey = DefaultTaintKey
}
if c.TaintEffect == "" {
c.TaintEffect = DefaultTaintEffect
}
if c.KubeConfigPath == "" {
c.KubeConfigPath = os.Getenv("KUBECONFIG")
if c.KubeConfigPath == "" {
home, _ := homedir.Dir()
if home != "" {
c.KubeConfigPath = filepath.Join(home, ".kube", "config")
}
}
}
return nil
}

View File

@@ -0,0 +1,268 @@
// Copyright © 2017 The virtual-kubelet authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package root
import (
"context"
"os"
"path"
"time"
"github.com/pkg/errors"
"github.com/spf13/cobra"
"github.com/virtual-kubelet/virtual-kubelet/errdefs"
"github.com/virtual-kubelet/virtual-kubelet/log"
"github.com/virtual-kubelet/virtual-kubelet/manager"
"github.com/virtual-kubelet/virtual-kubelet/node"
"github.com/virtual-kubelet/virtual-kubelet/providers"
corev1 "k8s.io/api/core/v1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
kubeinformers "k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/kubernetes/typed/coordination/v1beta1"
corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/client-go/tools/record"
)
// NewCommand creates a new top-level command.
// This command is used to start the virtual-kubelet daemon
func NewCommand(ctx context.Context, name string, s *providers.Store, c Opts) *cobra.Command {
cmd := &cobra.Command{
Use: name,
Short: name + " provides a virtual kubelet interface for your kubernetes cluster.",
Long: name + ` implements the Kubelet interface with a pluggable
backend implementation allowing users to create kubernetes nodes without running the kubelet.
This allows users to schedule kubernetes workloads on nodes that aren't running Kubernetes.`,
RunE: func(cmd *cobra.Command, args []string) error {
return runRootCommand(ctx, s, c)
},
}
installFlags(cmd.Flags(), &c)
return cmd
}
func runRootCommand(ctx context.Context, s *providers.Store, c Opts) error {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
if ok := providers.ValidOperatingSystems[c.OperatingSystem]; !ok {
return errdefs.InvalidInputf("operating system %q is not supported", c.OperatingSystem)
}
if c.PodSyncWorkers == 0 {
return errdefs.InvalidInput("pod sync workers must be greater than 0")
}
var taint *corev1.Taint
if !c.DisableTaint {
var err error
taint, err = getTaint(c)
if err != nil {
return err
}
}
client, err := newClient(c.KubeConfigPath)
if err != nil {
return err
}
// Create a shared informer factory for Kubernetes pods in the current namespace (if specified) and scheduled to the current node.
podInformerFactory := kubeinformers.NewSharedInformerFactoryWithOptions(
client,
c.InformerResyncPeriod,
kubeinformers.WithNamespace(c.KubeNamespace),
kubeinformers.WithTweakListOptions(func(options *metav1.ListOptions) {
options.FieldSelector = fields.OneTermEqualSelector("spec.nodeName", c.NodeName).String()
}))
podInformer := podInformerFactory.Core().V1().Pods()
// Create another shared informer factory for Kubernetes secrets and configmaps (not subject to any selectors).
scmInformerFactory := kubeinformers.NewSharedInformerFactoryWithOptions(client, c.InformerResyncPeriod)
// Create a secret informer and a config map informer so we can pass their listers to the resource manager.
secretInformer := scmInformerFactory.Core().V1().Secrets()
configMapInformer := scmInformerFactory.Core().V1().ConfigMaps()
serviceInformer := scmInformerFactory.Core().V1().Services()
go podInformerFactory.Start(ctx.Done())
go scmInformerFactory.Start(ctx.Done())
rm, err := manager.NewResourceManager(podInformer.Lister(), secretInformer.Lister(), configMapInformer.Lister(), serviceInformer.Lister())
if err != nil {
return errors.Wrap(err, "could not create resource manager")
}
apiConfig, err := getAPIConfig(c)
if err != nil {
return err
}
if err := setupTracing(ctx, c); err != nil {
return err
}
initConfig := providers.InitConfig{
ConfigPath: c.ProviderConfigPath,
NodeName: c.NodeName,
OperatingSystem: c.OperatingSystem,
ResourceManager: rm,
DaemonPort: int32(c.ListenPort),
InternalIP: os.Getenv("VKUBELET_POD_IP"),
}
pInit := s.Get(c.Provider)
if pInit == nil {
return errors.Errorf("provider %q not found", c.Provider)
}
p, err := pInit(initConfig)
if err != nil {
return errors.Wrapf(err, "error initializing provider %s", c.Provider)
}
ctx = log.WithLogger(ctx, log.G(ctx).WithFields(log.Fields{
"provider": c.Provider,
"operatingSystem": c.OperatingSystem,
"node": c.NodeName,
"watchedNamespace": c.KubeNamespace,
}))
var leaseClient v1beta1.LeaseInterface
if c.EnableNodeLease {
leaseClient = client.CoordinationV1beta1().Leases(corev1.NamespaceNodeLease)
}
pNode := NodeFromProvider(ctx, c.NodeName, taint, p, c.Version)
nodeRunner, err := node.NewNodeController(
node.NaiveNodeProvider{},
pNode,
client.CoreV1().Nodes(),
node.WithNodeEnableLeaseV1Beta1(leaseClient, nil),
node.WithNodeStatusUpdateErrorHandler(func(ctx context.Context, err error) error {
if !k8serrors.IsNotFound(err) {
return err
}
log.G(ctx).Debug("node not found")
newNode := pNode.DeepCopy()
newNode.ResourceVersion = ""
_, err = client.CoreV1().Nodes().Create(newNode)
if err != nil {
return err
}
log.G(ctx).Debug("created new node")
return nil
}),
)
if err != nil {
log.G(ctx).Fatal(err)
}
eb := record.NewBroadcaster()
eb.StartLogging(log.G(ctx).Infof)
eb.StartRecordingToSink(&corev1client.EventSinkImpl{Interface: client.CoreV1().Events(c.KubeNamespace)})
pc, err := node.NewPodController(node.PodControllerConfig{
PodClient: client.CoreV1(),
PodInformer: podInformer,
EventRecorder: eb.NewRecorder(scheme.Scheme, corev1.EventSource{Component: path.Join(pNode.Name, "pod-controller")}),
Provider: p,
SecretInformer: secretInformer,
ConfigMapInformer: configMapInformer,
ServiceInformer: serviceInformer,
})
if err != nil {
return errors.Wrap(err, "error setting up pod controller")
}
cancelHTTP, err := setupHTTPServer(ctx, p, apiConfig)
if err != nil {
return err
}
defer cancelHTTP()
go func() {
if err := pc.Run(ctx, c.PodSyncWorkers); err != nil && errors.Cause(err) != context.Canceled {
log.G(ctx).Fatal(err)
}
}()
if c.StartupTimeout > 0 {
// If there is a startup timeout, it does two things:
// 1. It causes the VK to shutdown if we haven't gotten into an operational state in a time period
// 2. It prevents node advertisement from happening until we're in an operational state
err = waitFor(ctx, c.StartupTimeout, pc.Ready())
if err != nil {
return err
}
}
go func() {
if err := nodeRunner.Run(ctx); err != nil {
log.G(ctx).Fatal(err)
}
}()
log.G(ctx).Info("Initialized")
<-ctx.Done()
return nil
}
func waitFor(ctx context.Context, time time.Duration, ready <-chan struct{}) error {
ctx, cancel := context.WithTimeout(ctx, time)
defer cancel()
// Wait for the VK / PC close the the ready channel, or time out and return
log.G(ctx).Info("Waiting for pod controller / VK to be ready")
select {
case <-ready:
return nil
case <-ctx.Done():
return errors.Wrap(ctx.Err(), "Error while starting up VK")
}
}
func newClient(configPath string) (*kubernetes.Clientset, error) {
var config *rest.Config
// Check if the kubeConfig file exists.
if _, err := os.Stat(configPath); !os.IsNotExist(err) {
// Get the kubeconfig from the filepath.
config, err = clientcmd.BuildConfigFromFlags("", configPath)
if err != nil {
return nil, errors.Wrap(err, "error building client config")
}
} else {
// Set to in-cluster config.
config, err = rest.InClusterConfig()
if err != nil {
return nil, errors.Wrap(err, "error building in cluster config")
}
}
if masterURI := os.Getenv("MASTER_URI"); masterURI != "" {
config.Host = masterURI
}
return kubernetes.NewForConfig(config)
}

View File

@@ -0,0 +1,114 @@
// Copyright © 2017 The virtual-kubelet authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package root
import (
"context"
"net"
"net/http"
"os"
"strconv"
"strings"
"github.com/pkg/errors"
"github.com/virtual-kubelet/virtual-kubelet/errdefs"
"github.com/virtual-kubelet/virtual-kubelet/log"
octrace "go.opencensus.io/trace"
"go.opencensus.io/zpages"
)
var (
reservedTagNames = map[string]bool{
"operatingSystem": true,
"provider": true,
"nodeName": true,
}
)
func setupTracing(ctx context.Context, c Opts) error {
for k := range c.TraceConfig.Tags {
if reservedTagNames[k] {
return errdefs.InvalidInputf("invalid trace tag %q, must not use a reserved tag key", k)
}
}
if c.TraceConfig.Tags == nil {
c.TraceConfig.Tags = make(map[string]string, 3)
}
c.TraceConfig.Tags["operatingSystem"] = c.OperatingSystem
c.TraceConfig.Tags["provider"] = c.Provider
c.TraceConfig.Tags["nodeName"] = c.NodeName
for _, e := range c.TraceExporters {
if e == "zpages" {
setupZpages(ctx)
continue
}
exporter, err := GetTracingExporter(e, c.TraceConfig)
if err != nil {
return err
}
octrace.RegisterExporter(exporter)
}
if len(c.TraceExporters) > 0 {
var s octrace.Sampler
switch strings.ToLower(c.TraceSampleRate) {
case "":
case "always":
s = octrace.AlwaysSample()
case "never":
s = octrace.NeverSample()
default:
rate, err := strconv.Atoi(c.TraceSampleRate)
if err != nil {
return errdefs.AsInvalidInput(errors.Wrap(err, "unsupported trace sample rate"))
}
if rate < 0 || rate > 100 {
return errdefs.AsInvalidInput(errors.Wrap(err, "trace sample rate must be between 0 and 100"))
}
s = octrace.ProbabilitySampler(float64(rate) / 100)
}
if s != nil {
octrace.ApplyConfig(
octrace.Config{
DefaultSampler: s,
},
)
}
}
return nil
}
func setupZpages(ctx context.Context) {
p := os.Getenv("ZPAGES_PORT")
if p == "" {
log.G(ctx).Error("Missing ZPAGES_PORT env var, cannot setup zpages endpoint")
}
listener, err := net.Listen("tcp", p)
if err != nil {
log.G(ctx).WithError(err).Error("Cannot bind to ZPAGES PORT, cannot setup listener")
return
}
mux := http.NewServeMux()
zpages.Handle(mux, "/debug")
go func() {
// This should never terminate, if it does, it will always terminate with an error
e := http.Serve(listener, mux)
if e == http.ErrServerClosed {
return
}
log.G(ctx).WithError(e).Error("Zpages server exited")
}()
}

View File

@@ -1,33 +1,33 @@
package cmd
// Copyright © 2017 The virtual-kubelet authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package root
import (
"context"
"net/http"
"os"
"github.com/cpuguy83/strongerrors"
"github.com/pkg/errors"
"github.com/virtual-kubelet/virtual-kubelet/log"
"github.com/virtual-kubelet/virtual-kubelet/errdefs"
"go.opencensus.io/trace"
"go.opencensus.io/zpages"
)
var (
tracingExporters = make(map[string]TracingExporterInitFunc)
reservedTagNames = map[string]bool{
"operatingSystem": true,
"provider": true,
"nodeName": true,
}
)
// TracingExporterOptions is used to pass options to the configured tracer
type TracingExporterOptions struct {
Tags map[string]string
ServiceName string
}
var (
tracingExporters = make(map[string]TracingExporterInitFunc)
)
// TracingExporterInitFunc is the function that is called to initialize an exporter.
// This is used when registering an exporter and called when a user specifed they want to use the exporter.
type TracingExporterInitFunc func(TracingExporterOptions) (trace.Exporter, error)
@@ -43,7 +43,7 @@ func RegisterTracingExporter(name string, f TracingExporterInitFunc) {
func GetTracingExporter(name string, opts TracingExporterOptions) (trace.Exporter, error) {
f, ok := tracingExporters[name]
if !ok {
return nil, strongerrors.NotFound(errors.Errorf("tracing exporter %q not found", name))
return nil, errdefs.NotFoundf("tracing exporter %q not found", name)
}
return f(opts)
}
@@ -56,14 +56,3 @@ func AvailableTraceExporters() []string {
}
return out
}
func setupZpages() {
ctx := context.TODO()
p := os.Getenv("ZPAGES_PORT")
if p == "" {
log.G(ctx).Error("Missing ZPAGES_PORT env var, cannot setup zpages endpoint")
}
mux := http.NewServeMux()
zpages.Handle(mux, "/debug")
http.ListenAndServe(p, mux)
}

View File

@@ -1,6 +1,20 @@
// Copyright © 2017 The virtual-kubelet authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build !no_jaeger_exporter
package cmd
package root
import (
"errors"

View File

@@ -0,0 +1,50 @@
// Copyright © 2017 The virtual-kubelet authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build !no_ocagent_exporter
package root
import (
"os"
"contrib.go.opencensus.io/exporter/ocagent"
"github.com/virtual-kubelet/virtual-kubelet/errdefs"
"go.opencensus.io/trace"
)
func init() {
RegisterTracingExporter("ocagent", NewOCAgentExporter)
}
// NewOCAgentExporter creates a new opencensus tracing exporter using the opencensus agent forwarder.
func NewOCAgentExporter(opts TracingExporterOptions) (trace.Exporter, error) {
agentOpts := append([]ocagent.ExporterOption{}, ocagent.WithServiceName(opts.ServiceName))
if endpoint := os.Getenv("OCAGENT_ENDPOINT"); endpoint != "" {
agentOpts = append(agentOpts, ocagent.WithAddress(endpoint))
} else {
return nil, errdefs.InvalidInput("must set endpoint address in OCAGENT_ENDPOINT")
}
switch os.Getenv("OCAGENT_INSECURE") {
case "0", "no", "n", "off", "":
case "1", "yes", "y", "on":
agentOpts = append(agentOpts, ocagent.WithInsecure())
default:
return nil, errdefs.InvalidInput("invalid value for OCAGENT_INSECURE")
}
return ocagent.NewExporter(agentOpts...)
}

View File

@@ -1,10 +1,23 @@
package cmd
// Copyright © 2017 The virtual-kubelet authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package root
import (
"testing"
"github.com/cpuguy83/strongerrors"
"github.com/virtual-kubelet/virtual-kubelet/errdefs"
"go.opencensus.io/trace"
)
@@ -16,7 +29,7 @@ func TestGetTracingExporter(t *testing.T) {
}
_, err := GetTracingExporter("notexist", TracingExporterOptions{})
if !strongerrors.IsNotFound(err) {
if !errdefs.IsNotFound(err) {
t.Fatalf("expected not found error, got: %v", err)
}

View File

@@ -1,10 +1,10 @@
// Copyright 2016 VMware, Inc. All Rights Reserved.
// Copyright © 2017 The virtual-kubelet authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
@@ -12,29 +12,22 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// Package errors provides error handling functions.
//
package errors
package version
import (
"fmt"
"github.com/spf13/cobra"
)
func ErrorStack(err error) string {
return err.Error()
}
func Errorf(format string, a ...interface{}) error {
return fmt.Errorf(format, a...)
}
func New(err string) error {
return fmt.Errorf("%s", err)
}
func Trace(err error) error {
if err == nil {
return nil
// NewCommand creates a new version subcommand command
func NewCommand(version, buildTime string) *cobra.Command {
return &cobra.Command{
Use: "version",
Short: "Show the version of the program",
Long: `Show the version of the program`,
Run: func(cmd *cobra.Command, args []string) {
fmt.Printf("Version: %s, Built: %s\n", version, buildTime)
},
}
return err
}

View File

@@ -0,0 +1,94 @@
// Copyright © 2017 The virtual-kubelet authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"context"
"os"
"os/signal"
"path/filepath"
"strings"
"syscall"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
cmdproviders "github.com/virtual-kubelet/virtual-kubelet/cmd/virtual-kubelet/commands/providers"
"github.com/virtual-kubelet/virtual-kubelet/cmd/virtual-kubelet/commands/root"
"github.com/virtual-kubelet/virtual-kubelet/cmd/virtual-kubelet/commands/version"
"github.com/virtual-kubelet/virtual-kubelet/log"
logruslogger "github.com/virtual-kubelet/virtual-kubelet/log/logrus"
"github.com/virtual-kubelet/virtual-kubelet/providers"
"github.com/virtual-kubelet/virtual-kubelet/trace"
"github.com/virtual-kubelet/virtual-kubelet/trace/opencensus"
)
var (
buildVersion = "N/A"
buildTime = "N/A"
k8sVersion = "v1.13.1" // This should follow the version of k8s.io/kubernetes we are importing
)
func main() {
ctx, cancel := context.WithCancel(context.Background())
sig := make(chan os.Signal, 1)
signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM)
go func() {
<-sig
cancel()
}()
log.L = logruslogger.FromLogrus(logrus.NewEntry(logrus.StandardLogger()))
trace.T = opencensus.Adapter{}
var opts root.Opts
optsErr := root.SetDefaultOpts(&opts)
opts.Version = strings.Join([]string{k8sVersion, "vk", buildVersion}, "-")
s := providers.NewStore()
registerMock(s)
rootCmd := root.NewCommand(ctx, filepath.Base(os.Args[0]), s, opts)
rootCmd.AddCommand(version.NewCommand(buildVersion, buildTime), cmdproviders.NewCommand(s))
preRun := rootCmd.PreRunE
var logLevel string
rootCmd.PreRunE = func(cmd *cobra.Command, args []string) error {
if optsErr != nil {
return optsErr
}
if preRun != nil {
return preRun(cmd, args)
}
return nil
}
rootCmd.PersistentFlags().StringVar(&logLevel, "log-level", "info", `set the log level, e.g. "debug", "info", "warn", "error"`)
rootCmd.PersistentPreRunE = func(cmd *cobra.Command, args []string) error {
if logLevel != "" {
lvl, err := logrus.ParseLevel(logLevel)
if err != nil {
return errors.Wrap(err, "could not parse log level")
}
logrus.SetLevel(lvl)
}
return nil
}
if err := rootCmd.Execute(); err != nil && errors.Cause(err) != context.Canceled {
log.G(ctx).Fatal(err)
}
}

View File

@@ -0,0 +1,28 @@
package main
import (
"github.com/virtual-kubelet/virtual-kubelet/providers"
"github.com/virtual-kubelet/virtual-kubelet/providers/mock"
)
func registerMock(s *providers.Store) {
s.Register("mock", func(cfg providers.InitConfig) (providers.Provider, error) {
return mock.NewMockProvider(
cfg.ConfigPath,
cfg.NodeName,
cfg.OperatingSystem,
cfg.InternalIP,
cfg.DaemonPort,
)
})
s.Register("mockV0", func(cfg providers.InitConfig) (providers.Provider, error) {
return mock.NewMockProvider(
cfg.ConfigPath,
cfg.NodeName,
cfg.OperatingSystem,
cfg.InternalIP,
cfg.DaemonPort,
)
})
}

View File

@@ -0,0 +1,29 @@
# 2019 Virtual Kubelet roadmap
**work in progress**
## Core ideals for next year
1. Reduce the barrier of entry.
2. Create a stable community.
3. Release virtual kubelet 1.0.
### Requirements and goals
1. **Core.** Split out providers from the virtual kubelet core tree. Today the provider dependencies within the virtual kubelet cause more harm than good.
2. **Interface.** Stablize the virtual kubelet interface so minimal changes to no changes will be needed in the future.
3. **Developer.** Reduce the barrier of entry to create with virtual kubelet.
4. **Community.** Grow and nurture the community for virtual kubelet core. Includes compiling use-cases past the cloud providers interests. IoT use-cases will be a big focus.
### Testing
1. Improve our e2e testing in virtual kubelet core.
2. Add options & integration points to include any provider.
3. Create a baseline for how *quickly* virtual kubelet can process requests. Test at high scale throughput.
4. Work with sig-architecture in Kubernetes to develop a conformance test profile for virtual kubelet.
### Use cases
1. Explore what it means to use virtual kubelet in an IoT Edge usecases and work with wg-io-edge in Kubernetes to develop a standard within virtual kubelet to enable those usecases.
2. Create provider agnostic tools to scale into virtual kubelet.
3. tba

65
errdefs/invalid.go Normal file
View File

@@ -0,0 +1,65 @@
package errdefs
import (
"errors"
"fmt"
)
// InvalidInput is an error interface which denotes whether the opration failed due
// to a the resource not being found.
type ErrInvalidInput interface {
InvalidInput() bool
error
}
type invalidInputError struct {
error
}
func (e *invalidInputError) InvalidInput() bool {
return true
}
func (e *invalidInputError) Cause() error {
return e.error
}
// AsInvalidInput wraps the passed in error to make it of type ErrInvalidInput
//
// Callers should make sure the passed in error has exactly the error message
// it wants as this function does not decorate the message.
func AsInvalidInput(err error) error {
if err == nil {
return nil
}
return &invalidInputError{err}
}
// InvalidInput makes an ErrInvalidInput from the provided error message
func InvalidInput(msg string) error {
return &invalidInputError{errors.New(msg)}
}
// InvalidInputf makes an ErrInvalidInput from the provided error format and args
func InvalidInputf(format string, args ...interface{}) error {
return &invalidInputError{fmt.Errorf(format, args...)}
}
// IsInvalidInput determines if the passed in error is of type ErrInvalidInput
//
// This will traverse the causal chain (`Cause() error`), until it finds an error
// which implements the `InvalidInput` interface.
func IsInvalidInput(err error) bool {
if err == nil {
return false
}
if e, ok := err.(ErrInvalidInput); ok {
return e.InvalidInput()
}
if e, ok := err.(causal); ok {
return IsInvalidInput(e.Cause())
}
return false
}

82
errdefs/invalid_test.go Normal file
View File

@@ -0,0 +1,82 @@
package errdefs
import (
"fmt"
"testing"
"github.com/pkg/errors"
"gotest.tools/assert"
"gotest.tools/assert/cmp"
)
type testingInvalidInputError bool
func (e testingInvalidInputError) Error() string {
return fmt.Sprintf("%v", bool(e))
}
func (e testingInvalidInputError) InvalidInput() bool {
return bool(e)
}
func TestIsInvalidInput(t *testing.T) {
type testCase struct {
name string
err error
xMsg string
xInvalidInput bool
}
for _, c := range []testCase{
{
name: "InvalidInputf",
err: InvalidInputf("%s not found", "foo"),
xMsg: "foo not found",
xInvalidInput: true,
},
{
name: "AsInvalidInput",
err: AsInvalidInput(errors.New("this is a test")),
xMsg: "this is a test",
xInvalidInput: true,
},
{
name: "AsInvalidInputWithNil",
err: AsInvalidInput(nil),
xMsg: "",
xInvalidInput: false,
},
{
name: "nilError",
err: nil,
xMsg: "",
xInvalidInput: false,
},
{
name: "customInvalidInputFalse",
err: testingInvalidInputError(false),
xMsg: "false",
xInvalidInput: false,
},
{
name: "customInvalidInputTrue",
err: testingInvalidInputError(true),
xMsg: "true",
xInvalidInput: true,
},
} {
t.Run(c.name, func(t *testing.T) {
assert.Check(t, cmp.Equal(IsInvalidInput(c.err), c.xInvalidInput))
if c.err != nil {
assert.Check(t, cmp.Equal(c.err.Error(), c.xMsg))
}
})
}
}
func TestInvalidInputCause(t *testing.T) {
err := errors.New("test")
e := &invalidInputError{err}
assert.Check(t, cmp.Equal(e.Cause(), err))
assert.Check(t, IsInvalidInput(errors.Wrap(e, "some details")))
}

65
errdefs/notfound.go Normal file
View File

@@ -0,0 +1,65 @@
package errdefs
import (
"errors"
"fmt"
)
// NotFound is an error interface which denotes whether the opration failed due
// to a the resource not being found.
type ErrNotFound interface {
NotFound() bool
error
}
type notFoundError struct {
error
}
func (e *notFoundError) NotFound() bool {
return true
}
func (e *notFoundError) Cause() error {
return e.error
}
// AsNotFound wraps the passed in error to make it of type ErrNotFound
//
// Callers should make sure the passed in error has exactly the error message
// it wants as this function does not decorate the message.
func AsNotFound(err error) error {
if err == nil {
return nil
}
return &notFoundError{err}
}
// NotFound makes an ErrNotFound from the provided error message
func NotFound(msg string) error {
return &notFoundError{errors.New(msg)}
}
// NotFoundf makes an ErrNotFound from the provided error format and args
func NotFoundf(format string, args ...interface{}) error {
return &notFoundError{fmt.Errorf(format, args...)}
}
// IsNotFound determines if the passed in error is of type ErrNotFound
//
// This will traverse the causal chain (`Cause() error`), until it finds an error
// which implements the `NotFound` interface.
func IsNotFound(err error) bool {
if err == nil {
return false
}
if e, ok := err.(ErrNotFound); ok {
return e.NotFound()
}
if e, ok := err.(causal); ok {
return IsNotFound(e.Cause())
}
return false
}

82
errdefs/notfound_test.go Normal file
View File

@@ -0,0 +1,82 @@
package errdefs
import (
"fmt"
"testing"
"github.com/pkg/errors"
"gotest.tools/assert"
"gotest.tools/assert/cmp"
)
type testingNotFoundError bool
func (e testingNotFoundError) Error() string {
return fmt.Sprintf("%v", bool(e))
}
func (e testingNotFoundError) NotFound() bool {
return bool(e)
}
func TestIsNotFound(t *testing.T) {
type testCase struct {
name string
err error
xMsg string
xNotFound bool
}
for _, c := range []testCase{
{
name: "NotFoundf",
err: NotFoundf("%s not found", "foo"),
xMsg: "foo not found",
xNotFound: true,
},
{
name: "AsNotFound",
err: AsNotFound(errors.New("this is a test")),
xMsg: "this is a test",
xNotFound: true,
},
{
name: "AsNotFoundWithNil",
err: AsNotFound(nil),
xMsg: "",
xNotFound: false,
},
{
name: "nilError",
err: nil,
xMsg: "",
xNotFound: false,
},
{
name: "customNotFoundFalse",
err: testingNotFoundError(false),
xMsg: "false",
xNotFound: false,
},
{
name: "customNotFoundTrue",
err: testingNotFoundError(true),
xMsg: "true",
xNotFound: true,
},
} {
t.Run(c.name, func(t *testing.T) {
assert.Check(t, cmp.Equal(IsNotFound(c.err), c.xNotFound))
if c.err != nil {
assert.Check(t, cmp.Equal(c.err.Error(), c.xMsg))
}
})
}
}
func TestNotFoundCause(t *testing.T) {
err := errors.New("test")
e := &notFoundError{err}
assert.Check(t, cmp.Equal(e.Cause(), err))
assert.Check(t, IsNotFound(errors.Wrap(e, "some details")))
}

10
errdefs/wrapped.go Normal file
View File

@@ -0,0 +1,10 @@
package errdefs
// Causal is an error interface for errors which have wrapped another error
// in a non-opaque way.
//
// This pattern is used by github.com/pkg/errors
type causal interface {
Cause() error
error
}

View File

@@ -0,0 +1,32 @@
apiVersion: v1
kind: Pod
metadata:
name: nginx-named-ports
spec:
containers:
- image: nginx
imagePullPolicy: Always
name: nginx
ports:
- containerPort: 80
name: http
protocol: TCP
- containerPort: 443
name: https
livenessProbe:
httpGet:
path: /
port: http
readinessProbe:
httpGet:
path: /
port: http
nodeSelector:
kubernetes.io/role: agent
beta.kubernetes.io/os: linux
type: virtual-kubelet
tolerations:
- key: virtual-kubelet.io/provider
operator: Exists
- key: azure.com/aci
effect: NoSchedule

View File

@@ -0,0 +1,33 @@
apiVersion: v1
kind: Pod
metadata:
name: nginx-named-ports
spec:
containers:
- image: nginx
imagePullPolicy: Always
name: nginx
ports:
- containerPort: 80
name: http
protocol: TCP
- containerPort: 443
name: https
livenessProbe:
httpGet:
path: /
port: 80
readinessProbe:
httpGet:
path: /
port: 80
dnsPolicy: ClusterFirst
nodeSelector:
kubernetes.io/role: agent
beta.kubernetes.io/os: linux
type: virtual-kubelet
tolerations:
- key: virtual-kubelet.io/provider
operator: Exists
- key: azure.com/aci
effect: NoSchedule

View File

@@ -5,7 +5,7 @@ ENV APISERVER_KEY_LOCATION /vkubelet-mock-0-key.pem
ENV KUBELET_PORT 10250
# Use the pre-built binary in "bin/virtual-kubelet".
COPY bin/virtual-kubelet /virtual-kubelet
COPY bin/e2e/virtual-kubelet /virtual-kubelet
# Copy the configuration file for the mock provider.
COPY hack/skaffold/virtual-kubelet/vkubelet-mock-0-cfg.json /vkubelet-mock-0-cfg.json
# Copy the certificate for the HTTPS server.

View File

@@ -13,6 +13,7 @@ rules:
resources:
- configmaps
- secrets
- services
verbs:
- get
- list
@@ -26,6 +27,7 @@ rules:
- get
- list
- watch
- patch
- apiGroups:
- ""
resources:
@@ -39,12 +41,14 @@ rules:
- nodes/status
verbs:
- update
- patch
- apiGroups:
- ""
resources:
- pods/status
verbs:
- update
- patch
- apiGroups:
- ""
resources:

View File

@@ -16,6 +16,20 @@ spec:
- mock
- --provider-config
- /vkubelet-mock-0-cfg.json
- --startup-timeout
- 10s
- --klog.v
- "2"
- --klog.logtostderr
- --log-level
- debug
env:
- name: KUBELET_PORT
value: "10250"
- name: VKUBELET_POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
ports:
- name: metrics
containerPort: 10255

View File

@@ -0,0 +1,364 @@
// +build e2e
package e2e
import (
"fmt"
"testing"
"time"
"github.com/virtual-kubelet/virtual-kubelet/node"
"gotest.tools/assert"
v1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
)
const (
// deleteGracePeriodForProvider is the maximum amount of time we allow for the provider to react to deletion of a pod
// before proceeding to assert that the pod has been deleted.
deleteGracePeriodForProvider = 1 * time.Second
)
// TestGetStatsSummary creates a pod having two containers and queries the /stats/summary endpoint of the virtual-kubelet.
// It expects this endpoint to return stats for the current node, as well as for the aforementioned pod and each of its two containers.
func TestGetStatsSummary(t *testing.T) {
// Create a pod with prefix "nginx-" having three containers.
pod, err := f.CreatePod(f.CreateDummyPodObjectWithPrefix(t.Name(), "nginx-", "foo", "bar", "baz"))
if err != nil {
t.Fatal(err)
}
// Delete the "nginx-0-X" pod after the test finishes.
defer func() {
if err := f.DeletePodImmediately(pod.Namespace, pod.Name); err != nil && !apierrors.IsNotFound(err) {
t.Error(err)
}
}()
// Wait for the "nginx-" pod to be reported as running and ready.
if _, err := f.WaitUntilPodReady(pod.Namespace, pod.Name); err != nil {
t.Fatal(err)
}
// Grab the stats from the provider.
stats, err := f.GetStatsSummary()
if err != nil {
t.Fatal(err)
}
// Make sure that we've got stats for the current node.
if stats.Node.NodeName != f.NodeName {
t.Fatalf("expected stats for node %s, got stats for node %s", f.NodeName, stats.Node.NodeName)
}
// Make sure the "nginx-" pod exists in the slice of PodStats.
idx, err := findPodInPodStats(stats, pod)
if err != nil {
t.Fatal(err)
}
// Make sure that we've got stats for all the containers in the "nginx-" pod.
desiredContainerStatsCount := len(pod.Spec.Containers)
currentContainerStatsCount := len(stats.Pods[idx].Containers)
if currentContainerStatsCount != desiredContainerStatsCount {
t.Fatalf("expected stats for %d containers, got stats for %d containers", desiredContainerStatsCount, currentContainerStatsCount)
}
}
// TestPodLifecycleGracefulDelete creates a pod and verifies that the provider has been asked to create it.
// Then, it deletes the pods and verifies that the provider has been asked to delete it.
// These verifications are made using the /stats/summary endpoint of the virtual-kubelet, by checking for the presence or absence of the pods.
// Hence, the provider being tested must implement the PodMetricsProvider interface.
func TestPodLifecycleGracefulDelete(t *testing.T) {
// Create a pod with prefix "nginx-" having a single container.
podSpec := f.CreateDummyPodObjectWithPrefix(t.Name(), "nginx-", "foo")
podSpec.Spec.NodeName = nodeName
pod, err := f.CreatePod(podSpec)
if err != nil {
t.Fatal(err)
}
// Delete the pod after the test finishes.
defer func() {
if err := f.DeletePodImmediately(pod.Namespace, pod.Name); err != nil && !apierrors.IsNotFound(err) {
t.Error(err)
}
}()
t.Logf("Created pod: %s", pod.Name)
// Wait for the "nginx-" pod to be reported as running and ready.
if _, err := f.WaitUntilPodReady(pod.Namespace, pod.Name); err != nil {
t.Fatal(err)
}
t.Logf("Pod %s ready", pod.Name)
// Grab the pods from the provider.
pods, err := f.GetRunningPods()
assert.NilError(t, err)
// Check if the pod exists in the slice of PodStats.
assert.NilError(t, findPodInPods(pods, pod))
podCh := make(chan error)
var podLast *v1.Pod
go func() {
// Close the podCh channel, signaling we've observed deletion of the pod.
defer close(podCh)
var err error
podLast, err = f.WaitUntilPodDeleted(pod.Namespace, pod.Name)
if err != nil {
// Propagate the error to the outside so we can fail the test.
podCh <- err
}
}()
// Gracefully delete the "nginx-" pod.
if err := f.DeletePod(pod.Namespace, pod.Name); err != nil {
t.Fatal(err)
}
t.Logf("Deleted pod: %s", pod.Name)
// Wait for the delete event to be ACKed.
if err := <-podCh; err != nil {
t.Fatal(err)
}
time.Sleep(deleteGracePeriodForProvider)
// Give the provider some time to react to the MODIFIED/DELETED events before proceeding.
// Grab the pods from the provider.
pods, err = f.GetRunningPods()
assert.NilError(t, err)
// Make sure the pod DOES NOT exist in the provider's set of running pods
assert.Assert(t, findPodInPods(pods, pod) != nil)
// Make sure we saw the delete event, and the delete event was graceful
assert.Assert(t, podLast != nil)
assert.Assert(t, podLast.ObjectMeta.GetDeletionGracePeriodSeconds() != nil)
assert.Assert(t, *podLast.ObjectMeta.GetDeletionGracePeriodSeconds() > 0)
}
// TestPodLifecycleNonGracefulDelete creates one podsand verifies that the provider has created them
// and put them in the running lifecycle. It then does a force delete on the pod, and verifies the provider
// has deleted it.
func TestPodLifecycleForceDelete(t *testing.T) {
podSpec := f.CreateDummyPodObjectWithPrefix(t.Name(), "nginx-", "foo")
// Create a pod with prefix having a single container.
pod, err := f.CreatePod(podSpec)
if err != nil {
t.Fatal(err)
}
// Delete the pod after the test finishes.
defer func() {
if err := f.DeletePodImmediately(pod.Namespace, pod.Name); err != nil && !apierrors.IsNotFound(err) {
t.Error(err)
}
}()
t.Logf("Created pod: %s", pod.Name)
// Wait for the "nginx-" pod to be reported as running and ready.
if _, err := f.WaitUntilPodReady(pod.Namespace, pod.Name); err != nil {
t.Fatal(err)
}
t.Logf("Pod %s ready", pod.Name)
// Grab the pods from the provider.
pods, err := f.GetRunningPods()
assert.NilError(t, err)
// Check if the pod exists in the slice of Pods.
assert.NilError(t, findPodInPods(pods, pod))
// Wait for the pod to be deleted in a separate goroutine.
// This ensures that we don't possibly miss the MODIFIED/DELETED events due to establishing the watch too late in the process.
// It also makes sure that in light of soft deletes, we properly handle non-graceful pod deletion
podCh := make(chan error)
var podLast *v1.Pod
go func() {
// Close the podCh channel, signaling we've observed deletion of the pod.
defer close(podCh)
var err error
// Wait for the pod to be reported as having been deleted.
podLast, err = f.WaitUntilPodDeleted(pod.Namespace, pod.Name)
if err != nil {
// Propagate the error to the outside so we can fail the test.
podCh <- err
}
}()
time.Sleep(deleteGracePeriodForProvider)
// Forcibly delete the pod.
if err := f.DeletePodImmediately(pod.Namespace, pod.Name); err != nil {
t.Logf("Last saw pod in state: %+v", podLast)
t.Fatal(err)
}
t.Log("Force deleted pod: ", pod.Name)
// Wait for the delete event to be ACKed.
if err := <-podCh; err != nil {
t.Logf("Last saw pod in state: %+v", podLast)
t.Fatal(err)
}
// Give the provider some time to react to the MODIFIED/DELETED events before proceeding.
time.Sleep(deleteGracePeriodForProvider)
// Grab the pods from the provider.
pods, err = f.GetRunningPods()
assert.NilError(t, err)
// Make sure the "nginx-" pod DOES NOT exist in the slice of Pods anymore.
assert.Assert(t, findPodInPods(pods, pod) != nil)
t.Logf("Pod ended as phase: %+v", podLast.Status.Phase)
}
// TestCreatePodWithOptionalInexistentSecrets tries to create a pod referencing optional, inexistent secrets.
// It then verifies that the pod is created successfully.
func TestCreatePodWithOptionalInexistentSecrets(t *testing.T) {
// Create a pod with a single container referencing optional, inexistent secrets.
pod, err := f.CreatePod(f.CreatePodObjectWithOptionalSecretKey(t.Name()))
if err != nil {
t.Fatal(err)
}
// Delete the pod after the test finishes.
defer func() {
if err := f.DeletePodImmediately(pod.Namespace, pod.Name); err != nil && !apierrors.IsNotFound(err) {
t.Error(err)
}
}()
// Wait for the pod to be reported as running and ready.
if _, err := f.WaitUntilPodReady(pod.Namespace, pod.Name); err != nil {
t.Fatal(err)
}
// Wait for an event concerning the missing secret to be reported on the pod.
if err := f.WaitUntilPodEventWithReason(pod, node.ReasonOptionalSecretNotFound); err != nil {
t.Fatal(err)
}
// Grab the pods from the provider.
pods, err := f.GetRunningPods()
assert.NilError(t, err)
// Check if the pod exists in the slice of Pods.
assert.NilError(t, findPodInPods(pods, pod))
}
// TestCreatePodWithMandatoryInexistentSecrets tries to create a pod referencing inexistent secrets.
// It then verifies that the pod is not created.
func TestCreatePodWithMandatoryInexistentSecrets(t *testing.T) {
// Create a pod with a single container referencing inexistent secrets.
pod, err := f.CreatePod(f.CreatePodObjectWithMandatorySecretKey(t.Name()))
if err != nil {
t.Fatal(err)
}
// Delete the pod after the test finishes.
defer func() {
if err := f.DeletePodImmediately(pod.Namespace, pod.Name); err != nil && !apierrors.IsNotFound(err) {
t.Error(err)
}
}()
// Wait for an event concerning the missing secret to be reported on the pod.
if err := f.WaitUntilPodEventWithReason(pod, node.ReasonMandatorySecretNotFound); err != nil {
t.Fatal(err)
}
// Grab the pods from the provider.
pods, err := f.GetRunningPods()
assert.NilError(t, err)
// Check if the pod exists in the slice of PodStats.
assert.Assert(t, findPodInPods(pods, pod) != nil)
}
// TestCreatePodWithOptionalInexistentConfigMap tries to create a pod referencing optional, inexistent config map.
// It then verifies that the pod is created successfully.
func TestCreatePodWithOptionalInexistentConfigMap(t *testing.T) {
// Create a pod with a single container referencing optional, inexistent config map.
pod, err := f.CreatePod(f.CreatePodObjectWithOptionalConfigMapKey(t.Name()))
if err != nil {
t.Fatal(err)
}
// Delete the pod after the test finishes.
defer func() {
if err := f.DeletePodImmediately(pod.Namespace, pod.Name); err != nil && !apierrors.IsNotFound(err) {
t.Error(err)
}
}()
// Wait for the pod to be reported as running and ready.
if _, err := f.WaitUntilPodReady(pod.Namespace, pod.Name); err != nil {
t.Fatal(err)
}
// Wait for an event concerning the missing config map to be reported on the pod.
if err := f.WaitUntilPodEventWithReason(pod, node.ReasonOptionalConfigMapNotFound); err != nil {
t.Fatal(err)
}
// Grab the pods from the provider.
pods, err := f.GetRunningPods()
assert.NilError(t, err)
// Check if the pod exists in the slice of PodStats.
assert.NilError(t, findPodInPods(pods, pod))
}
// TestCreatePodWithMandatoryInexistentConfigMap tries to create a pod referencing inexistent secrets.
// It then verifies that the pod is not created.
func TestCreatePodWithMandatoryInexistentConfigMap(t *testing.T) {
// Create a pod with a single container referencing inexistent config map.
pod, err := f.CreatePod(f.CreatePodObjectWithMandatoryConfigMapKey(t.Name()))
if err != nil {
t.Fatal(err)
}
// Delete the pod after the test finishes.
defer func() {
if err := f.DeletePodImmediately(pod.Namespace, pod.Name); err != nil && !apierrors.IsNotFound(err) {
t.Error(err)
}
}()
// Wait for an event concerning the missing config map to be reported on the pod.
if err := f.WaitUntilPodEventWithReason(pod, node.ReasonMandatoryConfigMapNotFound); err != nil {
t.Fatal(err)
}
// Grab the pods from the provider.
pods, err := f.GetRunningPods()
assert.NilError(t, err)
// Check if the pod exists in the slice of PodStats.
assert.Assert(t, findPodInPods(pods, pod) != nil)
}
// findPodInPodStats returns the index of the specified pod in the .pods field of the specified Summary object.
// It returns an error if the specified pod is not found.
func findPodInPodStats(summary *v1alpha1.Summary, pod *v1.Pod) (int, error) {
for i, p := range summary.Pods {
if p.PodRef.Namespace == pod.Namespace && p.PodRef.Name == pod.Name && string(p.PodRef.UID) == string(pod.UID) {
return i, nil
}
}
return -1, fmt.Errorf("failed to find pod \"%s/%s\" in the slice of pod stats", pod.Namespace, pod.Name)
}
// findPodInPodStats returns the index of the specified pod in the .pods field of the specified PodList object.
// It returns error if the pod doesn't exist in the podlist
func findPodInPods(pods *v1.PodList, pod *v1.Pod) error {
for _, p := range pods.Items {
if p.Namespace == pod.Namespace && p.Name == pod.Name && string(p.UID) == string(pod.UID) {
return nil
}
}
return fmt.Errorf("failed to find pod \"%s/%s\" in the slice of pod list", pod.Namespace, pod.Name)
}

View File

@@ -10,8 +10,8 @@ var (
)
// CreatePodObjectWithMandatoryConfigMapKey creates a pod object that references the "key_0" key from the "config-map-0" config map as mandatory.
func (f *Framework) CreatePodObjectWithMandatoryConfigMapKey() *corev1.Pod {
return f.CreatePodObjectWithEnv([]corev1.EnvVar{
func (f *Framework) CreatePodObjectWithMandatoryConfigMapKey(testName string) *corev1.Pod {
return f.CreatePodObjectWithEnv(testName, []corev1.EnvVar{
{
Name: "CONFIG_MAP_0_KEY_0",
ValueFrom: &corev1.EnvVarSource{
@@ -26,8 +26,8 @@ func (f *Framework) CreatePodObjectWithMandatoryConfigMapKey() *corev1.Pod {
}
// CreatePodObjectWithOptionalConfigMapKey creates a pod object that references the "key_0" key from the "config-map-0" config map as optional.
func (f *Framework) CreatePodObjectWithOptionalConfigMapKey() *corev1.Pod {
return f.CreatePodObjectWithEnv([]corev1.EnvVar{
func (f *Framework) CreatePodObjectWithOptionalConfigMapKey(testName string) *corev1.Pod {
return f.CreatePodObjectWithEnv(testName, []corev1.EnvVar{
{
Name: "CONFIG_MAP_0_KEY_0",
ValueFrom: &corev1.EnvVarSource{
@@ -42,8 +42,8 @@ func (f *Framework) CreatePodObjectWithOptionalConfigMapKey() *corev1.Pod {
}
// CreatePodObjectWithMandatorySecretKey creates a pod object that references the "key_0" key from the "secret-0" config map as mandatory.
func (f *Framework) CreatePodObjectWithMandatorySecretKey() *corev1.Pod {
return f.CreatePodObjectWithEnv([]corev1.EnvVar{
func (f *Framework) CreatePodObjectWithMandatorySecretKey(testName string) *corev1.Pod {
return f.CreatePodObjectWithEnv(testName, []corev1.EnvVar{
{
Name: "SECRET_0_KEY_0",
ValueFrom: &corev1.EnvVarSource{
@@ -58,8 +58,8 @@ func (f *Framework) CreatePodObjectWithMandatorySecretKey() *corev1.Pod {
}
// CreatePodObjectWithOptionalSecretKey creates a pod object that references the "key_0" key from the "secret-0" config map as optional.
func (f *Framework) CreatePodObjectWithOptionalSecretKey() *corev1.Pod {
return f.CreatePodObjectWithEnv([]corev1.EnvVar{
func (f *Framework) CreatePodObjectWithOptionalSecretKey(testName string) *corev1.Pod {
return f.CreatePodObjectWithEnv(testName, []corev1.EnvVar{
{
Name: "SECRET_0_KEY_0",
ValueFrom: &corev1.EnvVarSource{
@@ -74,8 +74,8 @@ func (f *Framework) CreatePodObjectWithOptionalSecretKey() *corev1.Pod {
}
// CreatePodObjectWithEnv creates a pod object whose name starts with "env-test-" and that uses the specified environment configuration for its first container.
func (f *Framework) CreatePodObjectWithEnv(env []corev1.EnvVar) *corev1.Pod {
pod := f.CreateDummyPodObjectWithPrefix("env-test-", "foo")
func (f *Framework) CreatePodObjectWithEnv(testName string, env []corev1.EnvVar) *corev1.Pod {
pod := f.CreateDummyPodObjectWithPrefix(testName, "env-test-", "foo")
pod.Spec.Containers[0].Env = env
return pod
}

View File

@@ -8,23 +8,17 @@ import (
// Framework encapsulates the configuration for the current run, and provides helper methods to be used during testing.
type Framework struct {
KubeClient kubernetes.Interface
Namespace string
NodeName string
TaintKey string
TaintValue string
TaintEffect string
KubeClient kubernetes.Interface
Namespace string
NodeName string
}
// NewTestingFramework returns a new instance of the testing framework.
func NewTestingFramework(kubeconfig, namespace, nodeName, taintKey, taintValue, taintEffect string) *Framework {
func NewTestingFramework(kubeconfig, namespace, nodeName string) *Framework {
return &Framework{
KubeClient: createKubeClient(kubeconfig),
Namespace: namespace,
NodeName: nodeName,
TaintKey: taintKey,
TaintValue: taintValue,
TaintEffect: taintEffect,
KubeClient: createKubeClient(kubeconfig),
Namespace: namespace,
NodeName: nodeName,
}
}

View File

@@ -0,0 +1,60 @@
package framework
import (
"context"
"fmt"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/runtime"
watchapi "k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/tools/watch"
)
// WaitUntilNodeCondition establishes a watch on the vk node.
// Then, it waits for the specified condition function to be verified.
func (f *Framework) WaitUntilNodeCondition(fn watch.ConditionFunc) error {
// Create a field selector that matches the specified Pod resource.
fs := fields.OneTermEqualSelector("metadata.name", f.NodeName).String()
// Create a ListWatch so we can receive events for the matched Pod resource.
lw := &cache.ListWatch{
ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
options.FieldSelector = fs
return f.KubeClient.CoreV1().Nodes().List(options)
},
WatchFunc: func(options metav1.ListOptions) (watchapi.Interface, error) {
options.FieldSelector = fs
return f.KubeClient.CoreV1().Nodes().Watch(options)
},
}
// Watch for updates to the Pod resource until fn is satisfied, or until the timeout is reached.
ctx, cancel := context.WithTimeout(context.Background(), defaultWatchTimeout)
defer cancel()
last, err := watch.UntilWithSync(ctx, lw, &corev1.Node{}, nil, fn)
if err != nil {
return err
}
if last == nil {
return fmt.Errorf("no events received for node %q", f.NodeName)
}
return nil
}
// DeleteNode deletes the vk node used by the framework
func (f *Framework) DeleteNode() error {
var gracePeriod int64
propagation := metav1.DeletePropagationBackground
opts := metav1.DeleteOptions{
PropagationPolicy: &propagation,
GracePeriodSeconds: &gracePeriod,
}
return f.KubeClient.CoreV1().Nodes().Delete(f.NodeName, &opts)
}
// GetNode gets the vk nodeused by the framework
func (f *Framework) GetNode() (*corev1.Node, error) {
return f.KubeClient.CoreV1().Nodes().Get(f.NodeName, metav1.GetOptions{})
}

View File

@@ -3,6 +3,7 @@ package framework
import (
"context"
"fmt"
"strings"
"time"
corev1 "k8s.io/api/core/v1"
@@ -15,32 +16,29 @@ import (
podutil "k8s.io/kubernetes/pkg/api/v1/pod"
)
const (
defaultWatchTimeout = 2 * time.Minute
hostnameNodeSelectorLabel = "kubernetes.io/hostname"
)
const defaultWatchTimeout = 2 * time.Minute
// CreateDummyPodObjectWithPrefix creates a dujmmy pod object using the specified prefix as the value of .metadata.generateName.
// A variable number of strings can be provided.
// For each one of these strings, a container that uses the string as its image will be appended to the pod.
// This method DOES NOT create the pod in the Kubernetes API.
func (f *Framework) CreateDummyPodObjectWithPrefix(prefix string, images ...string) *corev1.Pod {
func (f *Framework) CreateDummyPodObjectWithPrefix(testName string, prefix string, images ...string) *corev1.Pod {
// Safe the test name
if testName != "" {
testName = strings.Replace(testName, "/", "-", -1)
testName = strings.ToLower(testName)
prefix = prefix + "-" + testName + "-"
}
enableServiceLink := false
pod := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
GenerateName: prefix,
Namespace: f.Namespace,
},
Spec: corev1.PodSpec{
NodeSelector: map[string]string{
hostnameNodeSelectorLabel: f.NodeName,
},
Tolerations: []corev1.Toleration{
{
Key: f.TaintKey,
Value: f.TaintValue,
Effect: corev1.TaintEffect(f.TaintEffect),
},
},
NodeName: f.NodeName,
EnableServiceLinks: &enableServiceLink,
},
}
for idx, img := range images {
@@ -75,7 +73,7 @@ func (f *Framework) DeletePodImmediately(namespace, name string) error {
// WaitUntilPodCondition establishes a watch on the pod with the specified name and namespace.
// Then, it waits for the specified condition function to be verified.
func (f *Framework) WaitUntilPodCondition(namespace, name string, fn watch.ConditionFunc) error {
func (f *Framework) WaitUntilPodCondition(namespace, name string, fn watch.ConditionFunc) (*corev1.Pod, error) {
// Create a field selector that matches the specified Pod resource.
fs := fields.ParseSelectorOrDie(fmt.Sprintf("metadata.namespace==%s,metadata.name==%s", namespace, name))
// Create a ListWatch so we can receive events for the matched Pod resource.
@@ -94,27 +92,41 @@ func (f *Framework) WaitUntilPodCondition(namespace, name string, fn watch.Condi
defer cfn()
last, err := watch.UntilWithSync(ctx, lw, &corev1.Pod{}, nil, fn)
if err != nil {
return err
return nil, err
}
if last == nil {
return fmt.Errorf("no events received for pod %q", name)
return nil, fmt.Errorf("no events received for pod %q", name)
}
return nil
pod := last.Object.(*corev1.Pod)
return pod, nil
}
// WaitUntilPodReady blocks until the pod with the specified name and namespace is reported to be running and ready.
func (f *Framework) WaitUntilPodReady(namespace, name string) error {
func (f *Framework) WaitUntilPodReady(namespace, name string) (*corev1.Pod, error) {
return f.WaitUntilPodCondition(namespace, name, func(event watchapi.Event) (bool, error) {
pod := event.Object.(*corev1.Pod)
return pod.Status.Phase == corev1.PodRunning && podutil.IsPodReady(pod) && pod.Status.PodIP != "", nil
})
}
// WaitUntilPodDeleted blocks until the pod with the specified name and namespace is marked for deletion (or, alternatively, effectively deleted).
func (f *Framework) WaitUntilPodDeleted(namespace, name string) error {
// WaitUntilPodDeleted blocks until the pod with the specified name and namespace is deleted from apiserver.
func (f *Framework) WaitUntilPodDeleted(namespace, name string) (*corev1.Pod, error) {
return f.WaitUntilPodCondition(namespace, name, func(event watchapi.Event) (bool, error) {
pod := event.Object.(*corev1.Pod)
return event.Type == watchapi.Deleted || pod.DeletionTimestamp != nil, nil
return event.Type == watchapi.Deleted || pod.ObjectMeta.DeletionTimestamp != nil, nil
})
}
// WaitUntilPodInPhase blocks until the pod with the specified name and namespace is in one of the specified phases
func (f *Framework) WaitUntilPodInPhase(namespace, name string, phases ...corev1.PodPhase) (*corev1.Pod, error) {
return f.WaitUntilPodCondition(namespace, name, func(event watchapi.Event) (bool, error) {
pod := event.Object.(*corev1.Pod)
for _, p := range phases {
if pod.Status.Phase == p {
return true, nil
}
}
return false, nil
})
}
@@ -155,3 +167,20 @@ func (f *Framework) WaitUntilPodEventWithReason(pod *corev1.Pod, reason string)
}
return nil
}
// GetRunningPods gets the running pods from the provider of the virtual kubelet
func (f *Framework) GetRunningPods() (*corev1.PodList, error) {
result := &corev1.PodList{}
err := f.KubeClient.CoreV1().
RESTClient().
Get().
Resource("nodes").
Name(f.NodeName).
SubResource("proxy").
Suffix("runningpods/").
Do().
Into(result)
return result, err
}

View File

@@ -7,17 +7,14 @@ import (
"os"
"testing"
"k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
"github.com/virtual-kubelet/virtual-kubelet/test/e2e/framework"
"github.com/virtual-kubelet/virtual-kubelet/internal/test/e2e/framework"
)
const (
defaultNamespace = v1.NamespaceDefault
defaultNodeName = "vkubelet-mock-0"
defaultTaintKey = "virtual-kubelet.io/provider"
defaultTaintValue = "mock"
defaultTaintEffect = string(v1.TaintEffectNoSchedule)
defaultNamespace = v1.NamespaceDefault
defaultNodeName = "vkubelet-mock-0"
)
var (
@@ -30,21 +27,12 @@ var (
namespace string
// nodeName is the name of the virtual-kubelet node to test.
nodeName string
// taintKey is the key of the taint that is expected to be associated with the virtual-kubelet node to test.
taintKey string
// taintValue is the value of the taint that is expected to be associated with the virtual-kubelet node to test.
taintValue string
// taintEffect is the effect of the taint that is expected to be associated with the virtual-kubelet node to test.
taintEffect string
)
func init() {
flag.StringVar(&kubeconfig, "kubeconfig", "", "path to the kubeconfig file to use when running the test suite outside a kubernetes cluster")
flag.StringVar(&namespace, "namespace", defaultNamespace, "the name of the kubernetes namespace to use for running the test suite (i.e. where to create pods)")
flag.StringVar(&nodeName, "node-name", defaultNodeName, "the name of the virtual-kubelet node to test")
flag.StringVar(&taintKey, "taint-key", defaultTaintKey, "the key of the taint that is expected to be associated with the virtual-kubelet node to test")
flag.StringVar(&taintValue, "taint-value", defaultTaintValue, "the value of the taint that is expected to be associated with the virtual-kubelet node to test")
flag.StringVar(&taintEffect, "taint-effect", defaultTaintEffect, "the effect of the taint that is expected to be associated with the virtual-kubelet node to test")
flag.Parse()
}
@@ -52,9 +40,9 @@ func TestMain(m *testing.M) {
// Set sane defaults in case no values (or empty ones) have been provided.
setDefaults()
// Create a new instance of the test framework targeting the specified node.
f = framework.NewTestingFramework(kubeconfig, namespace, nodeName, taintKey, taintValue, taintEffect)
f = framework.NewTestingFramework(kubeconfig, namespace, nodeName)
// Wait for the virtual-kubelet pod to be ready.
if err := f.WaitUntilPodReady(namespace, nodeName); err != nil {
if _, err := f.WaitUntilPodReady(namespace, nodeName); err != nil {
panic(err)
}
// Run the test suite.
@@ -69,13 +57,4 @@ func setDefaults() {
if nodeName == "" {
nodeName = defaultNodeName
}
if taintKey == "" {
taintKey = defaultTaintKey
}
if taintValue == "" {
taintValue = defaultTaintValue
}
if taintEffect == "" {
taintEffect = defaultTaintEffect
}
}

View File

@@ -0,0 +1,63 @@
// +build e2e
package e2e
import (
"context"
"testing"
"time"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
watchapi "k8s.io/apimachinery/pkg/watch"
)
// TestNodeCreateAfterDelete makes sure that a node is automatically recreated
// if it is deleted while VK is running.
func TestNodeCreateAfterDelete(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
podList, err := f.KubeClient.CoreV1().Pods(f.Namespace).List(metav1.ListOptions{
FieldSelector: fields.OneTermEqualSelector("spec.nodeName", f.NodeName).String(),
})
assert.NilError(t, err)
assert.Assert(t, is.Len(podList.Items, 0), "Kubernetes does not allow node deletion with dependent objects (pods) in existence: %v")
chErr := make(chan error, 1)
originalNode, err := f.GetNode()
assert.NilError(t, err)
ctx, cancel = context.WithTimeout(ctx, time.Minute)
defer cancel()
go func() {
wait := func(e watchapi.Event) (bool, error) {
err = ctx.Err()
// Our timeout has expired
if err != nil {
return true, err
}
if e.Type == watchapi.Deleted || e.Type == watchapi.Error {
return false, nil
}
return originalNode.ObjectMeta.UID != e.Object.(*v1.Node).ObjectMeta.UID, nil
}
chErr <- f.WaitUntilNodeCondition(wait)
}()
assert.NilError(t, f.DeleteNode())
select {
case result := <-chErr:
assert.NilError(t, result, "Did not observe new node object created after deletion")
case <-ctx.Done():
t.Fatal("Test timed out while waiting for node object to be deleted / recreated")
}
}

View File

@@ -24,6 +24,8 @@ func FakeEventRecorder(bufferSize int) *record.FakeRecorder {
// FakePodWithSingleContainer returns a pod with the specified namespace and name, and having a single container with the specified image.
func FakePodWithSingleContainer(namespace, name, image string) *corev1.Pod {
enableServiceLink := corev1.DefaultEnableServiceLinks
return &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Namespace: namespace,
@@ -36,6 +38,7 @@ func FakePodWithSingleContainer(namespace, name, image string) *corev1.Pod {
Image: image,
},
},
EnableServiceLinks: &enableServiceLink,
},
}
}
@@ -54,3 +57,20 @@ func FakeSecret(namespace, name string, data map[string]string) *corev1.Secret {
}
return res
}
// FakeService returns a service with the specified namespace and name and service info.
func FakeService(namespace, name, clusterIP, protocol string, port int32) *corev1.Service {
return &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Namespace: namespace,
Name: name,
},
Spec: corev1.ServiceSpec{
Ports: []corev1.ServicePort{{
Protocol: corev1.Protocol(protocol),
Port: port,
}},
ClusterIP: clusterIP,
},
}
}

View File

@@ -17,22 +17,24 @@ import (
func FakeResourceManager(objects ...runtime.Object) *manager.ResourceManager {
// Create a fake Kubernetes client that will list the specified objects.
kubeClient := fake.NewSimpleClientset(objects...)
// Create a shared informer factory from where we can grab informers and listers for pods, configmaps and secrets.
// Create a shared informer factory from where we can grab informers and listers for pods, configmaps, secrets and services.
kubeInformerFactory := informers.NewSharedInformerFactory(kubeClient, 30*time.Second)
// Grab informers for pods, configmaps and secrets.
pInformer := kubeInformerFactory.Core().V1().Pods()
mInformer := kubeInformerFactory.Core().V1().ConfigMaps()
sInformer := kubeInformerFactory.Core().V1().Secrets()
svcInformer := kubeInformerFactory.Core().V1().Services()
// Start all the required informers.
go pInformer.Informer().Run(wait.NeverStop)
go mInformer.Informer().Run(wait.NeverStop)
go sInformer.Informer().Run(wait.NeverStop)
go svcInformer.Informer().Run(wait.NeverStop)
// Wait for the caches to be synced.
if !cache.WaitForCacheSync(wait.NeverStop, pInformer.Informer().HasSynced, mInformer.Informer().HasSynced, sInformer.Informer().HasSynced) {
if !cache.WaitForCacheSync(wait.NeverStop, pInformer.Informer().HasSynced, mInformer.Informer().HasSynced, sInformer.Informer().HasSynced, svcInformer.Informer().HasSynced) {
panic("failed to wait for caches to be synced")
}
// Create a new instance of the resource manager using the listers for pods, configmaps and secrets.
r, err := manager.NewResourceManager(pInformer.Lister(), sInformer.Lister(), mInformer.Lister())
r, err := manager.NewResourceManager(pInformer.Lister(), sInformer.Lister(), mInformer.Lister(), svcInformer.Lister())
if err != nil {
panic(err)
}

View File

@@ -14,77 +14,71 @@
limitations under the License.
*/
// Package log defines the interfaces used for logging in virtual-kubelet.
// It uses a context.Context to store logger details. Additionally you can set
// the default logger to use by setting log.L. This is used when no logger is
// stored in the passed in context.
package log
import (
"context"
"sync/atomic"
"github.com/Sirupsen/logrus"
)
var (
// G is an alias for GetLogger.
//
// We may want to define this locally to a package to get package tagged log
// messages.
G = GetLogger
// L is an alias for the the standard logger.
L = logrus.NewEntry(logrus.StandardLogger())
// L is the default logger. It should be initialized before using `G` or `GetLogger`
// If L is unitialized and no logger is available in a provided context, a
// panic will occur.
L Logger = nopLogger{}
)
type (
loggerKey struct{}
)
// TraceLevel is the log level for tracing. Trace level is lower than debug level,
// and is usually used to trace detailed behavior of the program.
const TraceLevel = logrus.Level(uint32(logrus.DebugLevel + 1))
// Logger is the interface used for logging in virtual-kubelet
//
// virtual-kubelet will access the logger via context using `GetLogger` (or its alias, `G`)
// You can set the default logger to use by setting the `L` variable.
type Logger interface {
Debug(...interface{})
Debugf(string, ...interface{})
Info(...interface{})
Infof(string, ...interface{})
Warn(...interface{})
Warnf(string, ...interface{})
Error(...interface{})
Errorf(string, ...interface{})
Fatal(...interface{})
Fatalf(string, ...interface{})
// RFC3339NanoFixed is time.RFC3339Nano with nanoseconds padded using zeros to
// ensure the formatted time is always the same number of characters.
const RFC3339NanoFixed = "2006-01-02T15:04:05.000000000Z07:00"
// ParseLevel takes a string level and returns the Logrus log level constant.
// It supports trace level.
func ParseLevel(lvl string) (logrus.Level, error) {
if lvl == "trace" {
return TraceLevel, nil
}
return logrus.ParseLevel(lvl)
WithField(string, interface{}) Logger
WithFields(Fields) Logger
WithError(error) Logger
}
// Fields allows setting multiple fields on a logger at one time.
type Fields map[string]interface{}
// WithLogger returns a new context with the provided logger. Use in
// combination with logger.WithField(s) for great effect.
func WithLogger(ctx context.Context, logger *logrus.Entry) context.Context {
func WithLogger(ctx context.Context, logger Logger) context.Context {
return context.WithValue(ctx, loggerKey{}, logger)
}
// GetLogger retrieves the current logger from the context. If no logger is
// available, the default logger is returned.
func GetLogger(ctx context.Context) *logrus.Entry {
func GetLogger(ctx context.Context) Logger {
logger := ctx.Value(loggerKey{})
if logger == nil {
if L == nil {
panic("default logger not initialized")
}
return L
}
return logger.(*logrus.Entry)
}
// Trace logs a message at level Trace with the log entry passed-in.
func Trace(e *logrus.Entry, args ...interface{}) {
level := logrus.Level(atomic.LoadUint32((*uint32)(&e.Logger.Level)))
if level >= TraceLevel {
e.Debug(args...)
}
}
// Tracef logs a message at level Trace with the log entry passed-in.
func Tracef(e *logrus.Entry, format string, args ...interface{}) {
level := logrus.Level(atomic.LoadUint32((*uint32)(&e.Logger.Level)))
if level >= TraceLevel {
e.Debugf(format, args...)
}
return logger.(Logger)
}

48
log/logrus/logrus.go Normal file
View File

@@ -0,0 +1,48 @@
// Copyright © 2017 The virtual-kubelet authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package logrus implements a github.com/virtual-kubelet/virtual-kubelet/log.Logger using Logrus as a backend
// You can use this by creating a logrus logger and calling `FromLogrus(entry)`.
// If you want this to be the default logger for virtual-kubelet, set `log.L` to the value returned by `FromLogrus`
package logrus
import (
"github.com/sirupsen/logrus"
"github.com/virtual-kubelet/virtual-kubelet/log"
)
// Adapter implements the `log.Logger` interface for logrus
type Adapter struct {
*logrus.Entry
}
// FromLogrus creates a new `log.Logger` from the provided entry
func FromLogrus(entry *logrus.Entry) log.Logger {
return &Adapter{entry}
}
// WithField adds a field to the log entry.
func (l *Adapter) WithField(key string, val interface{}) log.Logger {
return FromLogrus(l.Entry.WithField(key, val))
}
// WithFields adds multiple fields to a log entry.
func (l *Adapter) WithFields(f log.Fields) log.Logger {
return FromLogrus(l.Entry.WithFields(logrus.Fields(f)))
}
// WithError adds an error to the log entry
func (l *Adapter) WithError(err error) log.Logger {
return FromLogrus(l.Entry.WithError(err))
}

View File

@@ -1,10 +1,10 @@
// Copyright 2017 VMware, Inc. All Rights Reserved.
// Copyright © 2017 The virtual-kubelet authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
@@ -12,16 +12,19 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// NOTE: This file is borrowed directly from docker in order to use unexported
// functions that it provides.
// https://github.com/moby/moby/blob/master/pkg/archive/archive_unix.go
package archive
package logrus
import (
"os"
"testing"
"github.com/sirupsen/logrus"
"github.com/virtual-kubelet/virtual-kubelet/log"
)
func hasHardlinks(fi os.FileInfo) bool {
return false
func TestImplementsLoggerInterface(t *testing.T) {
l := FromLogrus(&logrus.Entry{})
if _, ok := l.(log.Logger); !ok {
t.Fatal("does not implement log.Logger interface")
}
}

32
log/nop.go Normal file
View File

@@ -0,0 +1,32 @@
// Copyright © 2017 The virtual-kubelet authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package log
type nopLogger struct{}
func (nopLogger) Debug(...interface{}) {}
func (nopLogger) Debugf(string, ...interface{}) {}
func (nopLogger) Info(...interface{}) {}
func (nopLogger) Infof(string, ...interface{}) {}
func (nopLogger) Warn(...interface{}) {}
func (nopLogger) Warnf(string, ...interface{}) {}
func (nopLogger) Error(...interface{}) {}
func (nopLogger) Errorf(string, ...interface{}) {}
func (nopLogger) Fatal(...interface{}) {}
func (nopLogger) Fatalf(string, ...interface{}) {}
func (l nopLogger) WithField(string, interface{}) Logger { return l }
func (l nopLogger) WithFields(Fields) Logger { return l }
func (l nopLogger) WithError(error) Logger { return l }

View File

@@ -1,10 +1,10 @@
// Copyright 2015 go-swagger maintainers
// Copyright © 2017 The virtual-kubelet authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
@@ -12,7 +12,10 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// Package strfmt contains custom string formats
// Package manager provides access to kubernetes resources for providers.
//
// TODO: add info on how to define and register a custom format
package strfmt
// DEPRECATION WARNING:
// Though this package is still in use, it should be considered deprecated as it
// is just wrapping a k8s client and not much else.
// Implementers should look at replacing their use of this with something else.
package manager

View File

@@ -1,9 +1,21 @@
// Copyright © 2017 The virtual-kubelet authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package manager
import (
"sync"
"k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/labels"
corev1listers "k8s.io/client-go/listers/core/v1"
@@ -13,19 +25,19 @@ import (
// ResourceManager acts as a passthrough to a cache (lister) for pods assigned to the current node.
// It is also a passthrough to a cache (lister) for Kubernetes secrets and config maps.
type ResourceManager struct {
sync.RWMutex
podLister corev1listers.PodLister
secretLister corev1listers.SecretLister
configMapLister corev1listers.ConfigMapLister
serviceLister corev1listers.ServiceLister
}
// NewResourceManager returns a ResourceManager with the internal maps initialized.
func NewResourceManager(podLister corev1listers.PodLister, secretLister corev1listers.SecretLister, configMapLister corev1listers.ConfigMapLister) (*ResourceManager, error) {
func NewResourceManager(podLister corev1listers.PodLister, secretLister corev1listers.SecretLister, configMapLister corev1listers.ConfigMapLister, serviceLister corev1listers.ServiceLister) (*ResourceManager, error) {
rm := ResourceManager{
podLister: podLister,
secretLister: secretLister,
configMapLister: configMapLister,
serviceLister: serviceLister,
}
return &rm, nil
}
@@ -49,3 +61,8 @@ func (rm *ResourceManager) GetConfigMap(name, namespace string) (*v1.ConfigMap,
func (rm *ResourceManager) GetSecret(name, namespace string) (*v1.Secret, error) {
return rm.secretLister.Secrets(namespace).Get(name)
}
// ListServices retrieves the list of services from Kubernetes.
func (rm *ResourceManager) ListServices() ([]*v1.Service, error) {
return rm.serviceLister.List(labels.Everything())
}

View File

@@ -1,15 +1,29 @@
// Copyright © 2017 The virtual-kubelet authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package manager_test
import (
"testing"
"k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
corev1listers "k8s.io/client-go/listers/core/v1"
"k8s.io/client-go/tools/cache"
testutil "github.com/virtual-kubelet/virtual-kubelet/internal/test/util"
"github.com/virtual-kubelet/virtual-kubelet/manager"
testutil "github.com/virtual-kubelet/virtual-kubelet/test/util"
)
// TestGetPods verifies that the resource manager acts as a passthrough to a pod lister.
@@ -29,7 +43,7 @@ func TestGetPods(t *testing.T) {
podLister := corev1listers.NewPodLister(indexer)
// Create a new instance of the resource manager based on the pod lister.
rm, err := manager.NewResourceManager(podLister, nil, nil)
rm, err := manager.NewResourceManager(podLister, nil, nil, nil)
if err != nil {
t.Fatal(err)
}
@@ -58,7 +72,7 @@ func TestGetSecret(t *testing.T) {
secretLister := corev1listers.NewSecretLister(indexer)
// Create a new instance of the resource manager based on the secret lister.
rm, err := manager.NewResourceManager(nil, secretLister, nil)
rm, err := manager.NewResourceManager(nil, secretLister, nil, nil)
if err != nil {
t.Fatal(err)
}
@@ -97,7 +111,7 @@ func TestGetConfigMap(t *testing.T) {
configMapLister := corev1listers.NewConfigMapLister(indexer)
// Create a new instance of the resource manager based on the config map lister.
rm, err := manager.NewResourceManager(nil, nil, configMapLister)
rm, err := manager.NewResourceManager(nil, nil, configMapLister, nil)
if err != nil {
t.Fatal(err)
}
@@ -118,3 +132,35 @@ func TestGetConfigMap(t *testing.T) {
t.Fatalf("expected a 'not found' error, got %v", err)
}
}
// TestListServices verifies that the resource manager acts as a passthrough to a service lister.
func TestListServices(t *testing.T) {
var (
lsServices = []*v1.Service{
testutil.FakeService("namespace-0", "service-0", "1.2.3.1", "TCP", 8081),
testutil.FakeService("namespace-1", "service-1", "1.2.3.2", "TCP", 8082),
}
)
// Create a pod lister that will list the pods defined above.
indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})
for _, service := range lsServices {
indexer.Add(service)
}
serviceLister := corev1listers.NewServiceLister(indexer)
// Create a new instance of the resource manager based on the pod lister.
rm, err := manager.NewResourceManager(nil, nil, nil, serviceLister)
if err != nil {
t.Fatal(err)
}
// Check that the resource manager returns two pods in the call to "GetPods".
services, err := rm.ListServices()
if err != nil {
t.Fatal(err)
}
if len(lsServices) != len(services) {
t.Fatalf("expected %d services, found %d", len(lsServices), len(services))
}
}

13
netlify.toml Normal file
View File

@@ -0,0 +1,13 @@
[build]
base = "website"
publish = "website/public"
command = "make production-build"
[build.environment]
HUGO_VERSION = "0.55.0"
[context.deploy-preview]
command = "make preview-build"
[context.branch-deploy]
command = "make preview-build"

View File

@@ -12,10 +12,6 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import "github.com/virtual-kubelet/virtual-kubelet/cmd"
func main() {
cmd.Execute()
}
// Package api implements HTTP handlers for handling requests that the kubelet
// would normally implement, such as pod logs, exec, etc.
package api

188
node/api/exec.go Normal file
View File

@@ -0,0 +1,188 @@
// Copyright © 2017 The virtual-kubelet authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package api
import (
"context"
"io"
"net/http"
"strings"
"time"
"github.com/gorilla/mux"
"github.com/pkg/errors"
"github.com/virtual-kubelet/virtual-kubelet/errdefs"
"k8s.io/apimachinery/pkg/types"
remoteutils "k8s.io/client-go/tools/remotecommand"
api "k8s.io/kubernetes/pkg/apis/core"
"k8s.io/kubernetes/pkg/kubelet/server/remotecommand"
)
// ContainerExecHandlerFunc defines the handler function used for "execing" into a
// container in a pod.
type ContainerExecHandlerFunc func(ctx context.Context, namespace, podName, containerName string, cmd []string, attach AttachIO) error
// AttachIO is used to pass in streams to attach to a container process
type AttachIO interface {
Stdin() io.Reader
Stdout() io.WriteCloser
Stderr() io.WriteCloser
TTY() bool
Resize() <-chan TermSize
}
// TermSize is used to set the terminal size from attached clients.
type TermSize struct {
Width uint16
Height uint16
}
// HandleContainerExec makes an http handler func from a Provider which execs a command in a pod's container
// Note that this handler currently depends on gorrilla/mux to get url parts as variables.
// TODO(@cpuguy83): don't force gorilla/mux on consumers of this function
func HandleContainerExec(h ContainerExecHandlerFunc) http.HandlerFunc {
if h == nil {
return NotImplemented
}
return handleError(func(w http.ResponseWriter, req *http.Request) error {
vars := mux.Vars(req)
namespace := vars["namespace"]
pod := vars["pod"]
container := vars["container"]
supportedStreamProtocols := strings.Split(req.Header.Get("X-Stream-Protocol-Version"), ",")
q := req.URL.Query()
command := q["command"]
streamOpts, err := getExecOptions(req)
if err != nil {
return errdefs.AsInvalidInput(err)
}
idleTimeout := time.Second * 30
streamCreationTimeout := time.Second * 30
ctx, cancel := context.WithCancel(context.TODO())
defer cancel()
exec := &containerExecContext{ctx: ctx, h: h, pod: pod, namespace: namespace, container: container}
remotecommand.ServeExec(w, req, exec, "", "", container, command, streamOpts, idleTimeout, streamCreationTimeout, supportedStreamProtocols)
return nil
})
}
func getExecOptions(req *http.Request) (*remotecommand.Options, error) {
tty := req.FormValue(api.ExecTTYParam) == "1"
stdin := req.FormValue(api.ExecStdinParam) == "1"
stdout := req.FormValue(api.ExecStdoutParam) == "1"
stderr := req.FormValue(api.ExecStderrParam) == "1"
if tty && stderr {
return nil, errors.New("cannot exec with tty and stderr")
}
if !stdin && !stdout && !stderr {
return nil, errors.New("you must specify at least one of stdin, stdout, stderr")
}
return &remotecommand.Options{
Stdin: stdin,
Stdout: stdout,
Stderr: stderr,
TTY: tty,
}, nil
}
type containerExecContext struct {
h ContainerExecHandlerFunc
eio *execIO
namespace, pod, container string
ctx context.Context
}
// ExecInContainer Implements remotecommand.Executor
// This is called by remotecommand.ServeExec
func (c *containerExecContext) ExecInContainer(name string, uid types.UID, container string, cmd []string, in io.Reader, out, err io.WriteCloser, tty bool, resize <-chan remoteutils.TerminalSize, timeout time.Duration) error {
eio := &execIO{
tty: tty,
stdin: in,
stdout: out,
stderr: err,
}
if tty {
eio.chResize = make(chan TermSize)
}
ctx, cancel := context.WithCancel(c.ctx)
defer cancel()
if tty {
go func() {
send := func(s remoteutils.TerminalSize) bool {
select {
case eio.chResize <- TermSize{Width: s.Width, Height: s.Height}:
return false
case <-ctx.Done():
return true
}
}
for {
select {
case s := <-resize:
if send(s) {
return
}
case <-ctx.Done():
return
}
}
}()
}
return c.h(c.ctx, c.namespace, c.pod, c.container, cmd, eio)
}
type execIO struct {
tty bool
stdin io.Reader
stdout io.WriteCloser
stderr io.WriteCloser
chResize chan TermSize
}
func (e *execIO) TTY() bool {
return e.tty
}
func (e *execIO) Stdin() io.Reader {
return e.stdin
}
func (e *execIO) Stdout() io.WriteCloser {
return e.stdout
}
func (e *execIO) Stderr() io.WriteCloser {
return e.stderr
}
func (e *execIO) Resize() <-chan TermSize {
return e.chResize
}

84
node/api/helpers.go Normal file
View File

@@ -0,0 +1,84 @@
// Copyright © 2017 The virtual-kubelet authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package api
import (
"io"
"net/http"
"github.com/virtual-kubelet/virtual-kubelet/errdefs"
"github.com/virtual-kubelet/virtual-kubelet/log"
)
type handlerFunc func(http.ResponseWriter, *http.Request) error
func handleError(f handlerFunc) http.HandlerFunc {
return func(w http.ResponseWriter, req *http.Request) {
err := f(w, req)
if err == nil {
return
}
code := httpStatusCode(err)
w.WriteHeader(code)
io.WriteString(w, err.Error())
logger := log.G(req.Context()).WithError(err).WithField("httpStatusCode", code)
if code >= 500 {
logger.Error("Internal server error on request")
} else {
logger.Debug("Error on request")
}
}
}
func flushOnWrite(w io.Writer) io.Writer {
if fw, ok := w.(writeFlusher); ok {
return &flushWriter{fw}
}
return w
}
type flushWriter struct {
w writeFlusher
}
type writeFlusher interface {
Flush() error
Write([]byte) (int, error)
}
func (fw *flushWriter) Write(p []byte) (int, error) {
n, err := fw.w.Write(p)
if n > 0 {
if err := fw.w.Flush(); err != nil {
return n, err
}
}
return n, err
}
func httpStatusCode(err error) int {
switch {
case err == nil:
return http.StatusOK
case errdefs.IsNotFound(err):
return http.StatusNotFound
case errdefs.IsInvalidInput(err):
return http.StatusBadRequest
default:
return http.StatusInternalServerError
}
}

94
node/api/logs.go Normal file
View File

@@ -0,0 +1,94 @@
// Copyright © 2017 The virtual-kubelet authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package api
import (
"context"
"io"
"net/http"
"strconv"
"time"
"github.com/gorilla/mux"
"github.com/pkg/errors"
"github.com/virtual-kubelet/virtual-kubelet/errdefs"
"github.com/virtual-kubelet/virtual-kubelet/log"
)
// ContainerLogsHandlerFunc is used in place of backend implementations for getting container logs
type ContainerLogsHandlerFunc func(ctx context.Context, namespace, podName, containerName string, opts ContainerLogOpts) (io.ReadCloser, error)
// ContainerLogOpts are used to pass along options to be set on the container
// log stream.
type ContainerLogOpts struct {
Tail int
Since time.Duration
LimitBytes int
Timestamps bool
}
// HandleContainerLogs creates an http handler function from a provider to serve logs from a pod
func HandleContainerLogs(h ContainerLogsHandlerFunc) http.HandlerFunc {
if h == nil {
return NotImplemented
}
return handleError(func(w http.ResponseWriter, req *http.Request) error {
vars := mux.Vars(req)
if len(vars) != 3 {
return errdefs.NotFound("not found")
}
ctx := req.Context()
namespace := vars["namespace"]
pod := vars["pod"]
container := vars["container"]
tail := 10
q := req.URL.Query()
if queryTail := q.Get("tailLines"); queryTail != "" {
t, err := strconv.Atoi(queryTail)
if err != nil {
return errdefs.AsInvalidInput(errors.Wrap(err, "could not parse \"tailLines\""))
}
tail = t
}
// TODO(@cpuguy83): support v1.PodLogOptions
// The kubelet decoding here is not straight forward, so this needs to be disected
opts := ContainerLogOpts{
Tail: tail,
}
logs, err := h(ctx, namespace, pod, container, opts)
if err != nil {
return errors.Wrap(err, "error getting container logs?)")
}
defer logs.Close()
req.Header.Set("Transfer-Encoding", "chunked")
if _, ok := w.(writeFlusher); !ok {
log.G(ctx).Debug("http response writer does not support flushes")
}
if _, err := io.Copy(flushOnWrite(w), logs); err != nil {
return errors.Wrap(err, "error writing response to client")
}
return nil
})
}

62
node/api/pods.go Normal file
View File

@@ -0,0 +1,62 @@
// Copyright © 2017 The virtual-kubelet authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package api
import (
"context"
"net/http"
"github.com/virtual-kubelet/virtual-kubelet/log"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/serializer"
)
type PodListerFunc func(context.Context) ([]*v1.Pod, error)
func HandleRunningPods(getPods PodListerFunc) http.HandlerFunc {
scheme := runtime.NewScheme()
v1.SchemeBuilder.AddToScheme(scheme)
codecs := serializer.NewCodecFactory(scheme)
return handleError(func(w http.ResponseWriter, req *http.Request) error {
ctx := req.Context()
ctx = log.WithLogger(ctx, log.L)
pods, err := getPods(ctx)
if err != nil {
return err
}
// Borrowed from github.com/kubernetes/kubernetes/pkg/kubelet/server/server.go
// encodePods creates an v1.PodList object from pods and returns the encoded
// PodList.
podList := new(v1.PodList)
for _, pod := range pods {
podList.Items = append(podList.Items, *pod)
}
codec := codecs.LegacyCodec(v1.SchemeGroupVersion)
data, err := runtime.Encode(codec, podList)
if err != nil {
return err
}
w.Header().Set("Content-Type", "application/json")
_, err = w.Write(data)
if err != nil {
return err
}
return nil
})
}

135
node/api/server.go Normal file
View File

@@ -0,0 +1,135 @@
// Copyright © 2017 The virtual-kubelet authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package api
import (
"net/http"
"github.com/gorilla/mux"
"github.com/virtual-kubelet/virtual-kubelet/log"
"go.opencensus.io/plugin/ochttp"
"go.opencensus.io/plugin/ochttp/propagation/b3"
)
// ServeMux defines an interface used to attach routes to an existing http
// serve mux.
// It is used to enable callers creating a new server to completely manage
// their own HTTP server while allowing us to attach the required routes to
// satisfy the Kubelet HTTP interfaces.
type ServeMux interface {
Handle(path string, h http.Handler)
}
type PodHandlerConfig struct {
RunInContainer ContainerExecHandlerFunc
GetContainerLogs ContainerLogsHandlerFunc
GetPods PodListerFunc
}
// PodHandler creates an http handler for interacting with pods/containers.
func PodHandler(p PodHandlerConfig, debug bool) http.Handler {
r := mux.NewRouter()
// This matches the behaviour in the reference kubelet
r.StrictSlash(true)
if debug {
r.HandleFunc("/runningpods/", HandleRunningPods(p.GetPods)).Methods("GET")
}
r.HandleFunc("/containerLogs/{namespace}/{pod}/{container}", HandleContainerLogs(p.GetContainerLogs)).Methods("GET")
r.HandleFunc("/exec/{namespace}/{pod}/{container}", HandleContainerExec(p.RunInContainer)).Methods("POST")
r.NotFoundHandler = http.HandlerFunc(NotFound)
return r
}
// PodStatsSummaryHandler creates an http handler for serving pod metrics.
//
// If the passed in handler func is nil this will create handlers which only
// serves http.StatusNotImplemented
func PodStatsSummaryHandler(f PodStatsSummaryHandlerFunc) http.Handler {
if f == nil {
return http.HandlerFunc(NotImplemented)
}
r := mux.NewRouter()
const summaryRoute = "/stats/summary"
h := HandlePodStatsSummary(f)
r.Handle(summaryRoute, ochttp.WithRouteTag(h, "PodStatsSummaryHandler")).Methods("GET")
r.Handle(summaryRoute+"/", ochttp.WithRouteTag(h, "PodStatsSummaryHandler")).Methods("GET")
r.NotFoundHandler = http.HandlerFunc(NotFound)
return r
}
// AttachPodRoutes adds the http routes for pod stuff to the passed in serve mux.
//
// Callers should take care to namespace the serve mux as they see fit, however
// these routes get called by the Kubernetes API server.
func AttachPodRoutes(p PodHandlerConfig, mux ServeMux, debug bool) {
mux.Handle("/", InstrumentHandler(PodHandler(p, debug)))
}
// PodMetricsConfig stores the handlers for pod metrics routes
// It is used by AttachPodMetrics.
//
// The main reason for this struct is in case of expansion we do not need to break
// the package level API.
type PodMetricsConfig struct {
GetStatsSummary PodStatsSummaryHandlerFunc
}
// AttachPodMetricsRoutes adds the http routes for pod/node metrics to the passed in serve mux.
//
// Callers should take care to namespace the serve mux as they see fit, however
// these routes get called by the Kubernetes API server.
func AttachPodMetricsRoutes(p PodMetricsConfig, mux ServeMux) {
mux.Handle("/", InstrumentHandler(HandlePodStatsSummary(p.GetStatsSummary)))
}
func instrumentRequest(r *http.Request) *http.Request {
ctx := r.Context()
logger := log.G(ctx).WithFields(log.Fields{
"uri": r.RequestURI,
"vars": mux.Vars(r),
})
ctx = log.WithLogger(ctx, logger)
return r.WithContext(ctx)
}
// InstrumentHandler wraps an http.Handler and injects instrumentation into the request context.
func InstrumentHandler(h http.Handler) http.Handler {
instrumented := http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
req = instrumentRequest(req)
h.ServeHTTP(w, req)
})
return &ochttp.Handler{
Handler: instrumented,
Propagation: &b3.HTTPFormat{},
}
}
// NotFound provides a handler for cases where the requested endpoint doesn't exist
func NotFound(w http.ResponseWriter, r *http.Request) {
log.G(r.Context()).Debug("404 request not found")
http.Error(w, "404 request not found", http.StatusNotFound)
}
// NotImplemented provides a handler for cases where a provider does not implement a given API
func NotImplemented(w http.ResponseWriter, r *http.Request) {
log.G(r.Context()).Debug("501 not implemented")
http.Error(w, "501 not implemented", http.StatusNotImplemented)
}

69
node/api/stats.go Normal file
View File

@@ -0,0 +1,69 @@
// Copyright © 2017 The virtual-kubelet authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package api
import (
"context"
"encoding/json"
"net/http"
"github.com/pkg/errors"
stats "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
)
// PodStatsSummaryHandlerFunc defines the handler for getting pod stats summaries
type PodStatsSummaryHandlerFunc func(context.Context) (*stats.Summary, error)
// HandlePodStatsSummary makes an HTTP handler for implementing the kubelet summary stats endpoint
func HandlePodStatsSummary(h PodStatsSummaryHandlerFunc) http.HandlerFunc {
if h == nil {
return NotImplemented
}
return handleError(func(w http.ResponseWriter, req *http.Request) error {
stats, err := h(req.Context())
if err != nil {
if isCancelled(err) {
return err
}
return errors.Wrap(err, "error getting status from provider")
}
b, err := json.Marshal(stats)
if err != nil {
return errors.Wrap(err, "error marshalling stats")
}
if _, err := w.Write(b); err != nil {
return errors.Wrap(err, "could not write to client")
}
return nil
})
}
func isCancelled(err error) bool {
if err == context.Canceled {
return true
}
if e, ok := err.(causal); ok {
return isCancelled(e.Cause())
}
return false
}
type causal interface {
Cause() error
error
}

62
node/doc.go Normal file
View File

@@ -0,0 +1,62 @@
// Copyright © 2017 The virtual-kubelet authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
/*
Package node implements the components for operating a node in Kubernetes.
This includes controllers for managin the node object, running scheduled pods,
and exporting HTTP endpoints expected by the Kubernets API server.
There are two primary controllers, the node runner and the pod runner.
nodeRunner, _ := node.NewNodeController(...)
// setup other things
podRunner, _ := node.NewPodController(...)
go podRunner.Run(ctx)
select {
case <-podRunner.Ready():
go nodeRunner.Run(ctx)
case <-ctx.Done()
return ctx.Err()
}
After calling start, cancelling the passed in context will shutdown the
controller.
Note this example elides error handling.
Up to this point you have an active node in Kubernetes which can have pods scheduled
to it. However the API server expects nodes to implement API endpoints in order
to support certain features such as fetching logs or execing a new process.
The api package provides some helpers for this:
`api.AttachPodRoutes` and `api.AttachMetricsRoutes`.
mux := http.NewServeMux()
api.AttachPodRoutes(provider, mux)
You must configure your own HTTP server, but these helpers will add handlers at
the correct URI paths to your serve mux. You are not required to use go's
built-in `*http.ServeMux`, but it does implement the `ServeMux` interface
defined in this package which is used for these helpers.
Note: The metrics routes may need to be attached to a different HTTP server,
depending on your configuration.
For more fine-grained control over the API, see the `node/api` package which
only implements the HTTP handlers that you can use in whatever way you want.
This uses open-cenesus to implement tracing (but no internal metrics yet) which
is propagated through the context. This is passed on even to the providers.
*/
package node

190
vkubelet/env.go → node/env.go Normal file → Executable file
View File

@@ -1,4 +1,18 @@
package vkubelet
// Copyright © 2017 The virtual-kubelet authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package node
import (
"context"
@@ -8,8 +22,15 @@ import (
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/sets"
apivalidation "k8s.io/apimachinery/pkg/util/validation"
"k8s.io/client-go/tools/record"
podshelper "k8s.io/kubernetes/pkg/apis/core/pods"
v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper"
fieldpath "k8s.io/kubernetes/pkg/fieldpath"
"k8s.io/kubernetes/pkg/kubelet/envvars"
"k8s.io/kubernetes/third_party/forked/golang/expansion"
"github.com/virtual-kubelet/virtual-kubelet/log"
"github.com/virtual-kubelet/virtual-kubelet/manager"
@@ -48,9 +69,12 @@ const (
ReasonInvalidEnvironmentVariableNames = "InvalidEnvironmentVariableNames"
)
var masterServices = sets.NewString("kubernetes")
// populateEnvironmentVariables populates the environment of each container (and init container) in the specified pod.
// TODO Make this the single exported function of a "pkg/environment" package in the future.
func populateEnvironmentVariables(ctx context.Context, pod *corev1.Pod, rm *manager.ResourceManager, recorder record.EventRecorder) error {
// Populate each init container's environment.
for idx := range pod.Spec.InitContainers {
if err := populateContainerEnvironment(ctx, pod, &pod.Spec.InitContainers[idx], rm, recorder); err != nil {
@@ -69,12 +93,13 @@ func populateEnvironmentVariables(ctx context.Context, pod *corev1.Pod, rm *mana
// populateContainerEnvironment populates the environment of a single container in the specified pod.
func populateContainerEnvironment(ctx context.Context, pod *corev1.Pod, container *corev1.Container, rm *manager.ResourceManager, recorder record.EventRecorder) error {
// Create an "environment map" based on the value of the specified container's ".envFrom" field.
envFrom, err := makeEnvironmentMapBasedOnEnvFrom(ctx, pod, container, rm, recorder)
tmpEnv, err := makeEnvironmentMapBasedOnEnvFrom(ctx, pod, container, rm, recorder)
if err != nil {
return err
}
// Create an "environment map" based on the value of the specified container's ".env" field.
env, err := makeEnvironmentMapBasedOnEnv(ctx, pod, container, rm, recorder)
// Create the final "environment map" for the container using the ".env" and ".envFrom" field
// and service environment variables.
err = makeEnvironmentMap(ctx, pod, container, rm, recorder, tmpEnv)
if err != nil {
return err
}
@@ -83,10 +108,67 @@ func populateContainerEnvironment(ctx context.Context, pod *corev1.Pod, containe
// This is in accordance with what the Kubelet itself does.
// https://github.com/kubernetes/kubernetes/blob/v1.13.1/pkg/kubelet/kubelet_pods.go#L557-L558
container.EnvFrom = []corev1.EnvFromSource{}
container.Env = mergeEnvironments(envFrom, env)
res := make([]corev1.EnvVar, 0)
for key, val := range tmpEnv {
res = append(res, corev1.EnvVar{
Name: key,
Value: val,
})
}
container.Env = res
return nil
}
// getServiceEnvVarMap makes a map[string]string of env vars for services a
// pod in namespace ns should see.
// Based on getServiceEnvVarMap in kubelet_pods.go.
func getServiceEnvVarMap(rm *manager.ResourceManager, ns string, enableServiceLinks bool) (map[string]string, error) {
var (
serviceMap = make(map[string]*corev1.Service)
m = make(map[string]string)
)
services, err := rm.ListServices()
if err != nil {
return nil, err
}
// project the services in namespace ns onto the master services
for i := range services {
service := services[i]
// ignore services where ClusterIP is "None" or empty
if !v1helper.IsServiceIPSet(service) {
continue
}
serviceName := service.Name
// We always want to add environment variables for master kubernetes service
// from the default namespace, even if enableServiceLinks is false.
// We also add environment variables for other services in the same
// namespace, if enableServiceLinks is true.
if service.Namespace == metav1.NamespaceDefault && masterServices.Has(serviceName) {
if _, exists := serviceMap[serviceName]; !exists {
serviceMap[serviceName] = service
}
} else if service.Namespace == ns && enableServiceLinks {
serviceMap[serviceName] = service
}
}
mappedServices := make([]*corev1.Service, 0, len(serviceMap))
for key := range serviceMap {
mappedServices = append(mappedServices, serviceMap[key])
}
for _, e := range envvars.FromServices(mappedServices) {
m[e.Name] = e.Value
}
return m, nil
}
// makeEnvironmentMapBasedOnEnvFrom returns a map representing the resolved environment of the specified container after being populated from the entries in the ".envFrom" field.
func makeEnvironmentMapBasedOnEnvFrom(ctx context.Context, pod *corev1.Pod, container *corev1.Container, rm *manager.ResourceManager, recorder record.EventRecorder) (map[string]string, error) {
// Create a map to hold the resulting environment.
@@ -209,17 +291,37 @@ loop:
return res, nil
}
// makeEnvironmentMapBasedOnEnv returns a map representing the resolved environment of the specified container after being populated from the entries in the ".env" field.
func makeEnvironmentMapBasedOnEnv(ctx context.Context, pod *corev1.Pod, container *corev1.Container, rm *manager.ResourceManager, recorder record.EventRecorder) (map[string]string, error) {
// Create a map to hold the resolved environment variables.
res := make(map[string]string, len(container.Env))
// makeEnvironmentMap returns a map representing the resolved environment of the specified container after being populated from the entries in the ".env" and ".envFrom" field.
func makeEnvironmentMap(ctx context.Context, pod *corev1.Pod, container *corev1.Container, rm *manager.ResourceManager, recorder record.EventRecorder, res map[string]string) error {
// TODO If pod.Spec.EnableServiceLinks is nil then fail as per 1.14 kubelet.
enableServiceLinks := corev1.DefaultEnableServiceLinks
if pod.Spec.EnableServiceLinks != nil {
enableServiceLinks = *pod.Spec.EnableServiceLinks
}
// Note that there is a race between Kubelet seeing the pod and kubelet seeing the service.
// To avoid this users can: (1) wait between starting a service and starting; or (2) detect
// missing service env var and exit and be restarted; or (3) use DNS instead of env vars
// and keep trying to resolve the DNS name of the service (recommended).
svcEnv, err := getServiceEnvVarMap(rm, pod.Namespace, enableServiceLinks)
if err != nil {
return err
}
// If the variable's Value is set, expand the `$(var)` references to other
// variables in the .Value field; the sources of variables are the declared
// variables of the container and the service environment variables.
mappingFunc := expansion.MappingFuncFor(res, svcEnv)
// Iterate over environment variables in order to populate the map.
loop:
for _, env := range container.Env {
switch {
// Handle values that have been directly provided.
case env.Value != "":
res[env.Name] = env.Value
// Expand variable references
res[env.Name] = expansion.Expand(env.Value, mappingFunc)
continue loop
// Handle population from a configmap key.
case env.ValueFrom != nil && env.ValueFrom.ConfigMapKeyRef != nil:
@@ -227,7 +329,7 @@ loop:
vf := env.ValueFrom.ConfigMapKeyRef
// Check whether the key reference is optional.
// This will control whether we fail when unable to read the requested key.
optional := vf != nil && *vf.Optional
optional := vf != nil && vf.Optional != nil && *vf.Optional
// Try to grab the referenced configmap.
m, err := rm.GetConfigMap(vf.Name, pod.Namespace)
if err != nil {
@@ -247,10 +349,10 @@ loop:
// Hence, we should return a meaningful error.
if errors.IsNotFound(err) {
recorder.Eventf(pod, corev1.EventTypeWarning, ReasonMandatoryConfigMapNotFound, "configmap %q not found", vf.Name)
return nil, fmt.Errorf("configmap %q not found", vf.Name)
return fmt.Errorf("configmap %q not found", vf.Name)
}
recorder.Eventf(pod, corev1.EventTypeWarning, ReasonFailedToReadMandatoryConfigMap, "failed to read configmap %q", vf.Name)
return nil, fmt.Errorf("failed to read configmap %q: %v", vf.Name, err)
return fmt.Errorf("failed to read configmap %q: %v", vf.Name, err)
}
// At this point we have successfully fetched the target configmap.
// We must now try to grab the requested key.
@@ -269,7 +371,7 @@ loop:
// At this point we know the key reference is mandatory.
// Hence, we should fail.
recorder.Eventf(pod, corev1.EventTypeWarning, ReasonMandatoryConfigMapKeyNotFound, "key %q does not exist in configmap %q", vf.Key, vf.Name)
return nil, fmt.Errorf("configmap %q doesn't contain the %q key required by pod %s", vf.Name, vf.Key, pod.Name)
return fmt.Errorf("configmap %q doesn't contain the %q key required by pod %s", vf.Name, vf.Key, pod.Name)
}
// Populate the environment variable and continue on to the next reference.
res[env.Name] = keyValue
@@ -279,7 +381,7 @@ loop:
vf := env.ValueFrom.SecretKeyRef
// Check whether the key reference is optional.
// This will control whether we fail when unable to read the requested key.
optional := vf != nil && *vf.Optional
optional := vf != nil && vf.Optional != nil && *vf.Optional
// Try to grab the referenced secret.
s, err := rm.GetSecret(vf.Name, pod.Namespace)
if err != nil {
@@ -299,10 +401,10 @@ loop:
// Hence, we should return a meaningful error.
if errors.IsNotFound(err) {
recorder.Eventf(pod, corev1.EventTypeWarning, ReasonMandatorySecretNotFound, "secret %q not found", vf.Name)
return nil, fmt.Errorf("secret %q not found", vf.Name)
return fmt.Errorf("secret %q not found", vf.Name)
}
recorder.Eventf(pod, corev1.EventTypeWarning, ReasonFailedToReadMandatorySecret, "failed to read secret %q", vf.Name)
return nil, fmt.Errorf("failed to read secret %q: %v", vf.Name, err)
return fmt.Errorf("failed to read secret %q: %v", vf.Name, err)
}
// At this point we have successfully fetched the target secret.
// We must now try to grab the requested key.
@@ -321,15 +423,23 @@ loop:
// At this point we know the key reference is mandatory.
// Hence, we should fail.
recorder.Eventf(pod, corev1.EventTypeWarning, ReasonMandatorySecretKeyNotFound, "key %q does not exist in secret %q", vf.Key, vf.Name)
return nil, fmt.Errorf("secret %q doesn't contain the %q key required by pod %s", vf.Name, vf.Key, pod.Name)
return fmt.Errorf("secret %q doesn't contain the %q key required by pod %s", vf.Name, vf.Key, pod.Name)
}
// Populate the environment variable and continue on to the next reference.
res[env.Name] = string(keyValue)
continue loop
// Handle population from a field (downward API).
case env.ValueFrom != nil && env.ValueFrom.FieldRef != nil:
// TODO Implement the downward API.
// https://github.com/virtual-kubelet/virtual-kubelet/issues/123
vf := env.ValueFrom.FieldRef
runtimeVal, err := podFieldSelectorRuntimeValue(vf, pod)
if err != nil {
return err
}
res[env.Name] = runtimeVal
continue loop
// Handle population from a resource request/limit.
case env.ValueFrom != nil && env.ValueFrom.ResourceFieldRef != nil:
@@ -337,28 +447,30 @@ loop:
continue loop
}
}
// Return the populated environment.
return res, nil
// Append service env vars.
for k, v := range svcEnv {
if _, present := res[k]; !present {
res[k] = v
}
}
return nil
}
// mergeEnvironments creates the final environment for a container by merging "envFrom" and "env".
// Values in "env" override any values with the same key defined in "envFrom".
// This is in accordance with what the Kubelet itself does.
// https://github.com/kubernetes/kubernetes/blob/v1.13.1/pkg/kubelet/kubelet_pods.go#L557-L558
func mergeEnvironments(envFrom map[string]string, env map[string]string) []corev1.EnvVar {
tmp := make(map[string]string, 0)
res := make([]corev1.EnvVar, 0)
for key, val := range envFrom {
tmp[key] = val
// podFieldSelectorRuntimeValue returns the runtime value of the given
// selector for a pod.
func podFieldSelectorRuntimeValue(fs *corev1.ObjectFieldSelector, pod *corev1.Pod) (string, error) {
internalFieldPath, _, err := podshelper.ConvertDownwardAPIFieldLabel(fs.APIVersion, fs.FieldPath, "")
if err != nil {
return "", err
}
for key, val := range env {
tmp[key] = val
switch internalFieldPath {
case "spec.nodeName":
return pod.Spec.NodeName, nil
case "spec.serviceAccountName":
return pod.Spec.ServiceAccountName, nil
}
for key, val := range tmp {
res = append(res, corev1.EnvVar{
Name: key,
Value: val,
})
}
return res
return fieldpath.ExtractFieldPathAsString(pod, internalFieldPath)
}

1092
node/env_internal_test.go Normal file

File diff suppressed because it is too large Load Diff

554
node/node.go Normal file
View File

@@ -0,0 +1,554 @@
// Copyright © 2017 The virtual-kubelet authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package node
import (
"context"
"encoding/json"
"fmt"
"time"
pkgerrors "github.com/pkg/errors"
"github.com/virtual-kubelet/virtual-kubelet/log"
"github.com/virtual-kubelet/virtual-kubelet/trace"
coord "k8s.io/api/coordination/v1beta1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/strategicpatch"
"k8s.io/client-go/kubernetes/typed/coordination/v1beta1"
v1 "k8s.io/client-go/kubernetes/typed/core/v1"
)
// NodeProvider is the interface used for registering a node and updating its
// status in Kubernetes.
//
// Note: Implementers can choose to manage a node themselves, in which case
// it is not needed to provide an implementation for this interface.
type NodeProvider interface {
// Ping checks if the node is still active.
// This is intended to be lightweight as it will be called periodically as a
// heartbeat to keep the node marked as ready in Kubernetes.
Ping(context.Context) error
// NotifyNodeStatus is used to asynchronously monitor the node.
// The passed in callback should be called any time there is a change to the
// node's status.
// This will generally trigger a call to the Kubernetes API server to update
// the status.
//
// NotifyNodeStatus should not block callers.
NotifyNodeStatus(ctx context.Context, cb func(*corev1.Node))
}
// NewNodeController creates a new node controller.
// This does not have any side-effects on the system or kubernetes.
//
// Use the node's `Run` method to register and run the loops to update the node
// in Kubernetes.
//
// Note: When if there are multiple NodeControllerOpts which apply against the same
// underlying options, the last NodeControllerOpt will win.
func NewNodeController(p NodeProvider, node *corev1.Node, nodes v1.NodeInterface, opts ...NodeControllerOpt) (*NodeController, error) {
n := &NodeController{p: p, n: node, nodes: nodes, chReady: make(chan struct{})}
for _, o := range opts {
if err := o(n); err != nil {
return nil, pkgerrors.Wrap(err, "error applying node option")
}
}
return n, nil
}
// NodeControllerOpt are the functional options used for configuring a node
type NodeControllerOpt func(*NodeController) error // nolint: golint
// WithNodeEnableLeaseV1Beta1 enables support for v1beta1 leases.
// If client is nil, leases will not be enabled.
// If baseLease is nil, a default base lease will be used.
//
// The lease will be updated after each successful node ping. To change the
// lease update interval, you must set the node ping interval.
// See WithNodePingInterval().
//
// This also affects the frequency of node status updates:
// - When leases are *not* enabled (or are disabled due to no support on the cluster)
// the node status is updated at every ping interval.
// - When node leases are enabled, node status updates are controlled by the
// node status update interval option.
// To set a custom node status update interval, see WithNodeStatusUpdateInterval().
func WithNodeEnableLeaseV1Beta1(client v1beta1.LeaseInterface, baseLease *coord.Lease) NodeControllerOpt {
return func(n *NodeController) error {
n.leases = client
n.lease = baseLease
return nil
}
}
// WithNodePingInterval sets the interval for checking node status
// If node leases are not supported (or not enabled), this is the frequency
// with which the node status will be updated in Kubernetes.
func WithNodePingInterval(d time.Duration) NodeControllerOpt {
return func(n *NodeController) error {
n.pingInterval = d
return nil
}
}
// WithNodeStatusUpdateInterval sets the interval for updating node status
// This is only used when leases are supported and only for updating the actual
// node status, not the node lease.
// When node leases are not enabled (or are not supported on the cluster) this
// has no affect and node status is updated on the "ping" interval.
func WithNodeStatusUpdateInterval(d time.Duration) NodeControllerOpt {
return func(n *NodeController) error {
n.statusInterval = d
return nil
}
}
// WithNodeStatusUpdateErrorHandler adds an error handler for cases where there is an error
// when updating the node status.
// This allows the caller to have some control on how errors are dealt with when
// updating a node's status.
//
// The error passed to the handler will be the error received from kubernetes
// when updating node status.
func WithNodeStatusUpdateErrorHandler(h ErrorHandler) NodeControllerOpt {
return func(n *NodeController) error {
n.nodeStatusUpdateErrorHandler = h
return nil
}
}
// ErrorHandler is a type of function used to allow callbacks for handling errors.
// It is expected that if a nil error is returned that the error is handled and
// progress can continue (or a retry is possible).
type ErrorHandler func(context.Context, error) error
// NodeController deals with creating and managing a node object in Kubernetes.
// It can register a node with Kubernetes and periodically update its status.
// NodeController manages a single node entity.
type NodeController struct { // nolint: golint
p NodeProvider
n *corev1.Node
leases v1beta1.LeaseInterface
nodes v1.NodeInterface
disableLease bool
pingInterval time.Duration
statusInterval time.Duration
lease *coord.Lease
chStatusUpdate chan *corev1.Node
nodeStatusUpdateErrorHandler ErrorHandler
chReady chan struct{}
}
// The default intervals used for lease and status updates.
const (
DefaultPingInterval = 10 * time.Second
DefaultStatusUpdateInterval = 1 * time.Minute
)
// Run registers the node in kubernetes and starts loops for updating the node
// status in Kubernetes.
//
// The node status must be updated periodically in Kubertnetes to keep the node
// active. Newer versions of Kubernetes support node leases, which are
// essentially light weight pings. Older versions of Kubernetes require updating
// the node status periodically.
//
// If Kubernetes supports node leases this will use leases with a much slower
// node status update (because some things still expect the node to be updated
// periodically), otherwise it will only use node status update with the configured
// ping interval.
func (n *NodeController) Run(ctx context.Context) error {
if n.pingInterval == time.Duration(0) {
n.pingInterval = DefaultPingInterval
}
if n.statusInterval == time.Duration(0) {
n.statusInterval = DefaultStatusUpdateInterval
}
n.chStatusUpdate = make(chan *corev1.Node)
n.p.NotifyNodeStatus(ctx, func(node *corev1.Node) {
n.chStatusUpdate <- node
})
if err := n.ensureNode(ctx); err != nil {
return err
}
if n.leases == nil {
n.disableLease = true
return n.controlLoop(ctx)
}
n.lease = newLease(n.lease)
setLeaseAttrs(n.lease, n.n, n.pingInterval*5)
l, err := ensureLease(ctx, n.leases, n.lease)
if err != nil {
if !errors.IsNotFound(err) {
return pkgerrors.Wrap(err, "error creating node lease")
}
log.G(ctx).Info("Node leases not supported, falling back to only node status updates")
n.disableLease = true
}
n.lease = l
log.G(ctx).Debug("Created node lease")
return n.controlLoop(ctx)
}
func (n *NodeController) ensureNode(ctx context.Context) error {
err := n.updateStatus(ctx, true)
if err == nil || !errors.IsNotFound(err) {
return err
}
node, err := n.nodes.Create(n.n)
if err != nil {
return pkgerrors.Wrap(err, "error registering node with kubernetes")
}
n.n = node
return nil
}
// Ready returns a channel that gets closed when the node is fully up and
// running. Note that if there is an error on startup this channel will never
// be started.
func (n *NodeController) Ready() <-chan struct{} {
return n.chReady
}
func (n *NodeController) controlLoop(ctx context.Context) error {
pingTimer := time.NewTimer(n.pingInterval)
defer pingTimer.Stop()
statusTimer := time.NewTimer(n.statusInterval)
defer statusTimer.Stop()
if n.disableLease {
// hack to make sure this channel always blocks since we won't be using it
if !statusTimer.Stop() {
<-statusTimer.C
}
}
close(n.chReady)
for {
select {
case <-ctx.Done():
return nil
case updated := <-n.chStatusUpdate:
var t *time.Timer
if n.disableLease {
t = pingTimer
} else {
t = statusTimer
}
log.G(ctx).Debug("Received node status update")
// Performing a status update so stop/reset the status update timer in this
// branch otherwise there could be an uneccessary status update.
if !t.Stop() {
<-t.C
}
n.n.Status = updated.Status
if err := n.updateStatus(ctx, false); err != nil {
log.G(ctx).WithError(err).Error("Error handling node status update")
}
t.Reset(n.statusInterval)
case <-statusTimer.C:
if err := n.updateStatus(ctx, false); err != nil {
log.G(ctx).WithError(err).Error("Error handling node status update")
}
statusTimer.Reset(n.statusInterval)
case <-pingTimer.C:
if err := n.handlePing(ctx); err != nil {
log.G(ctx).WithError(err).Error("Error while handling node ping")
} else {
log.G(ctx).Debug("Successful node ping")
}
pingTimer.Reset(n.pingInterval)
}
}
}
func (n *NodeController) handlePing(ctx context.Context) (retErr error) {
ctx, span := trace.StartSpan(ctx, "node.handlePing")
defer span.End()
defer func() {
span.SetStatus(retErr)
}()
if err := n.p.Ping(ctx); err != nil {
return pkgerrors.Wrap(err, "error while pinging the node provider")
}
if n.disableLease {
return n.updateStatus(ctx, false)
}
return n.updateLease(ctx)
}
func (n *NodeController) updateLease(ctx context.Context) error {
l, err := UpdateNodeLease(ctx, n.leases, newLease(n.lease))
if err != nil {
return err
}
n.lease = l
return nil
}
func (n *NodeController) updateStatus(ctx context.Context, skipErrorCb bool) error {
updateNodeStatusHeartbeat(n.n)
node, err := UpdateNodeStatus(ctx, n.nodes, n.n)
if err != nil {
if skipErrorCb || n.nodeStatusUpdateErrorHandler == nil {
return err
}
if err := n.nodeStatusUpdateErrorHandler(ctx, err); err != nil {
return err
}
node, err = UpdateNodeStatus(ctx, n.nodes, n.n)
if err != nil {
return err
}
}
n.n = node
return nil
}
func ensureLease(ctx context.Context, leases v1beta1.LeaseInterface, lease *coord.Lease) (*coord.Lease, error) {
l, err := leases.Create(lease)
if err != nil {
switch {
case errors.IsNotFound(err):
log.G(ctx).WithError(err).Info("Node lease not supported")
return nil, err
case errors.IsAlreadyExists(err):
if err := leases.Delete(lease.Name, nil); err != nil && !errors.IsNotFound(err) {
log.G(ctx).WithError(err).Error("could not delete old node lease")
return nil, pkgerrors.Wrap(err, "old lease exists but could not delete it")
}
l, err = leases.Create(lease)
}
}
return l, err
}
// UpdateNodeLease updates the node lease.
//
// If this function returns an errors.IsNotFound(err) error, this likely means
// that node leases are not supported, if this is the case, call UpdateNodeStatus
// instead.
//
// If you use this function, it is up to you to syncronize this with other operations.
func UpdateNodeLease(ctx context.Context, leases v1beta1.LeaseInterface, lease *coord.Lease) (*coord.Lease, error) {
ctx, span := trace.StartSpan(ctx, "node.UpdateNodeLease")
defer span.End()
ctx = span.WithFields(ctx, log.Fields{
"lease.name": lease.Name,
"lease.time": lease.Spec.RenewTime,
})
if lease.Spec.LeaseDurationSeconds != nil {
ctx = span.WithField(ctx, "lease.expiresSeconds", *lease.Spec.LeaseDurationSeconds)
}
l, err := leases.Update(lease)
if err != nil {
if errors.IsNotFound(err) {
log.G(ctx).Debug("lease not found")
l, err = ensureLease(ctx, leases, lease)
}
if err != nil {
span.SetStatus(err)
return nil, err
}
log.G(ctx).Debug("created new lease")
} else {
log.G(ctx).Debug("updated lease")
}
return l, nil
}
// just so we don't have to allocate this on every get request
var emptyGetOptions = metav1.GetOptions{}
// PatchNodeStatus patches node status.
// Copied from github.com/kubernetes/kubernetes/pkg/util/node
func PatchNodeStatus(nodes v1.NodeInterface, nodeName types.NodeName, oldNode *corev1.Node, newNode *corev1.Node) (*corev1.Node, []byte, error) {
patchBytes, err := preparePatchBytesforNodeStatus(nodeName, oldNode, newNode)
if err != nil {
return nil, nil, err
}
updatedNode, err := nodes.Patch(string(nodeName), types.StrategicMergePatchType, patchBytes, "status")
if err != nil {
return nil, nil, fmt.Errorf("failed to patch status %q for node %q: %v", patchBytes, nodeName, err)
}
return updatedNode, patchBytes, nil
}
func preparePatchBytesforNodeStatus(nodeName types.NodeName, oldNode *corev1.Node, newNode *corev1.Node) ([]byte, error) {
oldData, err := json.Marshal(oldNode)
if err != nil {
return nil, fmt.Errorf("failed to Marshal oldData for node %q: %v", nodeName, err)
}
// Reset spec to make sure only patch for Status or ObjectMeta is generated.
// Note that we don't reset ObjectMeta here, because:
// 1. This aligns with Nodes().UpdateStatus().
// 2. Some component does use this to update node annotations.
newNode.Spec = oldNode.Spec
newData, err := json.Marshal(newNode)
if err != nil {
return nil, fmt.Errorf("failed to Marshal newData for node %q: %v", nodeName, err)
}
patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, corev1.Node{})
if err != nil {
return nil, fmt.Errorf("failed to CreateTwoWayMergePatch for node %q: %v", nodeName, err)
}
return patchBytes, nil
}
// UpdateNodeStatus triggers an update to the node status in Kubernetes.
// It first fetches the current node details and then sets the status according
// to the passed in node object.
//
// If you use this function, it is up to you to syncronize this with other operations.
// This reduces the time to second-level precision.
func UpdateNodeStatus(ctx context.Context, nodes v1.NodeInterface, n *corev1.Node) (_ *corev1.Node, retErr error) {
ctx, span := trace.StartSpan(ctx, "UpdateNodeStatus")
defer func() {
span.End()
span.SetStatus(retErr)
}()
var node *corev1.Node
oldNode, err := nodes.Get(n.Name, emptyGetOptions)
if err != nil {
return nil, err
}
log.G(ctx).Debug("got node from api server")
node = oldNode.DeepCopy()
node.ResourceVersion = ""
node.Status = n.Status
ctx = addNodeAttributes(ctx, span, node)
// Patch the node status to merge other changes on the node.
updated, _, err := PatchNodeStatus(nodes, types.NodeName(n.Name), oldNode, node)
if err != nil {
return nil, err
}
log.G(ctx).WithField("node.resourceVersion", updated.ResourceVersion).
WithField("node.Status.Conditions", updated.Status.Conditions).
Debug("updated node status in api server")
return updated, nil
}
func newLease(base *coord.Lease) *coord.Lease {
var lease *coord.Lease
if base == nil {
lease = &coord.Lease{}
} else {
lease = base.DeepCopy()
}
lease.Spec.RenewTime = &metav1.MicroTime{Time: time.Now()}
return lease
}
func setLeaseAttrs(l *coord.Lease, n *corev1.Node, dur time.Duration) {
if l.Name == "" {
l.Name = n.Name
}
if l.Spec.HolderIdentity == nil {
l.Spec.HolderIdentity = &n.Name
}
if l.Spec.LeaseDurationSeconds == nil {
d := int32(dur.Seconds()) * 5
l.Spec.LeaseDurationSeconds = &d
}
}
func updateNodeStatusHeartbeat(n *corev1.Node) {
now := metav1.NewTime(time.Now())
for i := range n.Status.Conditions {
n.Status.Conditions[i].LastHeartbeatTime = now
}
}
// NaiveNodeProvider is a basic node provider that only uses the passed in context
// on `Ping` to determine if the node is healthy.
type NaiveNodeProvider struct{}
// Ping just implements the NodeProvider interface.
// It returns the error from the passed in context only.
func (NaiveNodeProvider) Ping(ctx context.Context) error {
return ctx.Err()
}
// NotifyNodeStatus implements the NodeProvider interface.
//
// This NaiveNodeProvider does not support updating node status and so this
// function is a no-op.
func (NaiveNodeProvider) NotifyNodeStatus(ctx context.Context, f func(*corev1.Node)) {
}
type taintsStringer []corev1.Taint
func (t taintsStringer) String() string {
var s string
for _, taint := range t {
if s == "" {
s = taint.Key + "=" + taint.Value + ":" + string(taint.Effect)
} else {
s += ", " + taint.Key + "=" + taint.Value + ":" + string(taint.Effect)
}
}
return s
}
func addNodeAttributes(ctx context.Context, span trace.Span, n *corev1.Node) context.Context {
return span.WithFields(ctx, log.Fields{
"node.UID": string(n.UID),
"node.name": n.Name,
"node.cluster": n.ClusterName,
"node.taints": taintsStringer(n.Spec.Taints),
})
}

393
node/node_test.go Normal file
View File

@@ -0,0 +1,393 @@
// Copyright © 2017 The virtual-kubelet authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package node
import (
"context"
"strings"
"testing"
"time"
"gotest.tools/assert"
"gotest.tools/assert/cmp"
coord "k8s.io/api/coordination/v1beta1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
watch "k8s.io/apimachinery/pkg/watch"
testclient "k8s.io/client-go/kubernetes/fake"
)
func TestNodeRun(t *testing.T) {
t.Run("WithoutLease", func(t *testing.T) { testNodeRun(t, false) })
t.Run("WithLease", func(t *testing.T) { testNodeRun(t, true) })
}
func testNodeRun(t *testing.T, enableLease bool) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
c := testclient.NewSimpleClientset()
testP := &testNodeProvider{NodeProvider: &NaiveNodeProvider{}}
nodes := c.CoreV1().Nodes()
leases := c.Coordination().Leases(corev1.NamespaceNodeLease)
interval := 1 * time.Millisecond
opts := []NodeControllerOpt{
WithNodePingInterval(interval),
WithNodeStatusUpdateInterval(interval),
}
if enableLease {
opts = append(opts, WithNodeEnableLeaseV1Beta1(leases, nil))
}
node, err := NewNodeController(testP, testNode(t), nodes, opts...)
assert.NilError(t, err)
chErr := make(chan error)
defer func() {
cancel()
assert.NilError(t, <-chErr)
}()
go func() {
chErr <- node.Run(ctx)
close(chErr)
}()
nw := makeWatch(t, nodes, node.n.Name)
defer nw.Stop()
nr := nw.ResultChan()
lw := makeWatch(t, leases, node.n.Name)
defer lw.Stop()
lr := lw.ResultChan()
var (
lBefore *coord.Lease
nodeUpdates int
leaseUpdates int
iters = 50
expectAtLeast = iters / 5
)
timeout := time.After(30 * time.Second)
for i := 0; i < iters; i++ {
var l *coord.Lease
select {
case <-timeout:
t.Fatal("timed out waiting for expected events")
case <-time.After(time.Second):
t.Errorf("timeout waiting for event")
continue
case err := <-chErr:
t.Fatal(err) // if this returns at all it is an error regardless if err is nil
case <-nr:
nodeUpdates++
continue
case le := <-lr:
l = le.Object.(*coord.Lease)
leaseUpdates++
assert.Assert(t, cmp.Equal(l.Spec.HolderIdentity != nil, true))
assert.Check(t, cmp.Equal(*l.Spec.HolderIdentity, node.n.Name))
if lBefore != nil {
assert.Check(t, before(lBefore.Spec.RenewTime.Time, l.Spec.RenewTime.Time))
}
lBefore = l
}
}
lw.Stop()
nw.Stop()
assert.Check(t, atLeast(nodeUpdates, expectAtLeast))
if enableLease {
assert.Check(t, atLeast(leaseUpdates, expectAtLeast))
} else {
assert.Check(t, cmp.Equal(leaseUpdates, 0))
}
// trigger an async node status update
n := node.n.DeepCopy()
newCondition := corev1.NodeCondition{
Type: corev1.NodeConditionType("UPDATED"),
LastTransitionTime: metav1.Now().Rfc3339Copy(),
}
n.Status.Conditions = append(n.Status.Conditions, newCondition)
nw = makeWatch(t, nodes, node.n.Name)
defer nw.Stop()
nr = nw.ResultChan()
testP.triggerStatusUpdate(n)
eCtx, eCancel := context.WithTimeout(ctx, 10*time.Second)
defer eCancel()
select {
case err := <-chErr:
t.Fatal(err) // if this returns at all it is an error regardless if err is nil
case err := <-waitForEvent(eCtx, nr, func(e watch.Event) bool {
node := e.Object.(*corev1.Node)
if len(node.Status.Conditions) == 0 {
return false
}
// Check if this is a node update we are looking for
// Since node updates happen periodically there could be some that occur
// before the status update that we are looking for happens.
c := node.Status.Conditions[len(n.Status.Conditions)-1]
if !c.LastTransitionTime.Equal(&newCondition.LastTransitionTime) {
return false
}
if c.Type != newCondition.Type {
return false
}
return true
}):
assert.NilError(t, err, "error waiting for updated node condition")
}
}
func TestNodeCustomUpdateStatusErrorHandler(t *testing.T) {
c := testclient.NewSimpleClientset()
testP := &testNodeProvider{NodeProvider: &NaiveNodeProvider{}}
nodes := c.CoreV1().Nodes()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
node, err := NewNodeController(testP, testNode(t), nodes,
WithNodeStatusUpdateErrorHandler(func(_ context.Context, err error) error {
cancel()
return nil
}),
)
assert.NilError(t, err)
chErr := make(chan error, 1)
go func() {
chErr <- node.Run(ctx)
}()
timer := time.NewTimer(10 * time.Second)
defer timer.Stop()
// wait for the node to be ready
select {
case <-timer.C:
t.Fatal("timeout waiting for node to be ready")
case <-chErr:
t.Fatalf("node.Run returned earlier than expected: %v", err)
case <-node.Ready():
}
err = nodes.Delete(node.n.Name, nil)
assert.NilError(t, err)
testP.triggerStatusUpdate(node.n.DeepCopy())
timer = time.NewTimer(10 * time.Second)
defer timer.Stop()
select {
case err := <-chErr:
assert.Equal(t, err, nil)
case <-timer.C:
t.Fatal("timeout waiting for node shutdown")
}
}
func TestEnsureLease(t *testing.T) {
c := testclient.NewSimpleClientset().Coordination().Leases(corev1.NamespaceNodeLease)
n := testNode(t)
ctx := context.Background()
lease := newLease(nil)
setLeaseAttrs(lease, n, 1*time.Second)
l1, err := ensureLease(ctx, c, lease.DeepCopy())
assert.NilError(t, err)
assert.Check(t, timeEqual(l1.Spec.RenewTime.Time, lease.Spec.RenewTime.Time))
l1.Spec.RenewTime.Time = time.Now().Add(1 * time.Second)
l2, err := ensureLease(ctx, c, l1.DeepCopy())
assert.NilError(t, err)
assert.Check(t, timeEqual(l2.Spec.RenewTime.Time, l1.Spec.RenewTime.Time))
}
func TestUpdateNodeStatus(t *testing.T) {
n := testNode(t)
n.Status.Conditions = append(n.Status.Conditions, corev1.NodeCondition{
LastHeartbeatTime: metav1.Now().Rfc3339Copy(),
})
n.Status.Phase = corev1.NodePending
nodes := testclient.NewSimpleClientset().CoreV1().Nodes()
ctx := context.Background()
updated, err := UpdateNodeStatus(ctx, nodes, n.DeepCopy())
assert.Equal(t, errors.IsNotFound(err), true, err)
_, err = nodes.Create(n)
assert.NilError(t, err)
updated, err = UpdateNodeStatus(ctx, nodes, n.DeepCopy())
assert.NilError(t, err)
assert.NilError(t, err)
assert.Check(t, cmp.DeepEqual(n.Status, updated.Status))
n.Status.Phase = corev1.NodeRunning
updated, err = UpdateNodeStatus(ctx, nodes, n.DeepCopy())
assert.NilError(t, err)
assert.Check(t, cmp.DeepEqual(n.Status, updated.Status))
err = nodes.Delete(n.Name, nil)
assert.NilError(t, err)
_, err = nodes.Get(n.Name, metav1.GetOptions{})
assert.Equal(t, errors.IsNotFound(err), true, err)
_, err = UpdateNodeStatus(ctx, nodes, updated.DeepCopy())
assert.Equal(t, errors.IsNotFound(err), true, err)
}
func TestUpdateNodeLease(t *testing.T) {
leases := testclient.NewSimpleClientset().Coordination().Leases(corev1.NamespaceNodeLease)
lease := newLease(nil)
n := testNode(t)
setLeaseAttrs(lease, n, 0)
ctx := context.Background()
l, err := UpdateNodeLease(ctx, leases, lease)
assert.NilError(t, err)
assert.Equal(t, l.Name, lease.Name)
assert.Assert(t, cmp.DeepEqual(l.Spec.HolderIdentity, lease.Spec.HolderIdentity))
compare, err := leases.Get(l.Name, emptyGetOptions)
assert.NilError(t, err)
assert.Equal(t, l.Spec.RenewTime.Time.Unix(), compare.Spec.RenewTime.Time.Unix())
assert.Equal(t, compare.Name, lease.Name)
assert.Assert(t, cmp.DeepEqual(compare.Spec.HolderIdentity, lease.Spec.HolderIdentity))
l.Spec.RenewTime.Time = time.Now().Add(10 * time.Second)
compare, err = UpdateNodeLease(ctx, leases, l.DeepCopy())
assert.NilError(t, err)
assert.Equal(t, compare.Spec.RenewTime.Time.Unix(), l.Spec.RenewTime.Time.Unix())
assert.Equal(t, compare.Name, lease.Name)
assert.Assert(t, cmp.DeepEqual(compare.Spec.HolderIdentity, lease.Spec.HolderIdentity))
}
func testNode(t *testing.T) *corev1.Node {
n := &corev1.Node{}
n.Name = strings.ToLower(t.Name())
return n
}
type testNodeProvider struct {
NodeProvider
statusHandlers []func(*corev1.Node)
}
func (p *testNodeProvider) NotifyNodeStatus(ctx context.Context, h func(*corev1.Node)) {
p.statusHandlers = append(p.statusHandlers, h)
}
func (p *testNodeProvider) triggerStatusUpdate(n *corev1.Node) {
for _, h := range p.statusHandlers {
h(n)
}
}
type watchGetter interface {
Watch(metav1.ListOptions) (watch.Interface, error)
}
func makeWatch(t *testing.T, wc watchGetter, name string) watch.Interface {
t.Helper()
w, err := wc.Watch(metav1.ListOptions{FieldSelector: "name=" + name})
assert.NilError(t, err)
return w
}
func atLeast(x, atLeast int) cmp.Comparison {
return func() cmp.Result {
if x < atLeast {
return cmp.ResultFailureTemplate(failTemplate("<"), map[string]interface{}{"x": x, "y": atLeast})
}
return cmp.ResultSuccess
}
}
func before(x, y time.Time) cmp.Comparison {
return func() cmp.Result {
if x.Before(y) {
return cmp.ResultSuccess
}
return cmp.ResultFailureTemplate(failTemplate(">="), map[string]interface{}{"x": x, "y": y})
}
}
func timeEqual(x, y time.Time) cmp.Comparison {
return func() cmp.Result {
if x.Equal(y) {
return cmp.ResultSuccess
}
return cmp.ResultFailureTemplate(failTemplate("!="), map[string]interface{}{"x": x, "y": y})
}
}
// waitForEvent waits for the `check` function to return true
// `check` is run when an event is received
// Cancelling the context will cancel the wait, with the context error sent on
// the returned channel.
func waitForEvent(ctx context.Context, chEvent <-chan watch.Event, check func(watch.Event) bool) <-chan error {
chErr := make(chan error, 1)
go func() {
for {
select {
case e := <-chEvent:
if check(e) {
chErr <- nil
return
}
case <-ctx.Done():
chErr <- ctx.Err()
return
}
}
}()
return chErr
}
func failTemplate(op string) string {
return `
{{- .Data.x}} (
{{- with callArg 0 }}{{ formatNode . }} {{end -}}
{{- printf "%T" .Data.x -}}
) ` + op + ` {{ .Data.y}} (
{{- with callArg 1 }}{{ formatNode . }} {{end -}}
{{- printf "%T" .Data.y -}}
)`
}

303
node/pod.go Normal file
View File

@@ -0,0 +1,303 @@
// Copyright © 2017 The virtual-kubelet authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package node
import (
"context"
"hash/fnv"
"time"
"github.com/davecgh/go-spew/spew"
pkgerrors "github.com/pkg/errors"
"github.com/virtual-kubelet/virtual-kubelet/errdefs"
"github.com/virtual-kubelet/virtual-kubelet/log"
"github.com/virtual-kubelet/virtual-kubelet/trace"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/util/workqueue"
)
const (
podStatusReasonProviderFailed = "ProviderFailed"
)
func addPodAttributes(ctx context.Context, span trace.Span, pod *corev1.Pod) context.Context {
return span.WithFields(ctx, log.Fields{
"uid": string(pod.GetUID()),
"namespace": pod.GetNamespace(),
"name": pod.GetName(),
"phase": string(pod.Status.Phase),
"reason": pod.Status.Reason,
})
}
func (pc *PodController) createOrUpdatePod(ctx context.Context, pod *corev1.Pod) error {
ctx, span := trace.StartSpan(ctx, "createOrUpdatePod")
defer span.End()
addPodAttributes(ctx, span, pod)
ctx = span.WithFields(ctx, log.Fields{
"pod": pod.GetName(),
"namespace": pod.GetNamespace(),
})
if err := populateEnvironmentVariables(ctx, pod, pc.resourceManager, pc.recorder); err != nil {
span.SetStatus(err)
return err
}
// Check if the pod is already known by the provider.
// NOTE: Some providers return a non-nil error in their GetPod implementation when the pod is not found while some other don't.
// Hence, we ignore the error and just act upon the pod if it is non-nil (meaning that the provider still knows about the pod).
if pp, _ := pc.provider.GetPod(ctx, pod.Namespace, pod.Name); pp != nil {
// Pod Update Only Permits update of:
// - `spec.containers[*].image`
// - `spec.initContainers[*].image`
// - `spec.activeDeadlineSeconds`
// - `spec.tolerations` (only additions to existing tolerations)
// compare the hashes of the pod specs to see if the specs actually changed
expected := hashPodSpec(pp.Spec)
if actual := hashPodSpec(pod.Spec); actual != expected {
log.G(ctx).Debugf("Pod %s exists, updating pod in provider", pp.Name)
if origErr := pc.provider.UpdatePod(ctx, pod); origErr != nil {
pc.handleProviderError(ctx, span, origErr, pod)
return origErr
}
log.G(ctx).Info("Updated pod in provider")
}
} else {
if origErr := pc.provider.CreatePod(ctx, pod); origErr != nil {
pc.handleProviderError(ctx, span, origErr, pod)
return origErr
}
log.G(ctx).Info("Created pod in provider")
}
return nil
}
// This is basically the kube runtime's hash container functionality.
// VK only operates at the Pod level so this is adapted for that
func hashPodSpec(spec corev1.PodSpec) uint64 {
hash := fnv.New32a()
printer := spew.ConfigState{
Indent: " ",
SortKeys: true,
DisableMethods: true,
SpewKeys: true,
}
printer.Fprintf(hash, "%#v", spec)
return uint64(hash.Sum32())
}
func (pc *PodController) handleProviderError(ctx context.Context, span trace.Span, origErr error, pod *corev1.Pod) {
podPhase := corev1.PodPending
if pod.Spec.RestartPolicy == corev1.RestartPolicyNever {
podPhase = corev1.PodFailed
}
pod.ResourceVersion = "" // Blank out resource version to prevent object has been modified error
pod.Status.Phase = podPhase
pod.Status.Reason = podStatusReasonProviderFailed
pod.Status.Message = origErr.Error()
logger := log.G(ctx).WithFields(log.Fields{
"podPhase": podPhase,
"reason": pod.Status.Reason,
})
_, err := pc.client.Pods(pod.Namespace).UpdateStatus(pod)
if err != nil {
logger.WithError(err).Warn("Failed to update pod status")
} else {
logger.Info("Updated k8s pod status")
}
span.SetStatus(origErr)
}
func (pc *PodController) deletePod(ctx context.Context, namespace, name string) error {
// Grab the pod as known by the provider.
// NOTE: Some providers return a non-nil error in their GetPod implementation when the pod is not found while some other don't.
// Hence, we ignore the error and just act upon the pod if it is non-nil (meaning that the provider still knows about the pod).
pod, _ := pc.provider.GetPod(ctx, namespace, name)
if pod == nil {
// The provider is not aware of the pod, but we must still delete the Kubernetes API resource.
return pc.forceDeletePodResource(ctx, namespace, name)
}
ctx, span := trace.StartSpan(ctx, "deletePod")
defer span.End()
ctx = addPodAttributes(ctx, span, pod)
var delErr error
if delErr = pc.provider.DeletePod(ctx, pod); delErr != nil && errors.IsNotFound(delErr) {
span.SetStatus(delErr)
return delErr
}
log.G(ctx).Debug("Deleted pod from provider")
if !errors.IsNotFound(delErr) {
if err := pc.forceDeletePodResource(ctx, namespace, name); err != nil {
span.SetStatus(err)
return err
}
log.G(ctx).Info("Deleted pod from Kubernetes")
}
return nil
}
func (pc *PodController) forceDeletePodResource(ctx context.Context, namespace, name string) error {
ctx, span := trace.StartSpan(ctx, "forceDeletePodResource")
defer span.End()
ctx = span.WithFields(ctx, log.Fields{
"namespace": namespace,
"name": name,
})
var grace int64
if err := pc.client.Pods(namespace).Delete(name, &metav1.DeleteOptions{GracePeriodSeconds: &grace}); err != nil {
if errors.IsNotFound(err) {
log.G(ctx).Debug("Pod does not exist in Kubernetes, nothing to delete")
return nil
}
span.SetStatus(err)
return pkgerrors.Wrap(err, "Failed to delete Kubernetes pod")
}
return nil
}
// updatePodStatuses syncs the providers pod status with the kubernetes pod status.
func (pc *PodController) updatePodStatuses(ctx context.Context, q workqueue.RateLimitingInterface) {
ctx, span := trace.StartSpan(ctx, "updatePodStatuses")
defer span.End()
// Update all the pods with the provider status.
pods, err := pc.podsLister.List(labels.Everything())
if err != nil {
err = pkgerrors.Wrap(err, "error getting pod list")
span.SetStatus(err)
log.G(ctx).WithError(err).Error("Error updating pod statuses")
return
}
ctx = span.WithField(ctx, "nPods", int64(len(pods)))
for _, pod := range pods {
if !shouldSkipPodStatusUpdate(pod) {
enqueuePodStatusUpdate(ctx, q, pod)
}
}
}
func shouldSkipPodStatusUpdate(pod *corev1.Pod) bool {
return pod.Status.Phase == corev1.PodSucceeded ||
pod.Status.Phase == corev1.PodFailed ||
pod.Status.Reason == podStatusReasonProviderFailed
}
func (pc *PodController) updatePodStatus(ctx context.Context, pod *corev1.Pod) error {
if shouldSkipPodStatusUpdate(pod) {
return nil
}
ctx, span := trace.StartSpan(ctx, "updatePodStatus")
defer span.End()
ctx = addPodAttributes(ctx, span, pod)
status, err := pc.provider.GetPodStatus(ctx, pod.Namespace, pod.Name)
if err != nil && !errdefs.IsNotFound(err) {
span.SetStatus(err)
return pkgerrors.Wrap(err, "error retreiving pod status")
}
// Update the pod's status
if status != nil {
pod.Status = *status
} else {
// Only change the status when the pod was already up
// Only doing so when the pod was successfully running makes sure we don't run into race conditions during pod creation.
if pod.Status.Phase == corev1.PodRunning || pod.ObjectMeta.CreationTimestamp.Add(time.Minute).Before(time.Now()) {
// Set the pod to failed, this makes sure if the underlying container implementation is gone that a new pod will be created.
pod.Status.Phase = corev1.PodFailed
pod.Status.Reason = "NotFound"
pod.Status.Message = "The pod status was not found and may have been deleted from the provider"
for i, c := range pod.Status.ContainerStatuses {
pod.Status.ContainerStatuses[i].State.Terminated = &corev1.ContainerStateTerminated{
ExitCode: -137,
Reason: "NotFound",
Message: "Container was not found and was likely deleted",
FinishedAt: metav1.NewTime(time.Now()),
StartedAt: c.State.Running.StartedAt,
ContainerID: c.ContainerID,
}
pod.Status.ContainerStatuses[i].State.Running = nil
}
}
}
if _, err := pc.client.Pods(pod.Namespace).UpdateStatus(pod); err != nil {
span.SetStatus(err)
return pkgerrors.Wrap(err, "error while updating pod status in kubernetes")
}
log.G(ctx).WithFields(log.Fields{
"new phase": string(pod.Status.Phase),
"new reason": pod.Status.Reason,
}).Debug("Updated pod status in kubernetes")
return nil
}
func enqueuePodStatusUpdate(ctx context.Context, q workqueue.RateLimitingInterface, pod *corev1.Pod) {
if key, err := cache.MetaNamespaceKeyFunc(pod); err != nil {
log.G(ctx).WithError(err).WithField("method", "enqueuePodStatusUpdate").Error("Error getting pod meta namespace key")
} else {
q.AddRateLimited(key)
}
}
func (pc *PodController) podStatusHandler(ctx context.Context, key string) (retErr error) {
ctx, span := trace.StartSpan(ctx, "podStatusHandler")
defer span.End()
ctx = span.WithField(ctx, "key", key)
log.G(ctx).Debug("processing pod status update")
defer func() {
span.SetStatus(retErr)
if retErr != nil {
log.G(ctx).WithError(retErr).Error("Error processing pod status update")
}
}()
namespace, name, err := cache.SplitMetaNamespaceKey(key)
if err != nil {
return pkgerrors.Wrap(err, "error spliting cache key")
}
pod, err := pc.podsLister.Pods(namespace).Get(name)
if err != nil {
if errors.IsNotFound(err) {
log.G(ctx).WithError(err).Debug("Skipping pod status update for pod missing in Kubernetes")
return nil
}
return pkgerrors.Wrap(err, "error looking up pod")
}
return pc.updatePodStatus(ctx, pod)
}

295
node/pod_test.go Normal file
View File

@@ -0,0 +1,295 @@
// Copyright © 2017 The virtual-kubelet authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package node
import (
"context"
"path"
"testing"
"github.com/virtual-kubelet/virtual-kubelet/errdefs"
testutil "github.com/virtual-kubelet/virtual-kubelet/internal/test/util"
"gotest.tools/assert"
is "gotest.tools/assert/cmp"
corev1 "k8s.io/api/core/v1"
"k8s.io/client-go/kubernetes/fake"
)
type mockProvider struct {
pods map[string]*corev1.Pod
creates int
updates int
deletes int
}
func (m *mockProvider) CreatePod(ctx context.Context, pod *corev1.Pod) error {
m.pods[path.Join(pod.GetNamespace(), pod.GetName())] = pod
m.creates++
return nil
}
func (m *mockProvider) UpdatePod(ctx context.Context, pod *corev1.Pod) error {
m.pods[path.Join(pod.GetNamespace(), pod.GetName())] = pod
m.updates++
return nil
}
func (m *mockProvider) GetPod(ctx context.Context, namespace, name string) (*corev1.Pod, error) {
p := m.pods[path.Join(namespace, name)]
if p == nil {
return nil, errdefs.NotFound("not found")
}
return p, nil
}
func (m *mockProvider) GetPodStatus(ctx context.Context, namespace, name string) (*corev1.PodStatus, error) {
p := m.pods[path.Join(namespace, name)]
if p == nil {
return nil, errdefs.NotFound("not found")
}
return &p.Status, nil
}
func (m *mockProvider) DeletePod(ctx context.Context, p *corev1.Pod) error {
delete(m.pods, path.Join(p.GetNamespace(), p.GetName()))
m.deletes++
return nil
}
func (m *mockProvider) GetPods(_ context.Context) ([]*corev1.Pod, error) {
ls := make([]*corev1.Pod, 0, len(m.pods))
for _, p := range ls {
ls = append(ls, p)
}
return ls, nil
}
type TestController struct {
*PodController
mock *mockProvider
client *fake.Clientset
}
func newMockProvider() *mockProvider {
return &mockProvider{pods: make(map[string]*corev1.Pod)}
}
func newTestController() *TestController {
fk8s := fake.NewSimpleClientset()
rm := testutil.FakeResourceManager()
p := newMockProvider()
return &TestController{
PodController: &PodController{
client: fk8s.CoreV1(),
provider: p,
resourceManager: rm,
recorder: testutil.FakeEventRecorder(5),
},
mock: p,
client: fk8s,
}
}
func TestPodHashingEqual(t *testing.T) {
p1 := corev1.PodSpec{
Containers: []corev1.Container{
corev1.Container{
Name: "nginx",
Image: "nginx:1.15.12-perl",
Ports: []corev1.ContainerPort{
corev1.ContainerPort{
ContainerPort: 443,
Protocol: "tcp",
},
},
},
},
}
h1 := hashPodSpec(p1)
p2 := corev1.PodSpec{
Containers: []corev1.Container{
corev1.Container{
Name: "nginx",
Image: "nginx:1.15.12-perl",
Ports: []corev1.ContainerPort{
corev1.ContainerPort{
ContainerPort: 443,
Protocol: "tcp",
},
},
},
},
}
h2 := hashPodSpec(p2)
assert.Check(t, is.Equal(h1, h2))
}
func TestPodHashingDifferent(t *testing.T) {
p1 := corev1.PodSpec{
Containers: []corev1.Container{
corev1.Container{
Name: "nginx",
Image: "nginx:1.15.12",
Ports: []corev1.ContainerPort{
corev1.ContainerPort{
ContainerPort: 443,
Protocol: "tcp",
},
},
},
},
}
h1 := hashPodSpec(p1)
p2 := corev1.PodSpec{
Containers: []corev1.Container{
corev1.Container{
Name: "nginx",
Image: "nginx:1.15.12-perl",
Ports: []corev1.ContainerPort{
corev1.ContainerPort{
ContainerPort: 443,
Protocol: "tcp",
},
},
},
},
}
h2 := hashPodSpec(p2)
assert.Check(t, h1 != h2)
}
func TestPodCreateNewPod(t *testing.T) {
svr := newTestController()
pod := &corev1.Pod{}
pod.ObjectMeta.Namespace = "default"
pod.ObjectMeta.Name = "nginx"
pod.Spec = corev1.PodSpec{
Containers: []corev1.Container{
corev1.Container{
Name: "nginx",
Image: "nginx:1.15.12",
Ports: []corev1.ContainerPort{
corev1.ContainerPort{
ContainerPort: 443,
Protocol: "tcp",
},
},
},
},
}
err := svr.createOrUpdatePod(context.Background(), pod)
assert.Check(t, is.Nil(err))
// createOrUpdate called CreatePod but did not call UpdatePod because the pod did not exist
assert.Check(t, is.Equal(svr.mock.creates, 1))
assert.Check(t, is.Equal(svr.mock.updates, 0))
}
func TestPodUpdateExisting(t *testing.T) {
svr := newTestController()
pod := &corev1.Pod{}
pod.ObjectMeta.Namespace = "default"
pod.ObjectMeta.Name = "nginx"
pod.Spec = corev1.PodSpec{
Containers: []corev1.Container{
corev1.Container{
Name: "nginx",
Image: "nginx:1.15.12",
Ports: []corev1.ContainerPort{
corev1.ContainerPort{
ContainerPort: 443,
Protocol: "tcp",
},
},
},
},
}
err := svr.provider.CreatePod(context.Background(), pod)
assert.Check(t, is.Nil(err))
assert.Check(t, is.Equal(svr.mock.creates, 1))
assert.Check(t, is.Equal(svr.mock.updates, 0))
pod2 := &corev1.Pod{}
pod2.ObjectMeta.Namespace = "default"
pod2.ObjectMeta.Name = "nginx"
pod2.Spec = corev1.PodSpec{
Containers: []corev1.Container{
corev1.Container{
Name: "nginx",
Image: "nginx:1.15.12-perl",
Ports: []corev1.ContainerPort{
corev1.ContainerPort{
ContainerPort: 443,
Protocol: "tcp",
},
},
},
},
}
err = svr.createOrUpdatePod(context.Background(), pod2)
assert.Check(t, is.Nil(err))
// createOrUpdate didn't call CreatePod but did call UpdatePod because the spec changed
assert.Check(t, is.Equal(svr.mock.creates, 1))
assert.Check(t, is.Equal(svr.mock.updates, 1))
}
func TestPodNoSpecChange(t *testing.T) {
svr := newTestController()
pod := &corev1.Pod{}
pod.ObjectMeta.Namespace = "default"
pod.ObjectMeta.Name = "nginx"
pod.Spec = corev1.PodSpec{
Containers: []corev1.Container{
corev1.Container{
Name: "nginx",
Image: "nginx:1.15.12",
Ports: []corev1.ContainerPort{
corev1.ContainerPort{
ContainerPort: 443,
Protocol: "tcp",
},
},
},
},
}
err := svr.mock.CreatePod(context.Background(), pod)
assert.Check(t, is.Nil(err))
assert.Check(t, is.Equal(svr.mock.creates, 1))
assert.Check(t, is.Equal(svr.mock.updates, 0))
err = svr.createOrUpdatePod(context.Background(), pod)
assert.Check(t, is.Nil(err))
// createOrUpdate didn't call CreatePod or UpdatePod, spec didn't change
assert.Check(t, is.Equal(svr.mock.creates, 1))
assert.Check(t, is.Equal(svr.mock.updates, 0))
}

View File

@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package vkubelet
package node
import (
"context"
@@ -22,61 +22,149 @@ import (
"sync"
"time"
"github.com/cpuguy83/strongerrors/status/ocstatus"
pkgerrors "github.com/pkg/errors"
"go.opencensus.io/trace"
"github.com/virtual-kubelet/virtual-kubelet/errdefs"
"github.com/virtual-kubelet/virtual-kubelet/log"
"github.com/virtual-kubelet/virtual-kubelet/manager"
"github.com/virtual-kubelet/virtual-kubelet/trace"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/informers/core/v1"
"k8s.io/client-go/kubernetes/scheme"
typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1"
corev1informers "k8s.io/client-go/informers/core/v1"
corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
corev1listers "k8s.io/client-go/listers/core/v1"
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/tools/record"
"k8s.io/client-go/util/workqueue"
"github.com/virtual-kubelet/virtual-kubelet/log"
)
const (
// maxRetries is the number of times we try to process a given key before permanently forgetting it.
maxRetries = 20
)
// PodLifecycleHandler defines the interface used by the PodController to react
// to new and changed pods scheduled to the node that is being managed.
//
// Errors produced by these methods should implement an interface from
// github.com/virtual-kubelet/virtual-kubelet/errdefs package in order for the
// core logic to be able to understand the type of failure.
type PodLifecycleHandler interface {
// CreatePod takes a Kubernetes Pod and deploys it within the provider.
CreatePod(ctx context.Context, pod *corev1.Pod) error
// UpdatePod takes a Kubernetes Pod and updates it within the provider.
UpdatePod(ctx context.Context, pod *corev1.Pod) error
// DeletePod takes a Kubernetes Pod and deletes it from the provider.
DeletePod(ctx context.Context, pod *corev1.Pod) error
// GetPod retrieves a pod by name from the provider (can be cached).
GetPod(ctx context.Context, namespace, name string) (*corev1.Pod, error)
// GetPodStatus retrieves the status of a pod by name from the provider.
GetPodStatus(ctx context.Context, namespace, name string) (*corev1.PodStatus, error)
// GetPods retrieves a list of all pods running on the provider (can be cached).
GetPods(context.Context) ([]*corev1.Pod, error)
}
// PodNotifier notifies callers of pod changes.
// Providers should implement this interface to enable callers to be notified
// of pod status updates asyncronously.
type PodNotifier interface {
// NotifyPods instructs the notifier to call the passed in function when
// the pod status changes.
//
// NotifyPods should not block callers.
NotifyPods(context.Context, func(*corev1.Pod))
}
// PodController is the controller implementation for Pod resources.
type PodController struct {
// server is the instance to which this controller belongs.
server *Server
provider PodLifecycleHandler
// podsInformer is an informer for Pod resources.
podsInformer v1.PodInformer
podsInformer corev1informers.PodInformer
// podsLister is able to list/get Pod resources from a shared informer's store.
podsLister corev1listers.PodLister
// workqueue is a rate limited work queue.
// This is used to queue work to be processed instead of performing it as soon as a change happens.
// This means we can ensure we only process a fixed amount of resources at a time, and makes it easy to ensure we are never processing the same item simultaneously in two different workers.
workqueue workqueue.RateLimitingInterface
// recorder is an event recorder for recording Event resources to the Kubernetes API.
recorder record.EventRecorder
// ready is a channel which will be closed once the pod controller is fully up and running.
// this channel will never be closed if there is an error on startup.
ready chan struct{}
client corev1client.PodsGetter
resourceManager *manager.ResourceManager
}
// NewPodController returns a new instance of PodController.
func NewPodController(server *Server) *PodController {
// Create an event broadcaster.
eventBroadcaster := record.NewBroadcaster()
eventBroadcaster.StartLogging(log.L.Infof)
eventBroadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: server.k8sClient.CoreV1().Events("")})
recorder := eventBroadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{Component: fmt.Sprintf("%s/pod-controller", server.nodeName)})
// PodControllerConfig is used to configure a new PodController.
type PodControllerConfig struct {
// PodClient is used to perform actions on the k8s API, such as updating pod status
// This field is required
PodClient corev1client.PodsGetter
// Create an instance of PodController having a work queue that uses the rate limiter created above.
pc := &PodController{
server: server,
podsInformer: server.podInformer,
podsLister: server.podInformer.Lister(),
workqueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "pods"),
recorder: recorder,
// PodInformer is used as a local cache for pods
// This should be configured to only look at pods scheduled to the node which the controller will be managing
PodInformer corev1informers.PodInformer
EventRecorder record.EventRecorder
Provider PodLifecycleHandler
// Informers used for filling details for things like downward API in pod spec.
//
// We are using informers here instead of listers because we'll need the
// informer for certain features (like notifications for updated ConfigMaps)
ConfigMapInformer corev1informers.ConfigMapInformer
SecretInformer corev1informers.SecretInformer
ServiceInformer corev1informers.ServiceInformer
}
func NewPodController(cfg PodControllerConfig) (*PodController, error) {
if cfg.PodClient == nil {
return nil, errdefs.InvalidInput("missing core client")
}
if cfg.EventRecorder == nil {
return nil, errdefs.InvalidInput("missing event recorder")
}
if cfg.PodInformer == nil {
return nil, errdefs.InvalidInput("missing pod informer")
}
if cfg.ConfigMapInformer == nil {
return nil, errdefs.InvalidInput("missing config map informer")
}
if cfg.SecretInformer == nil {
return nil, errdefs.InvalidInput("missing secret informer")
}
if cfg.ServiceInformer == nil {
return nil, errdefs.InvalidInput("missing service informer")
}
rm, err := manager.NewResourceManager(cfg.PodInformer.Lister(), cfg.SecretInformer.Lister(), cfg.ConfigMapInformer.Lister(), cfg.ServiceInformer.Lister())
if err != nil {
return nil, pkgerrors.Wrap(err, "could not create resource manager")
}
return &PodController{
client: cfg.PodClient,
podsInformer: cfg.PodInformer,
podsLister: cfg.PodInformer.Lister(),
provider: cfg.Provider,
resourceManager: rm,
ready: make(chan struct{}),
recorder: cfg.EventRecorder,
}, nil
}
// Run will set up the event handlers for types we are interested in, as well as syncing informer caches and starting workers.
// It will block until the context is cancelled, at which point it will shutdown the work queue and wait for workers to finish processing their current work items.
func (pc *PodController) Run(ctx context.Context, podSyncWorkers int) error {
k8sQ := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "syncPodsFromKubernetes")
defer k8sQ.ShutDown()
podStatusQueue := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "syncPodStatusFromProvider")
pc.runProviderSyncWorkers(ctx, podStatusQueue, podSyncWorkers)
pc.runSyncFromProvider(ctx, podStatusQueue)
defer podStatusQueue.ShutDown()
// Set up event handlers for when Pod resources change.
pc.podsInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
@@ -84,7 +172,7 @@ func NewPodController(server *Server) *PodController {
if key, err := cache.MetaNamespaceKeyFunc(pod); err != nil {
log.L.Error(err)
} else {
pc.workqueue.AddRateLimited(key)
k8sQ.AddRateLimited(key)
}
},
UpdateFunc: func(oldObj, newObj interface{}) {
@@ -103,46 +191,39 @@ func NewPodController(server *Server) *PodController {
if key, err := cache.MetaNamespaceKeyFunc(newPod); err != nil {
log.L.Error(err)
} else {
pc.workqueue.AddRateLimited(key)
k8sQ.AddRateLimited(key)
}
},
DeleteFunc: func(pod interface{}) {
if key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(pod); err != nil {
log.L.Error(err)
} else {
pc.workqueue.AddRateLimited(key)
k8sQ.AddRateLimited(key)
}
},
})
// Return the instance of PodController back to the caller.
return pc
}
// Run will set up the event handlers for types we are interested in, as well as syncing informer caches and starting workers.
// It will block until stopCh is closed, at which point it will shutdown the work queue and wait for workers to finish processing their current work items.
func (pc *PodController) Run(ctx context.Context, threadiness int) error {
defer pc.workqueue.ShutDown()
// Wait for the caches to be synced before starting workers.
// Wait for the caches to be synced *before* starting workers.
if ok := cache.WaitForCacheSync(ctx.Done(), pc.podsInformer.Informer().HasSynced); !ok {
return pkgerrors.New("failed to wait for caches to sync")
}
log.G(ctx).Info("Pod cache in-sync")
// Perform a reconciliation step that deletes any dangling pods from the provider.
// This happens only when the virtual-kubelet is starting, and operates on a "best-effort" basis.
// If by any reason the provider fails to delete a dangling pod, it will stay in the provider and deletion won't be retried.
pc.deleteDanglingPods(ctx, threadiness)
pc.deleteDanglingPods(ctx, podSyncWorkers)
// Launch "threadiness" workers to process Pod resources.
log.G(ctx).Info("starting workers")
for id := 0; id < threadiness; id++ {
for id := 0; id < podSyncWorkers; id++ {
go wait.Until(func() {
// Use the worker's "index" as its ID so we can use it for tracing.
pc.runWorker(ctx, strconv.Itoa(id))
pc.runWorker(ctx, strconv.Itoa(id), k8sQ)
}, time.Second, ctx.Done())
}
close(pc.ready)
log.G(ctx).Info("started workers")
<-ctx.Done()
log.G(ctx).Info("shutting down workers")
@@ -150,72 +231,29 @@ func (pc *PodController) Run(ctx context.Context, threadiness int) error {
return nil
}
// Ready returns a channel which gets closed once the PodController is ready to handle scheduled pods.
// This channel will never close if there is an error on startup.
// The status of this channel after sthudown is indeterminate.
func (pc *PodController) Ready() <-chan struct{} {
return pc.ready
}
// runWorker is a long-running function that will continually call the processNextWorkItem function in order to read and process an item on the work queue.
func (pc *PodController) runWorker(ctx context.Context, workerId string) {
for pc.processNextWorkItem(ctx, workerId) {
func (pc *PodController) runWorker(ctx context.Context, workerId string, q workqueue.RateLimitingInterface) {
for pc.processNextWorkItem(ctx, workerId, q) {
}
}
// processNextWorkItem will read a single work item off the work queue and attempt to process it,by calling the syncHandler.
func (pc *PodController) processNextWorkItem(ctx context.Context, workerId string) bool {
obj, shutdown := pc.workqueue.Get()
if shutdown {
return false
}
func (pc *PodController) processNextWorkItem(ctx context.Context, workerId string, q workqueue.RateLimitingInterface) bool {
// We create a span only after popping from the queue so that we can get an adequate picture of how long it took to process the item.
ctx, span := trace.StartSpan(ctx, "processNextWorkItem")
defer span.End()
// Add the ID of the current worker as an attribute to the current span.
span.AddAttributes(trace.StringAttribute("workerId", workerId))
// We wrap this block in a func so we can defer pc.workqueue.Done.
err := func(obj interface{}) error {
// We call Done here so the work queue knows we have finished processing this item.
// We also must remember to call Forget if we do not want this work item being re-queued.
// For example, we do not call Forget if a transient error occurs.
// Instead, the item is put back on the work queue and attempted again after a back-off period.
defer pc.workqueue.Done(obj)
var key string
var ok bool
// We expect strings to come off the work queue.
// These are of the form namespace/name.
// We do this as the delayed nature of the work queue means the items in the informer cache may actually be more up to date that when the item was initially put onto the workqueue.
if key, ok = obj.(string); !ok {
// As the item in the work queue is actually invalid, we call Forget here else we'd go into a loop of attempting to process a work item that is invalid.
pc.workqueue.Forget(obj)
log.G(ctx).Warnf("expected string in work queue but got %#v", obj)
return nil
}
// Add the current key as an attribute to the current span.
span.AddAttributes(trace.StringAttribute("key", key))
// Run the syncHandler, passing it the namespace/name string of the Pod resource to be synced.
if err := pc.syncHandler(ctx, key); err != nil {
if pc.workqueue.NumRequeues(key) < maxRetries {
// Put the item back on the work queue to handle any transient errors.
log.G(ctx).Warnf("requeuing %q due to failed sync: %v", key, err)
pc.workqueue.AddRateLimited(key)
return nil
}
// We've exceeded the maximum retries, so we must forget the key.
pc.workqueue.Forget(key)
return pkgerrors.Wrapf(err, "forgetting %q due to maximum retries reached", key)
}
// Finally, if no error occurs we Forget this item so it does not get queued again until another change happens.
pc.workqueue.Forget(obj)
return nil
}(obj)
if err != nil {
// We've actually hit an error, so we set the span's status based on the error.
span.SetStatus(ocstatus.FromError(err))
log.G(ctx).Error(err)
return true
}
return true
ctx = span.WithField(ctx, "workerId", workerId)
return handleQueueItem(ctx, q, pc.syncHandler)
}
// syncHandler compares the actual state with the desired, and attempts to converge the two.
@@ -224,7 +262,7 @@ func (pc *PodController) syncHandler(ctx context.Context, key string) error {
defer span.End()
// Add the current key as an attribute to the current span.
span.AddAttributes(trace.StringAttribute("key", key))
ctx = span.WithField(ctx, "key", key)
// Convert the namespace/name string into a distinct namespace and name.
namespace, name, err := cache.SplitMetaNamespaceKey(key)
@@ -241,14 +279,14 @@ func (pc *PodController) syncHandler(ctx context.Context, key string) error {
// We've failed to fetch the pod from the lister, but the error is not a 404.
// Hence, we add the key back to the work queue so we can retry processing it later.
err := pkgerrors.Wrapf(err, "failed to fetch pod with key %q from lister", key)
span.SetStatus(ocstatus.FromError(err))
span.SetStatus(err)
return err
}
// At this point we know the Pod resource doesn't exist, which most probably means it was deleted.
// Hence, we must delete it from the provider if it still exists there.
if err := pc.server.deletePod(ctx, namespace, name); err != nil {
if err := pc.deletePod(ctx, namespace, name); err != nil {
err := pkgerrors.Wrapf(err, "failed to delete pod %q in the provider", loggablePodNameFromCoordinates(namespace, name))
span.SetStatus(ocstatus.FromError(err))
span.SetStatus(err)
return err
}
return nil
@@ -263,14 +301,14 @@ func (pc *PodController) syncPodInProvider(ctx context.Context, pod *corev1.Pod)
defer span.End()
// Add the pod's attributes to the current span.
addPodAttributes(span, pod)
ctx = addPodAttributes(ctx, span, pod)
// Check whether the pod has been marked for deletion.
// If it does, guarantee it is deleted in the provider and Kubernetes.
if pod.DeletionTimestamp != nil {
if err := pc.server.deletePod(ctx, pod.Namespace, pod.Name); err != nil {
if err := pc.deletePod(ctx, pod.Namespace, pod.Name); err != nil {
err := pkgerrors.Wrapf(err, "failed to delete pod %q in the provider", loggablePodName(pod))
span.SetStatus(ocstatus.FromError(err))
span.SetStatus(err)
return err
}
return nil
@@ -283,9 +321,9 @@ func (pc *PodController) syncPodInProvider(ctx context.Context, pod *corev1.Pod)
}
// Create or update the pod in the provider.
if err := pc.server.createOrUpdatePod(ctx, pod, pc.recorder); err != nil {
if err := pc.createOrUpdatePod(ctx, pod); err != nil {
err := pkgerrors.Wrapf(err, "failed to sync pod %q in the provider", loggablePodName(pod))
span.SetStatus(ocstatus.FromError(err))
span.SetStatus(err)
return err
}
return nil
@@ -297,10 +335,10 @@ func (pc *PodController) deleteDanglingPods(ctx context.Context, threadiness int
defer span.End()
// Grab the list of pods known to the provider.
pps, err := pc.server.provider.GetPods(ctx)
pps, err := pc.provider.GetPods(ctx)
if err != nil {
err := pkgerrors.Wrap(err, "failed to fetch the list of pods from the provider")
span.SetStatus(ocstatus.FromError(err))
span.SetStatus(err)
log.G(ctx).Error(err)
return
}
@@ -319,7 +357,7 @@ func (pc *PodController) deleteDanglingPods(ctx context.Context, threadiness int
}
// For some reason we couldn't fetch the pod from the lister, so we propagate the error.
err := pkgerrors.Wrap(err, "failed to fetch pod from the lister")
span.SetStatus(ocstatus.FromError(err))
span.SetStatus(err)
log.G(ctx).Error(err)
return
}
@@ -344,10 +382,10 @@ func (pc *PodController) deleteDanglingPods(ctx context.Context, threadiness int
}()
// Add the pod's attributes to the current span.
addPodAttributes(span, pod)
ctx = addPodAttributes(ctx, span, pod)
// Actually delete the pod.
if err := pc.server.deletePod(ctx, pod.Namespace, pod.Name); err != nil {
span.SetStatus(ocstatus.FromError(err))
if err := pc.deletePod(ctx, pod.Namespace, pod.Name); err != nil {
span.SetStatus(err)
log.G(ctx).Errorf("failed to delete pod %q in provider", loggablePodName(pod))
} else {
log.G(ctx).Infof("deleted leaked pod %q in provider", loggablePodName(pod))

154
node/queue.go Normal file
View File

@@ -0,0 +1,154 @@
// Copyright © 2017 The virtual-kubelet authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package node
import (
"context"
"strconv"
"time"
pkgerrors "github.com/pkg/errors"
"github.com/virtual-kubelet/virtual-kubelet/log"
"github.com/virtual-kubelet/virtual-kubelet/trace"
corev1 "k8s.io/api/core/v1"
"k8s.io/client-go/util/workqueue"
)
const (
// maxRetries is the number of times we try to process a given key before permanently forgetting it.
maxRetries = 20
)
type queueHandler func(ctx context.Context, key string) error
func handleQueueItem(ctx context.Context, q workqueue.RateLimitingInterface, handler queueHandler) bool {
ctx, span := trace.StartSpan(ctx, "handleQueueItem")
defer span.End()
obj, shutdown := q.Get()
if shutdown {
return false
}
log.G(ctx).Debug("Got queue object")
err := func(obj interface{}) error {
defer log.G(ctx).Debug("Processed queue item")
// We call Done here so the work queue knows we have finished processing this item.
// We also must remember to call Forget if we do not want this work item being re-queued.
// For example, we do not call Forget if a transient error occurs.
// Instead, the item is put back on the work queue and attempted again after a back-off period.
defer q.Done(obj)
var key string
var ok bool
// We expect strings to come off the work queue.
// These are of the form namespace/name.
// We do this as the delayed nature of the work queue means the items in the informer cache may actually be more up to date that when the item was initially put onto the workqueue.
if key, ok = obj.(string); !ok {
// As the item in the work queue is actually invalid, we call Forget here else we'd go into a loop of attempting to process a work item that is invalid.
q.Forget(obj)
log.G(ctx).Warnf("expected string in work queue item but got %#v", obj)
return nil
}
// Add the current key as an attribute to the current span.
ctx = span.WithField(ctx, "key", key)
// Run the syncHandler, passing it the namespace/name string of the Pod resource to be synced.
if err := handler(ctx, key); err != nil {
if q.NumRequeues(key) < maxRetries {
// Put the item back on the work queue to handle any transient errors.
log.G(ctx).WithError(err).Warnf("requeuing %q due to failed sync", key)
q.AddRateLimited(key)
return nil
}
// We've exceeded the maximum retries, so we must forget the key.
q.Forget(key)
return pkgerrors.Wrapf(err, "forgetting %q due to maximum retries reached", key)
}
// Finally, if no error occurs we Forget this item so it does not get queued again until another change happens.
q.Forget(obj)
return nil
}(obj)
if err != nil {
// We've actually hit an error, so we set the span's status based on the error.
span.SetStatus(err)
log.G(ctx).Error(err)
return true
}
return true
}
func (pc *PodController) runProviderSyncWorkers(ctx context.Context, q workqueue.RateLimitingInterface, numWorkers int) {
for i := 0; i < numWorkers; i++ {
go func(index int) {
workerID := strconv.Itoa(index)
pc.runProviderSyncWorker(ctx, workerID, q)
}(i)
}
}
func (pc *PodController) runProviderSyncWorker(ctx context.Context, workerID string, q workqueue.RateLimitingInterface) {
for pc.processPodStatusUpdate(ctx, workerID, q) {
}
}
func (pc *PodController) processPodStatusUpdate(ctx context.Context, workerID string, q workqueue.RateLimitingInterface) bool {
ctx, span := trace.StartSpan(ctx, "processPodStatusUpdate")
defer span.End()
// Add the ID of the current worker as an attribute to the current span.
ctx = span.WithField(ctx, "workerID", workerID)
return handleQueueItem(ctx, q, pc.podStatusHandler)
}
// providerSyncLoop syncronizes pod states from the provider back to kubernetes
// Deprecated: This is only used when the provider does not support async updates
// Providers should implement async update support, even if it just means copying
// something like this in.
func (pc *PodController) providerSyncLoop(ctx context.Context, q workqueue.RateLimitingInterface) {
const sleepTime = 5 * time.Second
t := time.NewTimer(sleepTime)
defer t.Stop()
for {
select {
case <-ctx.Done():
return
case <-t.C:
t.Stop()
ctx, span := trace.StartSpan(ctx, "syncActualState")
pc.updatePodStatuses(ctx, q)
span.End()
// restart the timer
t.Reset(sleepTime)
}
}
}
func (pc *PodController) runSyncFromProvider(ctx context.Context, q workqueue.RateLimitingInterface) {
if pn, ok := pc.provider.(PodNotifier); ok {
pn.NotifyPods(ctx, func(pod *corev1.Pod) {
enqueuePodStatusUpdate(ctx, q, pod)
})
} else {
go pc.providerSyncLoop(ctx, q)
}
}

View File

@@ -1,57 +0,0 @@
# Alibaba Cloud ECI
<img src="eci.svg" width="200" height="200" />
Alibaba Cloud ECI(Elastic Container Instance) is a service that allow you run containers without having to manage servers or clusters.
You can find more infomation via [alibaba cloud ECI web portal](https://www.aliyun.com/product/eci)
## Alibaba Cloud ECI Virtual-Kubelet Provider
Alibaba ECI provider is an adapter to connect between k8s and ECI service to implement pod from k8s cluster on alibaba cloud platform
## Prerequisites
To using ECI service on alibaba cloud, you may need open ECI service on [web portal](https://www.aliyun.com/product/eci), and then the ECI service will be available
## Deployment of the ECI provider in your cluster
configure and launch virtual kubelet
```
export ECI_REGION=cn-hangzhou
export ECI_SECURITY_GROUP=sg-123
export ECI_VSWITCH=vsw-123
export ECI_ACCESS_KEY=123
export ECI_SECRET_KEY=123
VKUBELET_TAINT_KEY=alibabacloud.com/eci virtual-kubelet --provider alicloud
```
confirm the virtual kubelet is connected to k8s cluster
```
$kubectl get node
NAME STATUS ROLES AGE VERSION
cn-shanghai.i-uf69qodr5ntaxleqdhhk Ready <none> 1d v1.9.3
virtual-kubelet Ready agent 10s v1.8.3
```
## Schedule K8s Pod to ECI via virtual kubelet
You can assign pod to virtual kubelet via node-selector and toleration.
```
apiVersion: v1
kind: Pod
metadata:
name: mypod
spec:
nodeName: virtual-kubelet
containers:
- name: nginx
image: nginx
tolerations:
- key: alibabacloud.com/eci
operator: "Exists"
effect: NoSchedule
```
# Alibaba Cloud Serverless Kubernetes
Alibaba Cloud serverless kubernetes allows you to quickly create kubernetes container applications without
having to manage and maintain clusters and servers. It is based on ECI and fully compatible with the Kuberentes API.
You can find more infomation via [alibaba cloud serverless kubernetes product doc](https://www.alibabacloud.com/help/doc-detail/94078.htm)

View File

@@ -1,56 +0,0 @@
package alicloud
import (
"io"
"github.com/BurntSushi/toml"
"github.com/virtual-kubelet/virtual-kubelet/providers"
)
type providerConfig struct {
Region string
OperatingSystem string
CPU string
Memory string
Pods string
VSwitch string
SecureGroup string
ClusterName string
}
func (p *ECIProvider) loadConfig(r io.Reader) error {
var config providerConfig
if _, err := toml.DecodeReader(r, &config); err != nil {
return err
}
p.region = config.Region
if p.region == "" {
p.region = "cn-hangzhou"
}
p.vSwitch = config.VSwitch
p.secureGroup = config.SecureGroup
p.cpu = config.CPU
if p.cpu == "" {
p.cpu = "20"
}
p.memory = config.Memory
if p.memory == "" {
p.memory = "100Gi"
}
p.pods = config.Pods
if p.pods == "" {
p.pods = "20"
}
p.operatingSystem = config.OperatingSystem
if p.operatingSystem == "" {
p.operatingSystem = providers.OperatingSystemLinux
}
p.clusterName = config.ClusterName
if p.clusterName == "" {
p.clusterName = "default"
}
return nil
}

View File

@@ -1,903 +0,0 @@
package alicloud
import (
"bytes"
"context"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"strconv"
"strings"
"time"
"github.com/aliyun/alibaba-cloud-sdk-go/sdk/requests"
"github.com/cpuguy83/strongerrors"
"github.com/virtual-kubelet/virtual-kubelet/log"
"github.com/virtual-kubelet/virtual-kubelet/manager"
"github.com/virtual-kubelet/virtual-kubelet/providers/alicloud/eci"
"k8s.io/api/core/v1"
k8serr "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/tools/remotecommand"
)
// The service account secret mount path.
const serviceAccountSecretMountPath = "/var/run/secrets/kubernetes.io/serviceaccount"
const podTagTimeFormat = "2006-01-02T15-04-05Z"
const timeFormat = "2006-01-02T15:04:05Z"
// ECIProvider implements the virtual-kubelet provider interface and communicates with Alibaba Cloud's ECI APIs.
type ECIProvider struct {
eciClient *eci.Client
resourceManager *manager.ResourceManager
resourceGroup string
region string
nodeName string
operatingSystem string
clusterName string
cpu string
memory string
pods string
internalIP string
daemonEndpointPort int32
secureGroup string
vSwitch string
}
// AuthConfig is the secret returned from an ImageRegistryCredential
type AuthConfig struct {
Username string `json:"username,omitempty"`
Password string `json:"password,omitempty"`
Auth string `json:"auth,omitempty"`
Email string `json:"email,omitempty"`
ServerAddress string `json:"serveraddress,omitempty"`
IdentityToken string `json:"identitytoken,omitempty"`
RegistryToken string `json:"registrytoken,omitempty"`
}
var validEciRegions = []string{
"cn-hangzhou",
"cn-shanghai",
"cn-beijing",
"us-west-1",
}
// isValidECIRegion checks to make sure we're using a valid ECI region
func isValidECIRegion(region string) bool {
regionLower := strings.ToLower(region)
regionTrimmed := strings.Replace(regionLower, " ", "", -1)
for _, validRegion := range validEciRegions {
if regionTrimmed == validRegion {
return true
}
}
return false
}
// NewECIProvider creates a new ECIProvider.
func NewECIProvider(config string, rm *manager.ResourceManager, nodeName, operatingSystem string, internalIP string, daemonEndpointPort int32) (*ECIProvider, error) {
var p ECIProvider
var err error
p.resourceManager = rm
if config != "" {
f, err := os.Open(config)
if err != nil {
return nil, err
}
defer f.Close()
if err := p.loadConfig(f); err != nil {
return nil, err
}
}
if r := os.Getenv("ECI_CLUSTER_NAME"); r != "" {
p.clusterName = r
}
if p.clusterName == "" {
p.clusterName = "default"
}
if r := os.Getenv("ECI_REGION"); r != "" {
p.region = r
}
if p.region == "" {
return nil, errors.New("Region can't be empty please set ECI_REGION\n")
}
if r := p.region; !isValidECIRegion(r) {
unsupportedRegionMessage := fmt.Sprintf("Region %s is invalid. Current supported regions are: %s",
r, strings.Join(validEciRegions, ", "))
return nil, errors.New(unsupportedRegionMessage)
}
var accessKey, secretKey string
if ak := os.Getenv("ECI_ACCESS_KEY"); ak != "" {
accessKey = ak
}
if sk := os.Getenv("ECI_SECRET_KEY"); sk != "" {
secretKey = sk
}
if sg := os.Getenv("ECI_SECURITY_GROUP"); sg != "" {
p.secureGroup = sg
}
if vsw := os.Getenv("ECI_VSWITCH"); vsw != "" {
p.vSwitch = vsw
}
if p.secureGroup == "" {
return nil, errors.New("secureGroup can't be empty\n")
}
if p.vSwitch == "" {
return nil, errors.New("vSwitch can't be empty\n")
}
p.eciClient, err = eci.NewClientWithAccessKey(p.region, accessKey, secretKey)
if err != nil {
return nil, err
}
p.cpu = "1000"
p.memory = "4Ti"
p.pods = "1000"
if cpuQuota := os.Getenv("ECI_QUOTA_CPU"); cpuQuota != "" {
p.cpu = cpuQuota
}
if memoryQuota := os.Getenv("ECI_QUOTA_MEMORY"); memoryQuota != "" {
p.memory = memoryQuota
}
if podsQuota := os.Getenv("ECI_QUOTA_POD"); podsQuota != "" {
p.pods = podsQuota
}
p.operatingSystem = operatingSystem
p.nodeName = nodeName
p.internalIP = internalIP
p.daemonEndpointPort = daemonEndpointPort
return &p, err
}
// CreatePod accepts a Pod definition and creates
// an ECI deployment
func (p *ECIProvider) CreatePod(ctx context.Context, pod *v1.Pod) error {
//Ignore daemonSet Pod
if pod != nil && pod.OwnerReferences != nil && len(pod.OwnerReferences) != 0 && pod.OwnerReferences[0].Kind == "DaemonSet" {
msg := fmt.Sprintf("Skip to create DaemonSet pod %q", pod.Name)
log.G(ctx).WithField("Method", "CreatePod").Info(msg)
return nil
}
request := eci.CreateCreateContainerGroupRequest()
request.RestartPolicy = string(pod.Spec.RestartPolicy)
// get containers
containers, err := p.getContainers(pod, false)
initContainers, err := p.getContainers(pod, true)
if err != nil {
return err
}
// get registry creds
creds, err := p.getImagePullSecrets(pod)
if err != nil {
return err
}
// get volumes
volumes, err := p.getVolumes(pod)
if err != nil {
return err
}
// assign all the things
request.Containers = containers
request.InitContainers = initContainers
request.Volumes = volumes
request.ImageRegistryCredentials = creds
CreationTimestamp := pod.CreationTimestamp.UTC().Format(podTagTimeFormat)
tags := []eci.Tag{
eci.Tag{Key: "ClusterName", Value: p.clusterName},
eci.Tag{Key: "NodeName", Value: p.nodeName},
eci.Tag{Key: "NameSpace", Value: pod.Namespace},
eci.Tag{Key: "PodName", Value: pod.Name},
eci.Tag{Key: "UID", Value: string(pod.UID)},
eci.Tag{Key: "CreationTimestamp", Value: CreationTimestamp},
}
ContainerGroupName := containerGroupName(pod)
request.Tags = tags
request.SecurityGroupId = p.secureGroup
request.VSwitchId = p.vSwitch
request.ContainerGroupName = ContainerGroupName
msg := fmt.Sprintf("CreateContainerGroup request %+v", request)
log.G(ctx).WithField("Method", "CreatePod").Info(msg)
response, err := p.eciClient.CreateContainerGroup(request)
if err != nil {
return err
}
msg = fmt.Sprintf("CreateContainerGroup successed. %s, %s, %s", response.RequestId, response.ContainerGroupId, ContainerGroupName)
log.G(ctx).WithField("Method", "CreatePod").Info(msg)
return nil
}
func containerGroupName(pod *v1.Pod) string {
return fmt.Sprintf("%s-%s", pod.Namespace, pod.Name)
}
// UpdatePod is a noop, ECI currently does not support live updates of a pod.
func (p *ECIProvider) UpdatePod(ctx context.Context, pod *v1.Pod) error {
return nil
}
// DeletePod deletes the specified pod out of ECI.
func (p *ECIProvider) DeletePod(ctx context.Context, pod *v1.Pod) error {
eciId := ""
for _, cg := range p.GetCgs() {
if getECITagValue(&cg, "PodName") == pod.Name && getECITagValue(&cg, "NameSpace") == pod.Namespace {
eciId = cg.ContainerGroupId
break
}
}
if eciId == "" {
return strongerrors.NotFound(fmt.Errorf("DeletePod can't find Pod %s-%s", pod.Namespace, pod.Name))
}
request := eci.CreateDeleteContainerGroupRequest()
request.ContainerGroupId = eciId
_, err := p.eciClient.DeleteContainerGroup(request)
return wrapError(err)
}
// GetPod returns a pod by name that is running inside ECI
// returns nil if a pod by that name is not found.
func (p *ECIProvider) GetPod(ctx context.Context, namespace, name string) (*v1.Pod, error) {
pods, err := p.GetPods(ctx)
if err != nil {
return nil, err
}
for _, pod := range pods {
if pod.Name == name && pod.Namespace == namespace {
return pod, nil
}
}
return nil, nil
}
// GetContainerLogs returns the logs of a pod by name that is running inside ECI.
func (p *ECIProvider) GetContainerLogs(ctx context.Context, namespace, podName, containerName string, tail int) (string, error) {
eciId := ""
for _, cg := range p.GetCgs() {
if getECITagValue(&cg, "PodName") == podName && getECITagValue(&cg, "NameSpace") == namespace {
eciId = cg.ContainerGroupId
break
}
}
if eciId == "" {
return "", errors.New(fmt.Sprintf("GetContainerLogs can't find Pod %s-%s", namespace, podName))
}
request := eci.CreateDescribeContainerLogRequest()
request.ContainerGroupId = eciId
request.ContainerName = containerName
request.Tail = requests.Integer(tail)
// get logs from cg
logContent := ""
retry := 10
for i := 0; i < retry; i++ {
response, err := p.eciClient.DescribeContainerLog(request)
if err != nil {
msg := fmt.Sprint("Error getting container logs, retrying")
log.G(ctx).WithField("Method", "GetContainerLogs").Info(msg)
time.Sleep(5000 * time.Millisecond)
} else {
logContent = response.Content
break
}
}
return logContent, nil
}
// Get full pod name as defined in the provider context
func (p *ECIProvider) GetPodFullName(namespace string, pod string) string {
return fmt.Sprintf("%s-%s", namespace, pod)
}
// ExecInContainer executes a command in a container in the pod, copying data
// between in/out/err and the container's stdin/stdout/stderr.
func (p *ECIProvider) ExecInContainer(name string, uid types.UID, container string, cmd []string, in io.Reader, out, errstream io.WriteCloser, tty bool, resize <-chan remotecommand.TerminalSize, timeout time.Duration) error {
return nil
}
// GetPodStatus returns the status of a pod by name that is running inside ECI
// returns nil if a pod by that name is not found.
func (p *ECIProvider) GetPodStatus(ctx context.Context, namespace, name string) (*v1.PodStatus, error) {
pod, err := p.GetPod(ctx, namespace, name)
if err != nil {
return nil, err
}
if pod == nil {
return nil, nil
}
return &pod.Status, nil
}
func (p *ECIProvider) GetCgs() []eci.ContainerGroup {
cgs := make([]eci.ContainerGroup, 0)
request := eci.CreateDescribeContainerGroupsRequest()
for {
cgsResponse, err := p.eciClient.DescribeContainerGroups(request)
if err != nil || len(cgsResponse.ContainerGroups) == 0 {
break
}
request.NextToken = cgsResponse.NextToken
for _, cg := range cgsResponse.ContainerGroups {
if getECITagValue(&cg, "NodeName") != p.nodeName {
continue
}
cn := getECITagValue(&cg, "ClusterName")
if cn == "" {
cn = "default"
}
if cn != p.clusterName {
continue
}
cgs = append(cgs, cg)
}
if request.NextToken == "" {
break
}
}
return cgs
}
// GetPods returns a list of all pods known to be running within ECI.
func (p *ECIProvider) GetPods(ctx context.Context) ([]*v1.Pod, error) {
pods := make([]*v1.Pod, 0)
for _, cg := range p.GetCgs() {
c := cg
pod, err := containerGroupToPod(&c)
if err != nil {
msg := fmt.Sprint("error converting container group to pod", cg.ContainerGroupId, err)
log.G(context.TODO()).WithField("Method", "GetPods").Info(msg)
continue
}
pods = append(pods, pod)
}
return pods, nil
}
// Capacity returns a resource list containing the capacity limits set for ECI.
func (p *ECIProvider) Capacity(ctx context.Context) v1.ResourceList {
return v1.ResourceList{
"cpu": resource.MustParse(p.cpu),
"memory": resource.MustParse(p.memory),
"pods": resource.MustParse(p.pods),
}
}
// NodeConditions returns a list of conditions (Ready, OutOfDisk, etc), for updates to the node status
// within Kubernetes.
func (p *ECIProvider) NodeConditions(ctx context.Context) []v1.NodeCondition {
// TODO: Make these dynamic and augment with custom ECI specific conditions of interest
return []v1.NodeCondition{
{
Type: "Ready",
Status: v1.ConditionTrue,
LastHeartbeatTime: metav1.Now(),
LastTransitionTime: metav1.Now(),
Reason: "KubeletReady",
Message: "kubelet is ready.",
},
{
Type: "OutOfDisk",
Status: v1.ConditionFalse,
LastHeartbeatTime: metav1.Now(),
LastTransitionTime: metav1.Now(),
Reason: "KubeletHasSufficientDisk",
Message: "kubelet has sufficient disk space available",
},
{
Type: "MemoryPressure",
Status: v1.ConditionFalse,
LastHeartbeatTime: metav1.Now(),
LastTransitionTime: metav1.Now(),
Reason: "KubeletHasSufficientMemory",
Message: "kubelet has sufficient memory available",
},
{
Type: "DiskPressure",
Status: v1.ConditionFalse,
LastHeartbeatTime: metav1.Now(),
LastTransitionTime: metav1.Now(),
Reason: "KubeletHasNoDiskPressure",
Message: "kubelet has no disk pressure",
},
{
Type: "NetworkUnavailable",
Status: v1.ConditionFalse,
LastHeartbeatTime: metav1.Now(),
LastTransitionTime: metav1.Now(),
Reason: "RouteCreated",
Message: "RouteController created a route",
},
}
}
// NodeAddresses returns a list of addresses for the node status
// within Kubernetes.
func (p *ECIProvider) NodeAddresses(ctx context.Context) []v1.NodeAddress {
// TODO: Make these dynamic and augment with custom ECI specific conditions of interest
return []v1.NodeAddress{
{
Type: "InternalIP",
Address: p.internalIP,
},
}
}
// NodeDaemonEndpoints returns NodeDaemonEndpoints for the node status
// within Kubernetes.
func (p *ECIProvider) NodeDaemonEndpoints(ctx context.Context) *v1.NodeDaemonEndpoints {
return &v1.NodeDaemonEndpoints{
KubeletEndpoint: v1.DaemonEndpoint{
Port: p.daemonEndpointPort,
},
}
}
// OperatingSystem returns the operating system that was provided by the config.
func (p *ECIProvider) OperatingSystem() string {
return p.operatingSystem
}
func (p *ECIProvider) getImagePullSecrets(pod *v1.Pod) ([]eci.ImageRegistryCredential, error) {
ips := make([]eci.ImageRegistryCredential, 0, len(pod.Spec.ImagePullSecrets))
for _, ref := range pod.Spec.ImagePullSecrets {
secret, err := p.resourceManager.GetSecret(ref.Name, pod.Namespace)
if err != nil {
return ips, err
}
if secret == nil {
return nil, fmt.Errorf("error getting image pull secret")
}
// TODO: Check if secret type is v1.SecretTypeDockercfg and use DockerConfigKey instead of hardcoded value
// TODO: Check if secret type is v1.SecretTypeDockerConfigJson and use DockerConfigJsonKey to determine if it's in json format
// TODO: Return error if it's not one of these two types
switch secret.Type {
case v1.SecretTypeDockercfg:
ips, err = readDockerCfgSecret(secret, ips)
case v1.SecretTypeDockerConfigJson:
ips, err = readDockerConfigJSONSecret(secret, ips)
default:
return nil, fmt.Errorf("image pull secret type is not one of kubernetes.io/dockercfg or kubernetes.io/dockerconfigjson")
}
if err != nil {
return ips, err
}
}
return ips, nil
}
func readDockerCfgSecret(secret *v1.Secret, ips []eci.ImageRegistryCredential) ([]eci.ImageRegistryCredential, error) {
var err error
var authConfigs map[string]AuthConfig
repoData, ok := secret.Data[string(v1.DockerConfigKey)]
if !ok {
return ips, fmt.Errorf("no dockercfg present in secret")
}
err = json.Unmarshal(repoData, &authConfigs)
if err != nil {
return ips, fmt.Errorf("failed to unmarshal auth config %+v", err)
}
for server, authConfig := range authConfigs {
ips = append(ips, eci.ImageRegistryCredential{
Password: authConfig.Password,
Server: server,
UserName: authConfig.Username,
})
}
return ips, err
}
func readDockerConfigJSONSecret(secret *v1.Secret, ips []eci.ImageRegistryCredential) ([]eci.ImageRegistryCredential, error) {
var err error
repoData, ok := secret.Data[string(v1.DockerConfigJsonKey)]
if !ok {
return ips, fmt.Errorf("no dockerconfigjson present in secret")
}
var authConfigs map[string]map[string]AuthConfig
err = json.Unmarshal(repoData, &authConfigs)
if err != nil {
return ips, err
}
auths, ok := authConfigs["auths"]
if !ok {
return ips, fmt.Errorf("malformed dockerconfigjson in secret")
}
for server, authConfig := range auths {
ips = append(ips, eci.ImageRegistryCredential{
Password: authConfig.Password,
Server: server,
UserName: authConfig.Username,
})
}
return ips, err
}
func (p *ECIProvider) getContainers(pod *v1.Pod, init bool) ([]eci.CreateContainer, error) {
podContainers := pod.Spec.Containers
if init {
podContainers = pod.Spec.InitContainers
}
containers := make([]eci.CreateContainer, 0, len(podContainers))
for _, container := range podContainers {
c := eci.CreateContainer{
Name: container.Name,
Image: container.Image,
Commands: append(container.Command, container.Args...),
Ports: make([]eci.ContainerPort, 0, len(container.Ports)),
}
for _, p := range container.Ports {
c.Ports = append(c.Ports, eci.ContainerPort{
Port: requests.Integer(strconv.FormatInt(int64(p.ContainerPort), 10)),
Protocol: string(p.Protocol),
})
}
c.VolumeMounts = make([]eci.VolumeMount, 0, len(container.VolumeMounts))
for _, v := range container.VolumeMounts {
c.VolumeMounts = append(c.VolumeMounts, eci.VolumeMount{
Name: v.Name,
MountPath: v.MountPath,
ReadOnly: requests.Boolean(strconv.FormatBool(v.ReadOnly)),
})
}
c.EnvironmentVars = make([]eci.EnvironmentVar, 0, len(container.Env))
for _, e := range container.Env {
c.EnvironmentVars = append(c.EnvironmentVars, eci.EnvironmentVar{Key: e.Name, Value: e.Value})
}
// ECI CPU request must be times of 10m
cpuRequest := 1.00
if _, ok := container.Resources.Requests[v1.ResourceCPU]; ok {
cpuRequest = float64(container.Resources.Requests.Cpu().MilliValue()) / 1000.00
if cpuRequest < 0.01 {
cpuRequest = 0.01
}
}
c.Cpu = requests.Float(fmt.Sprintf("%.2f", cpuRequest))
// ECI memory request must be times of 0.1 GB
memoryRequest := 2.0
if _, ok := container.Resources.Requests[v1.ResourceMemory]; ok {
memoryRequest = float64(container.Resources.Requests.Memory().Value()) / 1000000000.0
if memoryRequest < 2.0 {
memoryRequest = 2.0
}
}
c.Memory = requests.Float(fmt.Sprintf("%.1f", memoryRequest))
c.ImagePullPolicy = string(container.ImagePullPolicy)
c.WorkingDir = container.WorkingDir
containers = append(containers, c)
}
return containers, nil
}
func (p *ECIProvider) getVolumes(pod *v1.Pod) ([]eci.Volume, error) {
volumes := make([]eci.Volume, 0, len(pod.Spec.Volumes))
for _, v := range pod.Spec.Volumes {
// Handle the case for the EmptyDir.
if v.EmptyDir != nil {
volumes = append(volumes, eci.Volume{
Type: eci.VOL_TYPE_EMPTYDIR,
Name: v.Name,
EmptyDirVolumeEnable: requests.Boolean(strconv.FormatBool(true)),
})
continue
}
// Handle the case for the NFS.
if v.NFS != nil {
volumes = append(volumes, eci.Volume{
Type: eci.VOL_TYPE_NFS,
Name: v.Name,
NfsVolumeServer: v.NFS.Server,
NfsVolumePath: v.NFS.Path,
NfsVolumeReadOnly: requests.Boolean(strconv.FormatBool(v.NFS.ReadOnly)),
})
continue
}
// Handle the case for ConfigMap volume.
if v.ConfigMap != nil {
ConfigFileToPaths := make([]eci.ConfigFileToPath, 0)
configMap, err := p.resourceManager.GetConfigMap(v.ConfigMap.Name, pod.Namespace)
if v.ConfigMap.Optional != nil && !*v.ConfigMap.Optional && k8serr.IsNotFound(err) {
return nil, fmt.Errorf("ConfigMap %s is required by Pod %s and does not exist", v.ConfigMap.Name, pod.Name)
}
if configMap == nil {
continue
}
for k, v := range configMap.Data {
var b bytes.Buffer
enc := base64.NewEncoder(base64.StdEncoding, &b)
enc.Write([]byte(v))
ConfigFileToPaths = append(ConfigFileToPaths, eci.ConfigFileToPath{Path: k, Content: b.String()})
}
if len(ConfigFileToPaths) != 0 {
volumes = append(volumes, eci.Volume{
Type: eci.VOL_TYPE_CONFIGFILEVOLUME,
Name: v.Name,
ConfigFileToPaths: ConfigFileToPaths,
})
}
continue
}
if v.Secret != nil {
ConfigFileToPaths := make([]eci.ConfigFileToPath, 0)
secret, err := p.resourceManager.GetSecret(v.Secret.SecretName, pod.Namespace)
if v.Secret.Optional != nil && !*v.Secret.Optional && k8serr.IsNotFound(err) {
return nil, fmt.Errorf("Secret %s is required by Pod %s and does not exist", v.Secret.SecretName, pod.Name)
}
if secret == nil {
continue
}
for k, v := range secret.Data {
var b bytes.Buffer
enc := base64.NewEncoder(base64.StdEncoding, &b)
enc.Write(v)
ConfigFileToPaths = append(ConfigFileToPaths, eci.ConfigFileToPath{Path: k, Content: b.String()})
}
if len(ConfigFileToPaths) != 0 {
volumes = append(volumes, eci.Volume{
Type: eci.VOL_TYPE_CONFIGFILEVOLUME,
Name: v.Name,
ConfigFileToPaths: ConfigFileToPaths,
})
}
continue
}
// If we've made it this far we have found a volume type that isn't supported
return nil, fmt.Errorf("Pod %s requires volume %s which is of an unsupported type\n", pod.Name, v.Name)
}
return volumes, nil
}
func containerGroupToPod(cg *eci.ContainerGroup) (*v1.Pod, error) {
var podCreationTimestamp, containerStartTime metav1.Time
CreationTimestamp := getECITagValue(cg, "CreationTimestamp")
if CreationTimestamp != "" {
if t, err := time.Parse(podTagTimeFormat, CreationTimestamp); err == nil {
podCreationTimestamp = metav1.NewTime(t)
}
}
if t, err := time.Parse(timeFormat, cg.Containers[0].CurrentState.StartTime); err == nil {
containerStartTime = metav1.NewTime(t)
}
// Use the Provisioning State if it's not Succeeded,
// otherwise use the state of the instance.
eciState := cg.Status
containers := make([]v1.Container, 0, len(cg.Containers))
containerStatuses := make([]v1.ContainerStatus, 0, len(cg.Containers))
for _, c := range cg.Containers {
container := v1.Container{
Name: c.Name,
Image: c.Image,
Command: c.Commands,
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: resource.MustParse(fmt.Sprintf("%.2f", c.Cpu)),
v1.ResourceMemory: resource.MustParse(fmt.Sprintf("%.1fG", c.Memory)),
},
},
}
container.Resources.Limits = v1.ResourceList{
v1.ResourceCPU: resource.MustParse(fmt.Sprintf("%.2f", c.Cpu)),
v1.ResourceMemory: resource.MustParse(fmt.Sprintf("%.1fG", c.Memory)),
}
containers = append(containers, container)
containerStatus := v1.ContainerStatus{
Name: c.Name,
State: eciContainerStateToContainerState(c.CurrentState),
LastTerminationState: eciContainerStateToContainerState(c.PreviousState),
Ready: eciStateToPodPhase(c.CurrentState.State) == v1.PodRunning,
RestartCount: int32(c.RestartCount),
Image: c.Image,
ImageID: "",
ContainerID: getContainerID(cg.ContainerGroupId, c.Name),
}
// Add to containerStatuses
containerStatuses = append(containerStatuses, containerStatus)
}
pod := v1.Pod{
TypeMeta: metav1.TypeMeta{
Kind: "Pod",
APIVersion: "v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: getECITagValue(cg, "PodName"),
Namespace: getECITagValue(cg, "NameSpace"),
ClusterName: getECITagValue(cg, "ClusterName"),
UID: types.UID(getECITagValue(cg, "UID")),
CreationTimestamp: podCreationTimestamp,
},
Spec: v1.PodSpec{
NodeName: getECITagValue(cg, "NodeName"),
Volumes: []v1.Volume{},
Containers: containers,
},
Status: v1.PodStatus{
Phase: eciStateToPodPhase(eciState),
Conditions: eciStateToPodConditions(eciState, podCreationTimestamp),
Message: "",
Reason: "",
HostIP: "",
PodIP: cg.IntranetIp,
StartTime: &containerStartTime,
ContainerStatuses: containerStatuses,
},
}
return &pod, nil
}
func getContainerID(cgID, containerName string) string {
if cgID == "" {
return ""
}
containerResourceID := fmt.Sprintf("%s/containers/%s", cgID, containerName)
h := sha256.New()
h.Write([]byte(strings.ToUpper(containerResourceID)))
hashBytes := h.Sum(nil)
return fmt.Sprintf("eci://%s", hex.EncodeToString(hashBytes))
}
func eciStateToPodPhase(state string) v1.PodPhase {
switch state {
case "Scheduling":
return v1.PodPending
case "ScheduleFailed":
return v1.PodFailed
case "Pending":
return v1.PodPending
case "Running":
return v1.PodRunning
case "Failed":
return v1.PodFailed
case "Succeeded":
return v1.PodSucceeded
}
return v1.PodUnknown
}
func eciStateToPodConditions(state string, transitionTime metav1.Time) []v1.PodCondition {
switch state {
case "Running", "Succeeded":
return []v1.PodCondition{
v1.PodCondition{
Type: v1.PodReady,
Status: v1.ConditionTrue,
LastTransitionTime: transitionTime,
}, v1.PodCondition{
Type: v1.PodInitialized,
Status: v1.ConditionTrue,
LastTransitionTime: transitionTime,
}, v1.PodCondition{
Type: v1.PodScheduled,
Status: v1.ConditionTrue,
LastTransitionTime: transitionTime,
},
}
}
return []v1.PodCondition{}
}
func eciContainerStateToContainerState(cs eci.ContainerState) v1.ContainerState {
t1, err := time.Parse(timeFormat, cs.StartTime)
if err != nil {
return v1.ContainerState{}
}
startTime := metav1.NewTime(t1)
// Handle the case where the container is running.
if cs.State == "Running" || cs.State == "Succeeded" {
return v1.ContainerState{
Running: &v1.ContainerStateRunning{
StartedAt: startTime,
},
}
}
t2, err := time.Parse(timeFormat, cs.FinishTime)
if err != nil {
return v1.ContainerState{}
}
finishTime := metav1.NewTime(t2)
// Handle the case where the container failed.
if cs.State == "Failed" || cs.State == "Canceled" {
return v1.ContainerState{
Terminated: &v1.ContainerStateTerminated{
ExitCode: int32(cs.ExitCode),
Reason: cs.State,
Message: cs.DetailStatus,
StartedAt: startTime,
FinishedAt: finishTime,
},
}
}
// Handle the case where the container is pending.
// Which should be all other eci states.
return v1.ContainerState{
Waiting: &v1.ContainerStateWaiting{
Reason: cs.State,
Message: cs.DetailStatus,
},
}
}
func getECITagValue(cg *eci.ContainerGroup, key string) string {
for _, tag := range cg.Tags {
if tag.Key == key {
return tag.Value
}
}
return ""
}

View File

@@ -1,16 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<!-- Generator: Adobe Illustrator 21.0.0, SVG Export Plug-In . SVG Version: 6.00 Build 0) -->
<svg version="1.0" id="layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
viewBox="0 0 48 48" style="enable-background:new 0 0 48 48;" xml:space="preserve">
<style type="text/css">
.st0{fill:#00C1DE;}
</style>
<g>
<polygon class="st0" points="41.4,22.7 41.4,25 43.7,25.6 43.7,31.8 24,36.3 4.3,31.8 4.3,25.6 6.5,25 6.5,22.7 2,23.7 2,33.7
24,38.7 46,33.7 46,23.7 "/>
<polygon class="st0" points="38.1,10.6 24,8 9.9,10.6 24,12.9 "/>
<polygon class="st0" points="22.8,15.1 8.8,12.6 8.8,30.2 22.8,33.4 "/>
<path class="st0" d="M25.2,33.4l14-3.2V12.6l-14,2.5V33.4z M35.2,15.3l2-0.4v13.7l-2,0.5V15.3z M31.3,15.9l2-0.4v13.9l-2,0.5V15.9z
M27.5,16.6l2-0.4v14l-2,0.5V16.6z"/>
</g>
</svg>

Before

Width:  |  Height:  |  Size: 878 B

View File

@@ -1,6 +0,0 @@
Region = "cn-hangzhou"
OperatingSystem = "Linux"
CPU = "20"
Memory = "100Gi"
Pods = "20"
ClusterName = "default"

View File

@@ -1,81 +0,0 @@
package eci
//Licensed under the Apache License, Version 2.0 (the "License");
//you may not use this file except in compliance with the License.
//You may obtain a copy of the License at
//
//http://www.apache.org/licenses/LICENSE-2.0
//
//Unless required by applicable law or agreed to in writing, software
//distributed under the License is distributed on an "AS IS" BASIS,
//WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//See the License for the specific language governing permissions and
//limitations under the License.
//
// Code generated by Alibaba Cloud SDK Code Generator.
// Changes may cause incorrect behavior and will be lost if the code is regenerated.
import (
"github.com/aliyun/alibaba-cloud-sdk-go/sdk"
"github.com/aliyun/alibaba-cloud-sdk-go/sdk/auth"
)
// Client is the sdk client struct, each func corresponds to an OpenAPI
type Client struct {
sdk.Client
}
// NewClient creates a sdk client with environment variables
func NewClient() (client *Client, err error) {
client = &Client{}
err = client.Init()
return
}
// NewClientWithOptions creates a sdk client with regionId/sdkConfig/credential
// this is the common api to create a sdk client
func NewClientWithOptions(regionId string, config *sdk.Config, credential auth.Credential) (client *Client, err error) {
client = &Client{}
err = client.InitWithOptions(regionId, config, credential)
return
}
// NewClientWithAccessKey is a shortcut to create sdk client with accesskey
// usage: https://help.aliyun.com/document_detail/66217.html
func NewClientWithAccessKey(regionId, accessKeyId, accessKeySecret string) (client *Client, err error) {
client = &Client{}
err = client.InitWithAccessKey(regionId, accessKeyId, accessKeySecret)
return
}
// NewClientWithStsToken is a shortcut to create sdk client with sts token
// usage: https://help.aliyun.com/document_detail/66222.html
func NewClientWithStsToken(regionId, stsAccessKeyId, stsAccessKeySecret, stsToken string) (client *Client, err error) {
client = &Client{}
err = client.InitWithStsToken(regionId, stsAccessKeyId, stsAccessKeySecret, stsToken)
return
}
// NewClientWithRamRoleArn is a shortcut to create sdk client with ram roleArn
// usage: https://help.aliyun.com/document_detail/66222.html
func NewClientWithRamRoleArn(regionId string, accessKeyId, accessKeySecret, roleArn, roleSessionName string) (client *Client, err error) {
client = &Client{}
err = client.InitWithRamRoleArn(regionId, accessKeyId, accessKeySecret, roleArn, roleSessionName)
return
}
// NewClientWithEcsRamRole is a shortcut to create sdk client with ecs ram role
// usage: https://help.aliyun.com/document_detail/66223.html
func NewClientWithEcsRamRole(regionId string, roleName string) (client *Client, err error) {
client = &Client{}
err = client.InitWithEcsRamRole(regionId, roleName)
return
}
// NewClientWithRsaKeyPair is a shortcut to create sdk client with rsa key pair
// attention: rsa key pair auth is only Japan regions available
func NewClientWithRsaKeyPair(regionId string, publicKeyId, privateKey string, sessionExpiration int) (client *Client, err error) {
client = &Client{}
err = client.InitWithRsaKeyPair(regionId, publicKeyId, privateKey, sessionExpiration)
return
}

View File

@@ -1,138 +0,0 @@
package eci
//Licensed under the Apache License, Version 2.0 (the "License");
//you may not use this file except in compliance with the License.
//You may obtain a copy of the License at
//
//http://www.apache.org/licenses/LICENSE-2.0
//
//Unless required by applicable law or agreed to in writing, software
//distributed under the License is distributed on an "AS IS" BASIS,
//WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//See the License for the specific language governing permissions and
//limitations under the License.
//
// Code generated by Alibaba Cloud SDK Code Generator.
// Changes may cause incorrect behavior and will be lost if the code is regenerated.
import (
"github.com/aliyun/alibaba-cloud-sdk-go/sdk/requests"
"github.com/aliyun/alibaba-cloud-sdk-go/sdk/responses"
)
// CreateContainerGroup invokes the eci.CreateContainerGroup API synchronously
// api document: https://help.aliyun.com/api/eci/createcontainergroup.html
func (client *Client) CreateContainerGroup(request *CreateContainerGroupRequest) (response *CreateContainerGroupResponse, err error) {
response = CreateCreateContainerGroupResponse()
err = client.DoAction(request, response)
return
}
// CreateContainerGroupWithChan invokes the eci.CreateContainerGroup API asynchronously
// api document: https://help.aliyun.com/api/eci/createcontainergroup.html
// asynchronous document: https://help.aliyun.com/document_detail/66220.html
func (client *Client) CreateContainerGroupWithChan(request *CreateContainerGroupRequest) (<-chan *CreateContainerGroupResponse, <-chan error) {
responseChan := make(chan *CreateContainerGroupResponse, 1)
errChan := make(chan error, 1)
err := client.AddAsyncTask(func() {
defer close(responseChan)
defer close(errChan)
response, err := client.CreateContainerGroup(request)
if err != nil {
errChan <- err
} else {
responseChan <- response
}
})
if err != nil {
errChan <- err
close(responseChan)
close(errChan)
}
return responseChan, errChan
}
// CreateContainerGroupWithCallback invokes the eci.CreateContainerGroup API asynchronously
// api document: https://help.aliyun.com/api/eci/createcontainergroup.html
// asynchronous document: https://help.aliyun.com/document_detail/66220.html
func (client *Client) CreateContainerGroupWithCallback(request *CreateContainerGroupRequest, callback func(response *CreateContainerGroupResponse, err error)) <-chan int {
result := make(chan int, 1)
err := client.AddAsyncTask(func() {
var response *CreateContainerGroupResponse
var err error
defer close(result)
response, err = client.CreateContainerGroup(request)
callback(response, err)
result <- 1
})
if err != nil {
defer close(result)
callback(nil, err)
result <- 0
}
return result
}
// CreateContainerGroupRequest is the request struct for api CreateContainerGroup
type CreateContainerGroupRequest struct {
*requests.RpcRequest
Containers []CreateContainer `position:"Query" name:"Container" type:"Repeated"`
InitContainers []CreateContainer `position:"Query" name:"InitContainer" type:"Repeated"`
ResourceOwnerId requests.Integer `position:"Query" name:"ResourceOwnerId"`
SecurityGroupId string `position:"Query" name:"SecurityGroupId"`
ImageRegistryCredentials []ImageRegistryCredential `position:"Query" name:"ImageRegistryCredential" type:"Repeated"`
Tags []Tag `position:"Query" name:"Tag" type:"Repeated"`
ResourceOwnerAccount string `position:"Query" name:"ResourceOwnerAccount"`
RestartPolicy string `position:"Query" name:"RestartPolicy"`
OwnerAccount string `position:"Query" name:"OwnerAccount"`
OwnerId requests.Integer `position:"Query" name:"OwnerId"`
VSwitchId string `position:"Query" name:"VSwitchId"`
Volumes []Volume `position:"Query" name:"Volume" type:"Repeated"`
ContainerGroupName string `position:"Query" name:"ContainerGroupName"`
ZoneId string `position:"Query" name:"ZoneId"`
}
type CreateContainer struct {
Name string `name:"Name"`
Image string `name:"Image"`
Memory requests.Float `name:"Memory"`
Cpu requests.Float `name:"Cpu"`
WorkingDir string `name:"WorkingDir"`
ImagePullPolicy string `name:"ImagePullPolicy"`
Commands []string `name:"Command" type:"Repeated"`
Args []string `name:"Arg" type:"Repeated"`
VolumeMounts []VolumeMount `name:"VolumeMount" type:"Repeated"`
Ports []ContainerPort `name:"Port" type:"Repeated"`
EnvironmentVars []EnvironmentVar `name:"EnvironmentVar" type:"Repeated"`
}
// CreateContainerGroupImageRegistryCredential is a repeated param struct in CreateContainerGroupRequest
type ImageRegistryCredential struct {
Server string `name:"Server"`
UserName string `name:"UserName"`
Password string `name:"Password"`
}
// CreateContainerGroupResponse is the response struct for api CreateContainerGroup
type CreateContainerGroupResponse struct {
*responses.BaseResponse
RequestId string
ContainerGroupId string
}
// CreateCreateContainerGroupRequest creates a request to invoke CreateContainerGroup API
func CreateCreateContainerGroupRequest() (request *CreateContainerGroupRequest) {
request = &CreateContainerGroupRequest{
RpcRequest: &requests.RpcRequest{},
}
request.InitWithApiInfo("Eci", "2018-08-08", "CreateContainerGroup", "eci", "openAPI")
return
}
// CreateCreateContainerGroupResponse creates a response to parse from CreateContainerGroup response
func CreateCreateContainerGroupResponse() (response *CreateContainerGroupResponse) {
response = &CreateContainerGroupResponse{
BaseResponse: &responses.BaseResponse{},
}
return
}

View File

@@ -1,107 +0,0 @@
package eci
//Licensed under the Apache License, Version 2.0 (the "License");
//you may not use this file except in compliance with the License.
//You may obtain a copy of the License at
//
//http://www.apache.org/licenses/LICENSE-2.0
//
//Unless required by applicable law or agreed to in writing, software
//distributed under the License is distributed on an "AS IS" BASIS,
//WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//See the License for the specific language governing permissions and
//limitations under the License.
//
// Code generated by Alibaba Cloud SDK Code Generator.
// Changes may cause incorrect behavior and will be lost if the code is regenerated.
import (
"github.com/aliyun/alibaba-cloud-sdk-go/sdk/requests"
"github.com/aliyun/alibaba-cloud-sdk-go/sdk/responses"
)
// DeleteContainerGroup invokes the eci.DeleteContainerGroup API synchronously
// api document: https://help.aliyun.com/api/eci/deletecontainergroup.html
func (client *Client) DeleteContainerGroup(request *DeleteContainerGroupRequest) (response *DeleteContainerGroupResponse, err error) {
response = CreateDeleteContainerGroupResponse()
err = client.DoAction(request, response)
return
}
// DeleteContainerGroupWithChan invokes the eci.DeleteContainerGroup API asynchronously
// api document: https://help.aliyun.com/api/eci/deletecontainergroup.html
// asynchronous document: https://help.aliyun.com/document_detail/66220.html
func (client *Client) DeleteContainerGroupWithChan(request *DeleteContainerGroupRequest) (<-chan *DeleteContainerGroupResponse, <-chan error) {
responseChan := make(chan *DeleteContainerGroupResponse, 1)
errChan := make(chan error, 1)
err := client.AddAsyncTask(func() {
defer close(responseChan)
defer close(errChan)
response, err := client.DeleteContainerGroup(request)
if err != nil {
errChan <- err
} else {
responseChan <- response
}
})
if err != nil {
errChan <- err
close(responseChan)
close(errChan)
}
return responseChan, errChan
}
// DeleteContainerGroupWithCallback invokes the eci.DeleteContainerGroup API asynchronously
// api document: https://help.aliyun.com/api/eci/deletecontainergroup.html
// asynchronous document: https://help.aliyun.com/document_detail/66220.html
func (client *Client) DeleteContainerGroupWithCallback(request *DeleteContainerGroupRequest, callback func(response *DeleteContainerGroupResponse, err error)) <-chan int {
result := make(chan int, 1)
err := client.AddAsyncTask(func() {
var response *DeleteContainerGroupResponse
var err error
defer close(result)
response, err = client.DeleteContainerGroup(request)
callback(response, err)
result <- 1
})
if err != nil {
defer close(result)
callback(nil, err)
result <- 0
}
return result
}
// DeleteContainerGroupRequest is the request struct for api DeleteContainerGroup
type DeleteContainerGroupRequest struct {
*requests.RpcRequest
ResourceOwnerId requests.Integer `position:"Query" name:"ResourceOwnerId"`
ContainerGroupId string `position:"Query" name:"ContainerGroupId"`
ResourceOwnerAccount string `position:"Query" name:"ResourceOwnerAccount"`
OwnerAccount string `position:"Query" name:"OwnerAccount"`
OwnerId requests.Integer `position:"Query" name:"OwnerId"`
}
// DeleteContainerGroupResponse is the response struct for api DeleteContainerGroup
type DeleteContainerGroupResponse struct {
*responses.BaseResponse
RequestId string `json:"RequestId" xml:"RequestId"`
}
// CreateDeleteContainerGroupRequest creates a request to invoke DeleteContainerGroup API
func CreateDeleteContainerGroupRequest() (request *DeleteContainerGroupRequest) {
request = &DeleteContainerGroupRequest{
RpcRequest: &requests.RpcRequest{},
}
request.InitWithApiInfo("Eci", "2018-08-08", "DeleteContainerGroup", "eci", "openAPI")
return
}
// CreateDeleteContainerGroupResponse creates a response to parse from DeleteContainerGroup response
func CreateDeleteContainerGroupResponse() (response *DeleteContainerGroupResponse) {
response = &DeleteContainerGroupResponse{
BaseResponse: &responses.BaseResponse{},
}
return
}

View File

@@ -1,122 +0,0 @@
package eci
//Licensed under the Apache License, Version 2.0 (the "License");
//you may not use this file except in compliance with the License.
//You may obtain a copy of the License at
//
//http://www.apache.org/licenses/LICENSE-2.0
//
//Unless required by applicable law or agreed to in writing, software
//distributed under the License is distributed on an "AS IS" BASIS,
//WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//See the License for the specific language governing permissions and
//limitations under the License.
//
// Code generated by Alibaba Cloud SDK Code Generator.
// Changes may cause incorrect behavior and will be lost if the code is regenerated.
import (
"github.com/aliyun/alibaba-cloud-sdk-go/sdk/requests"
"github.com/aliyun/alibaba-cloud-sdk-go/sdk/responses"
)
// DescribeContainerGroups invokes the eci.DescribeContainerGroups API synchronously
// api document: https://help.aliyun.com/api/eci/describecontainergroups.html
func (client *Client) DescribeContainerGroups(request *DescribeContainerGroupsRequest) (response *DescribeContainerGroupsResponse, err error) {
response = CreateDescribeContainerGroupsResponse()
err = client.DoAction(request, response)
return
}
// DescribeContainerGroupsWithChan invokes the eci.DescribeContainerGroups API asynchronously
// api document: https://help.aliyun.com/api/eci/describecontainergroups.html
// asynchronous document: https://help.aliyun.com/document_detail/66220.html
func (client *Client) DescribeContainerGroupsWithChan(request *DescribeContainerGroupsRequest) (<-chan *DescribeContainerGroupsResponse, <-chan error) {
responseChan := make(chan *DescribeContainerGroupsResponse, 1)
errChan := make(chan error, 1)
err := client.AddAsyncTask(func() {
defer close(responseChan)
defer close(errChan)
response, err := client.DescribeContainerGroups(request)
if err != nil {
errChan <- err
} else {
responseChan <- response
}
})
if err != nil {
errChan <- err
close(responseChan)
close(errChan)
}
return responseChan, errChan
}
// DescribeContainerGroupsWithCallback invokes the eci.DescribeContainerGroups API asynchronously
// api document: https://help.aliyun.com/api/eci/describecontainergroups.html
// asynchronous document: https://help.aliyun.com/document_detail/66220.html
func (client *Client) DescribeContainerGroupsWithCallback(request *DescribeContainerGroupsRequest, callback func(response *DescribeContainerGroupsResponse, err error)) <-chan int {
result := make(chan int, 1)
err := client.AddAsyncTask(func() {
var response *DescribeContainerGroupsResponse
var err error
defer close(result)
response, err = client.DescribeContainerGroups(request)
callback(response, err)
result <- 1
})
if err != nil {
defer close(result)
callback(nil, err)
result <- 0
}
return result
}
// DescribeContainerGroupsRequest is the request struct for api DescribeContainerGroups
type DescribeContainerGroupsRequest struct {
*requests.RpcRequest
ResourceOwnerId requests.Integer `position:"Query" name:"ResourceOwnerId"`
NextToken string `position:"Query" name:"NextToken"`
Limit requests.Integer `position:"Query" name:"Limit"`
Tags *[]DescribeContainerGroupsTag `position:"Query" name:"Tag" type:"Repeated"`
ContainerGroupId string `position:"Query" name:"ContainerGroupId"`
ResourceOwnerAccount string `position:"Query" name:"ResourceOwnerAccount"`
OwnerAccount string `position:"Query" name:"OwnerAccount"`
OwnerId requests.Integer `position:"Query" name:"OwnerId"`
VSwitchId string `position:"Query" name:"VSwitchId"`
ContainerGroupName string `position:"Query" name:"ContainerGroupName"`
ZoneId string `position:"Query" name:"ZoneId"`
}
// DescribeContainerGroupsTag is a repeated param struct in DescribeContainerGroupsRequest
type DescribeContainerGroupsTag struct {
Key string `name:"Key"`
Value string `name:"Value"`
}
// DescribeContainerGroupsResponse is the response struct for api DescribeContainerGroups
type DescribeContainerGroupsResponse struct {
*responses.BaseResponse
RequestId string `json:"RequestId" xml:"RequestId"`
NextToken string `json:"NextToken" xml:"NextToken"`
TotalCount int `json:"TotalCount" xml:"TotalCount"`
ContainerGroups []ContainerGroup `json:"ContainerGroups" xml:"ContainerGroups"`
}
// CreateDescribeContainerGroupsRequest creates a request to invoke DescribeContainerGroups API
func CreateDescribeContainerGroupsRequest() (request *DescribeContainerGroupsRequest) {
request = &DescribeContainerGroupsRequest{
RpcRequest: &requests.RpcRequest{},
}
request.InitWithApiInfo("Eci", "2018-08-08", "DescribeContainerGroups", "eci", "openAPI")
return
}
// CreateDescribeContainerGroupsResponse creates a response to parse from DescribeContainerGroups response
func CreateDescribeContainerGroupsResponse() (response *DescribeContainerGroupsResponse) {
response = &DescribeContainerGroupsResponse{
BaseResponse: &responses.BaseResponse{},
}
return
}

View File

@@ -1,112 +0,0 @@
package eci
//Licensed under the Apache License, Version 2.0 (the "License");
//you may not use this file except in compliance with the License.
//You may obtain a copy of the License at
//
//http://www.apache.org/licenses/LICENSE-2.0
//
//Unless required by applicable law or agreed to in writing, software
//distributed under the License is distributed on an "AS IS" BASIS,
//WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//See the License for the specific language governing permissions and
//limitations under the License.
//
// Code generated by Alibaba Cloud SDK Code Generator.
// Changes may cause incorrect behavior and will be lost if the code is regenerated.
import (
"github.com/aliyun/alibaba-cloud-sdk-go/sdk/requests"
"github.com/aliyun/alibaba-cloud-sdk-go/sdk/responses"
)
// DescribeContainerLog invokes the eci.DescribeContainerLog API synchronously
// api document: https://help.aliyun.com/api/eci/describecontainerlog.html
func (client *Client) DescribeContainerLog(request *DescribeContainerLogRequest) (response *DescribeContainerLogResponse, err error) {
response = CreateDescribeContainerLogResponse()
err = client.DoAction(request, response)
return
}
// DescribeContainerLogWithChan invokes the eci.DescribeContainerLog API asynchronously
// api document: https://help.aliyun.com/api/eci/describecontainerlog.html
// asynchronous document: https://help.aliyun.com/document_detail/66220.html
func (client *Client) DescribeContainerLogWithChan(request *DescribeContainerLogRequest) (<-chan *DescribeContainerLogResponse, <-chan error) {
responseChan := make(chan *DescribeContainerLogResponse, 1)
errChan := make(chan error, 1)
err := client.AddAsyncTask(func() {
defer close(responseChan)
defer close(errChan)
response, err := client.DescribeContainerLog(request)
if err != nil {
errChan <- err
} else {
responseChan <- response
}
})
if err != nil {
errChan <- err
close(responseChan)
close(errChan)
}
return responseChan, errChan
}
// DescribeContainerLogWithCallback invokes the eci.DescribeContainerLog API asynchronously
// api document: https://help.aliyun.com/api/eci/describecontainerlog.html
// asynchronous document: https://help.aliyun.com/document_detail/66220.html
func (client *Client) DescribeContainerLogWithCallback(request *DescribeContainerLogRequest, callback func(response *DescribeContainerLogResponse, err error)) <-chan int {
result := make(chan int, 1)
err := client.AddAsyncTask(func() {
var response *DescribeContainerLogResponse
var err error
defer close(result)
response, err = client.DescribeContainerLog(request)
callback(response, err)
result <- 1
})
if err != nil {
defer close(result)
callback(nil, err)
result <- 0
}
return result
}
// DescribeContainerLogRequest is the request struct for api DescribeContainerLog
type DescribeContainerLogRequest struct {
*requests.RpcRequest
ResourceOwnerId requests.Integer `position:"Query" name:"ResourceOwnerId"`
ContainerName string `position:"Query" name:"ContainerName"`
StartTime string `position:"Query" name:"StartTime"`
ContainerGroupId string `position:"Query" name:"ContainerGroupId"`
ResourceOwnerAccount string `position:"Query" name:"ResourceOwnerAccount"`
Tail requests.Integer `position:"Query" name:"Tail"`
OwnerAccount string `position:"Query" name:"OwnerAccount"`
OwnerId requests.Integer `position:"Query" name:"OwnerId"`
}
// DescribeContainerLogResponse is the response struct for api DescribeContainerLog
type DescribeContainerLogResponse struct {
*responses.BaseResponse
RequestId string `json:"RequestId" xml:"RequestId"`
ContainerName string `json:"ContainerName" xml:"ContainerName"`
Content string `json:"Content" xml:"Content"`
}
// CreateDescribeContainerLogRequest creates a request to invoke DescribeContainerLog API
func CreateDescribeContainerLogRequest() (request *DescribeContainerLogRequest) {
request = &DescribeContainerLogRequest{
RpcRequest: &requests.RpcRequest{},
}
request.InitWithApiInfo("Eci", "2018-08-08", "DescribeContainerLog", "eci", "openAPI")
return
}
// CreateDescribeContainerLogResponse creates a response to parse from DescribeContainerLog response
func CreateDescribeContainerLogResponse() (response *DescribeContainerLogResponse) {
response = &DescribeContainerLogResponse{
BaseResponse: &responses.BaseResponse{},
}
return
}

View File

@@ -1,22 +0,0 @@
package eci
//Licensed under the Apache License, Version 2.0 (the "License");
//you may not use this file except in compliance with the License.
//You may obtain a copy of the License at
//
//http://www.apache.org/licenses/LICENSE-2.0
//
//Unless required by applicable law or agreed to in writing, software
//distributed under the License is distributed on an "AS IS" BASIS,
//WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//See the License for the specific language governing permissions and
//limitations under the License.
//
// Code generated by Alibaba Cloud SDK Code Generator.
// Changes may cause incorrect behavior and will be lost if the code is regenerated.
// ConfigFileVolumeConfigFileToPath is a nested struct in eci response
type ConfigFileToPath struct {
Content string `name:"Content"`
Path string `name:"Path"`
}

View File

@@ -1,34 +0,0 @@
package eci
//Licensed under the Apache License, Version 2.0 (the "License");
//you may not use this file except in compliance with the License.
//You may obtain a copy of the License at
//
//http://www.apache.org/licenses/LICENSE-2.0
//
//Unless required by applicable law or agreed to in writing, software
//distributed under the License is distributed on an "AS IS" BASIS,
//WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//See the License for the specific language governing permissions and
//limitations under the License.
//
// Code generated by Alibaba Cloud SDK Code Generator.
// Changes may cause incorrect behavior and will be lost if the code is regenerated.
// Container is a nested struct in eci response
type Container struct {
Name string `json:"Name" xml:"Name" `
Image string `json:"Image" xml:"Image"`
Memory float64 `json:"Memory" xml:"Memory"`
Cpu float64 `json:"Cpu" xml:"Cpu"`
RestartCount int `json:"RestartCount" xml:"RestartCount"`
WorkingDir string `json:"WorkingDir" xml:"WorkingDir"`
ImagePullPolicy string `json:"ImagePullPolicy" xml:"ImagePullPolicy"`
Commands []string `json:"Commands" xml:"Commands"`
Args []string `json:"Args" xml:"Args"`
PreviousState ContainerState `json:"PreviousState" xml:"PreviousState"`
CurrentState ContainerState `json:"CurrentState" xml:"CurrentState"`
VolumeMounts []VolumeMount `json:"VolumeMounts" xml:"VolumeMounts"`
Ports []ContainerPort `json:"Ports" xml:"Ports"`
EnvironmentVars []EnvironmentVar `json:"EnvironmentVars" xml:"EnvironmentVars"`
}

View File

@@ -1,38 +0,0 @@
package eci
//Licensed under the Apache License, Version 2.0 (the "License");
//you may not use this file except in compliance with the License.
//You may obtain a copy of the License at
//
//http://www.apache.org/licenses/LICENSE-2.0
//
//Unless required by applicable law or agreed to in writing, software
//distributed under the License is distributed on an "AS IS" BASIS,
//WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//See the License for the specific language governing permissions and
//limitations under the License.
//
// Code generated by Alibaba Cloud SDK Code Generator.
// Changes may cause incorrect behavior and will be lost if the code is regenerated.
// ContainerGroup is a nested struct in eci response
type ContainerGroup struct {
ContainerGroupId string `json:"ContainerGroupId" xml:"ContainerGroupId"`
ContainerGroupName string `json:"ContainerGroupName" xml:"ContainerGroupName"`
RegionId string `json:"RegionId" xml:"RegionId"`
ZoneId string `json:"ZoneId" xml:"ZoneId"`
Memory float64 `json:"Memory" xml:"Memory"`
Cpu float64 `json:"Cpu" xml:"Cpu"`
VSwitchId string `json:"VSwitchId" xml:"VSwitchId"`
SecurityGroupId string `json:"SecurityGroupId" xml:"SecurityGroupId"`
RestartPolicy string `json:"RestartPolicy" xml:"RestartPolicy"`
IntranetIp string `json:"IntranetIp" xml:"IntranetIp"`
Status string `json:"Status" xml:"Status"`
InternetIp string `json:"InternetIp" xml:"InternetIp"`
CreationTime string `json:"CreationTime" xml:"CreationTime"`
SucceededTime string `json:"SucceededTime" xml:"SucceededTime"`
Tags []Tag `json:"Tags" xml:"Tags"`
Events []Event `json:"Events" xml:"Events"`
Containers []Container `json:"Containers" xml:"Containers"`
Volumes []Volume `json:"Volumes" xml:"Volumes"`
}

View File

@@ -1,26 +0,0 @@
package eci
import (
"github.com/aliyun/alibaba-cloud-sdk-go/sdk/requests"
)
//Licensed under the Apache License, Version 2.0 (the "License");
//you may not use this file except in compliance with the License.
//You may obtain a copy of the License at
//
//http://www.apache.org/licenses/LICENSE-2.0
//
//Unless required by applicable law or agreed to in writing, software
//distributed under the License is distributed on an "AS IS" BASIS,
//WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//See the License for the specific language governing permissions and
//limitations under the License.
//
// Code generated by Alibaba Cloud SDK Code Generator.
// Changes may cause incorrect behavior and will be lost if the code is regenerated.
// ContainerPort is a nested struct in eci response
type ContainerPort struct {
Port requests.Integer `name:"Port"`
Protocol string `name:"Protocol"`
}

View File

@@ -1,25 +0,0 @@
package eci
//Licensed under the Apache License, Version 2.0 (the "License");
//you may not use this file except in compliance with the License.
//You may obtain a copy of the License at
//
//http://www.apache.org/licenses/LICENSE-2.0
//
//Unless required by applicable law or agreed to in writing, software
//distributed under the License is distributed on an "AS IS" BASIS,
//WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//See the License for the specific language governing permissions and
//limitations under the License.
//
// Code generated by Alibaba Cloud SDK Code Generator.
// Changes may cause incorrect behavior and will be lost if the code is regenerated.
// CurrentState is a nested struct in eci response
type ContainerState struct {
State string `json:"State" xml:"State"`
DetailStatus string `json:"DetailStatus" xml:"DetailStatus"`
ExitCode int `json:"ExitCode" xml:"ExitCode"`
StartTime string `json:"StartTime" xml:"StartTime"`
FinishTime string `json:"FinishTime" xml:"FinishTime"`
}

View File

@@ -1,22 +0,0 @@
package eci
//Licensed under the Apache License, Version 2.0 (the "License");
//you may not use this file except in compliance with the License.
//You may obtain a copy of the License at
//
//http://www.apache.org/licenses/LICENSE-2.0
//
//Unless required by applicable law or agreed to in writing, software
//distributed under the License is distributed on an "AS IS" BASIS,
//WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//See the License for the specific language governing permissions and
//limitations under the License.
//
// Code generated by Alibaba Cloud SDK Code Generator.
// Changes may cause incorrect behavior and will be lost if the code is regenerated.
// EnvironmentVar is a nested struct in eci response
type EnvironmentVar struct {
Key string `name:"Key"`
Value string `name:"Value"`
}

View File

@@ -1,26 +0,0 @@
package eci
//Licensed under the Apache License, Version 2.0 (the "License");
//you may not use this file except in compliance with the License.
//You may obtain a copy of the License at
//
//http://www.apache.org/licenses/LICENSE-2.0
//
//Unless required by applicable law or agreed to in writing, software
//distributed under the License is distributed on an "AS IS" BASIS,
//WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//See the License for the specific language governing permissions and
//limitations under the License.
//
// Code generated by Alibaba Cloud SDK Code Generator.
// Changes may cause incorrect behavior and will be lost if the code is regenerated.
// Event is a nested struct in eci response
type Event struct {
Count int `json:"Count" xml:"Count"`
Type string `json:"Type" xml:"Type"`
Name string `json:"Name" xml:"Name"`
Message string `json:"Message" xml:"Message"`
FirstTimestamp string `json:"FirstTimestamp" xml:"FirstTimestamp"`
LastTimestamp string `json:"LastTimestamp" xml:"LastTimestamp"`
}

View File

@@ -1,22 +0,0 @@
package eci
//Licensed under the Apache License, Version 2.0 (the "License");
//you may not use this file except in compliance with the License.
//You may obtain a copy of the License at
//
//http://www.apache.org/licenses/LICENSE-2.0
//
//Unless required by applicable law or agreed to in writing, software
//distributed under the License is distributed on an "AS IS" BASIS,
//WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//See the License for the specific language governing permissions and
//limitations under the License.
//
// Code generated by Alibaba Cloud SDK Code Generator.
// Changes may cause incorrect behavior and will be lost if the code is regenerated.
// Label is a nested struct in eci response
type Tag struct {
Key string `name:"Key"`
Value string `name:"Value"`
}

View File

@@ -1,35 +0,0 @@
package eci
import "github.com/aliyun/alibaba-cloud-sdk-go/sdk/requests"
//Licensed under the Apache License, Version 2.0 (the "License");
//you may not use this file except in compliance with the License.
//You may obtain a copy of the License at
//
//http://www.apache.org/licenses/LICENSE-2.0
//
//Unless required by applicable law or agreed to in writing, software
//distributed under the License is distributed on an "AS IS" BASIS,
//WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//See the License for the specific language governing permissions and
//limitations under the License.
//
// Code generated by Alibaba Cloud SDK Code Generator.
// Changes may cause incorrect behavior and will be lost if the code is regenerated.
// Volume is a nested struct in eci response
const (
VOL_TYPE_NFS = "NFSVolume"
VOL_TYPE_EMPTYDIR = "EmptyDirVolume"
VOL_TYPE_CONFIGFILEVOLUME = "ConfigFileVolume"
)
type Volume struct {
Type string `name:"Type"`
Name string `name:"Name"`
NfsVolumePath string `name:"NFSVolume.Path"`
NfsVolumeServer string `name:"NFSVolume.Server"`
NfsVolumeReadOnly requests.Boolean `name:"NFSVolume.ReadOnly"`
EmptyDirVolumeEnable requests.Boolean `name:"EmptyDirVolume.Enable"`
ConfigFileToPaths []ConfigFileToPath `name:"ConfigFileVolume.ConfigFileToPath" type:"Repeated"`
}

View File

@@ -1,25 +0,0 @@
package eci
import "github.com/aliyun/alibaba-cloud-sdk-go/sdk/requests"
//Licensed under the Apache License, Version 2.0 (the "License");
//you may not use this file except in compliance with the License.
//You may obtain a copy of the License at
//
//http://www.apache.org/licenses/LICENSE-2.0
//
//Unless required by applicable law or agreed to in writing, software
//distributed under the License is distributed on an "AS IS" BASIS,
//WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//See the License for the specific language governing permissions and
//limitations under the License.
//
// Code generated by Alibaba Cloud SDK Code Generator.
// Changes may cause incorrect behavior and will be lost if the code is regenerated.
// VolumeMount is a nested struct in eci response
type VolumeMount struct {
MountPath string `name:"MountPath"`
ReadOnly requests.Boolean `name:"ReadOnly"`
Name string `name:"Name"`
}

Some files were not shown because too many files have changed in this diff Show More