Refactor provider init (#360)

* Refactor provider init

This moves provider init out of vkubelet setup, instead preferring to
initialize vkubelet with a provider.

* Split API server configuration from setup.

This makes sure that configuration (which is done primarily through env
vars) is separate from actually standing up the servers.

This also makes sure to abort daemon initialization if the API servers
are not able to start.

vkubelet/apiserver.go

@@ -2,39 +2,33 @@ package vkubelet
 
 import (
 	"context"
-	"fmt"
+	"net"
 	"net/http"
-	"os"
 
 	"github.com/Sirupsen/logrus"
 	"github.com/gorilla/mux"
 	"github.com/virtual-kubelet/virtual-kubelet/log"
+	"github.com/virtual-kubelet/virtual-kubelet/providers"
 	"github.com/virtual-kubelet/virtual-kubelet/vkubelet/api"
 )
 
-// KubeletServerStart starts the virtual kubelet HTTP server.
-func KubeletServerStart(p Provider) {
-	certFilePath := os.Getenv("APISERVER_CERT_LOCATION")
-	keyFilePath := os.Getenv("APISERVER_KEY_LOCATION")
-	port := os.Getenv("KUBELET_PORT")
-	addr := fmt.Sprintf(":%s", port)
-
+// PodHandler creates an http handler for interacting with pods/containers.
+func PodHandler(p providers.Provider) http.Handler {
 	r := mux.NewRouter()
 	r.HandleFunc("/containerLogs/{namespace}/{pod}/{container}", api.PodLogsHandlerFunc(p)).Methods("GET")
 	r.HandleFunc("/exec/{namespace}/{pod}/{container}", api.PodExecHandlerFunc(p)).Methods("POST")
 	r.NotFoundHandler = http.HandlerFunc(NotFound)
-
-	if err := http.ListenAndServeTLS(addr, certFilePath, keyFilePath, InstrumentHandler(r)); err != nil {
-		log.G(context.TODO()).WithError(err).Error("error setting up http server")
-	}
+	return r
 }
 
-// MetricsServerStart starts an HTTP server on the provided addr for serving the kubelset summary stats API.
-// TLS is never enabled on this endpoint.
-func MetricsServerStart(p Provider, addr string) {
+// MetricsSummaryHandler creates an http handler for serving pod metrics.
+//
+// If the passed in provider does not implement providers.PodMetricsProvider,
+// it will create handlers that just serves http.StatusNotImplemented
+func MetricsSummaryHandler(p providers.Provider) http.Handler {
 	r := mux.NewRouter()
 
-	mp, ok := p.(PodMetricsProvider)
+	mp, ok := p.(providers.PodMetricsProvider)
 	if !ok {
 		r.HandleFunc("/stats/summary", NotImplemented).Methods("GET")
 		r.HandleFunc("/stats/summary/", NotImplemented).Methods("GET")
@@ -43,7 +37,19 @@ func MetricsServerStart(p Provider, addr string) {
 		r.HandleFunc("/stats/summary/", api.PodMetricsHandlerFunc(mp)).Methods("GET")
 	}
 	r.NotFoundHandler = http.HandlerFunc(NotFound)
-	if err := http.ListenAndServe(addr, InstrumentHandler(r)); err != nil {
+	return r
+}
+
+// KubeletServerStart starts the virtual kubelet HTTP server.
+func KubeletServerStart(p providers.Provider, l net.Listener, cert, key string) {
+	if err := http.ServeTLS(l, InstrumentHandler(PodHandler(p)), cert, key); err != nil {
+		log.G(context.TODO()).WithError(err).Error("error setting up http server")
+	}
+}
+
+// MetricsServerStart starts an HTTP server on the provided addr for serving the kubelset summary stats API.
+func MetricsServerStart(p providers.Provider, l net.Listener) {
+	if err := http.Serve(l, InstrumentHandler(MetricsSummaryHandler(p))); err != nil {
 		log.G(context.TODO()).WithError(err).Error("Error starting http server")
 	}
 }

vkubelet/lookup.go

@@ -1,64 +0,0 @@
-// +build !windows,!darwin
-
-package vkubelet
-
-import (
-	"github.com/pkg/errors"
-	"github.com/virtual-kubelet/virtual-kubelet/manager"
-	"github.com/virtual-kubelet/virtual-kubelet/providers/alicloud"
-	"github.com/virtual-kubelet/virtual-kubelet/providers/aws"
-	"github.com/virtual-kubelet/virtual-kubelet/providers/azure"
-	"github.com/virtual-kubelet/virtual-kubelet/providers/azurebatch"
-	"github.com/virtual-kubelet/virtual-kubelet/providers/cri"
-	"github.com/virtual-kubelet/virtual-kubelet/providers/huawei"
-	"github.com/virtual-kubelet/virtual-kubelet/providers/hypersh"
-	"github.com/virtual-kubelet/virtual-kubelet/providers/mock"
-	"github.com/virtual-kubelet/virtual-kubelet/providers/sfmesh"
-	"github.com/virtual-kubelet/virtual-kubelet/providers/vic"
-	"github.com/virtual-kubelet/virtual-kubelet/providers/web"
-)
-
-// Compile time proof that our implementations meet the Provider interface.
-var _ Provider = (*alicloud.ECIProvider)(nil)
-var _ Provider = (*aws.FargateProvider)(nil)
-var _ Provider = (*azure.ACIProvider)(nil)
-var _ Provider = (*hypersh.HyperProvider)(nil)
-var _ Provider = (*web.BrokerProvider)(nil)
-var _ Provider = (*mock.MockProvider)(nil)
-var _ Provider = (*huawei.CCIProvider)(nil)
-var _ Provider = (*azurebatch.Provider)(nil)
-var _ Provider = (*sfmesh.SFMeshProvider)(nil)
-
-// start of providers not supported on windows
-var _ Provider = (*cri.CRIProvider)(nil)
-var _ Provider = (*vic.VicProvider)(nil)
-
-func lookupProvider(provider, providerConfig string, rm *manager.ResourceManager, nodeName, operatingSystem, internalIP string, daemonEndpointPort int32) (Provider, error) {
-	switch provider {
-	case "alicloud":
-		return alicloud.NewECIProvider(providerConfig, rm, nodeName, operatingSystem, internalIP, daemonEndpointPort)
-	case "aws":
-		return aws.NewFargateProvider(providerConfig, rm, nodeName, operatingSystem, internalIP, daemonEndpointPort)
-	case "azure":
-		return azure.NewACIProvider(providerConfig, rm, nodeName, operatingSystem, internalIP, daemonEndpointPort)
-	case "azurebatch":
-		return azurebatch.NewBatchProvider(providerConfig, rm, nodeName, operatingSystem, internalIP, daemonEndpointPort)
-	case "hyper":
-		return hypersh.NewHyperProvider(providerConfig, rm, nodeName, operatingSystem)
-	case "web":
-		return web.NewBrokerProvider(nodeName, operatingSystem, daemonEndpointPort)
-	case "mock":
-		return mock.NewMockProvider(providerConfig, nodeName, operatingSystem, internalIP, daemonEndpointPort)
-	case "huawei":
-		return huawei.NewCCIProvider(providerConfig, rm, nodeName, operatingSystem, internalIP, daemonEndpointPort)
-	case "sfmesh":
-		return sfmesh.NewSFMeshProvider(rm, nodeName, operatingSystem, internalIP, daemonEndpointPort)
-	// start of providers not supported on windows
-	case "cri":
-		return cri.NewCRIProvider(nodeName, operatingSystem, internalIP, rm, daemonEndpointPort)
-	case "vic":
-		return vic.NewVicProvider(providerConfig, rm, nodeName, operatingSystem)
-	default:
-		return nil, errors.New("provider not supported")
-	}
-}

vkubelet/lookup_darwin.go

@@ -1,50 +0,0 @@
-package vkubelet
-
-import (
-	"github.com/pkg/errors"
-	"github.com/virtual-kubelet/virtual-kubelet/manager"
-	"github.com/virtual-kubelet/virtual-kubelet/providers/aws"
-	"github.com/virtual-kubelet/virtual-kubelet/providers/azure"
-	"github.com/virtual-kubelet/virtual-kubelet/providers/azurebatch"
-	"github.com/virtual-kubelet/virtual-kubelet/providers/huawei"
-	"github.com/virtual-kubelet/virtual-kubelet/providers/hypersh"
-	"github.com/virtual-kubelet/virtual-kubelet/providers/mock"
-	"github.com/virtual-kubelet/virtual-kubelet/providers/sfmesh"
-	"github.com/virtual-kubelet/virtual-kubelet/providers/web"
-)
-
-// Compile time proof that our implementations meet the Provider interface.
-var _ Provider = (*alicloud.ECIProvider)(nil)
-var _ Provider = (*aws.FargateProvider)(nil)
-var _ Provider = (*azure.ACIProvider)(nil)
-var _ Provider = (*hypersh.HyperProvider)(nil)
-var _ Provider = (*web.BrokerProvider)(nil)
-var _ Provider = (*mock.MockProvider)(nil)
-var _ Provider = (*huawei.CCIProvider)(nil)
-var _ Provider = (*azurebatch.Provider)(nil)
-var _ Provider = (*sfmesh.SFMeshProvider)(nil)
-
-func lookupProvider(provider, providerConfig string, rm *manager.ResourceManager, nodeName, operatingSystem, internalIP string, daemonEndpointPort int32) (Provider, error) {
-	switch provider {
-	case "alicloud":
-		return alicloud.NewECIProvider(providerConfig, rm, nodeName, operatingSystem, internalIP, daemonEndpointPort)
-	case "aws":
-		return aws.NewFargateProvider(providerConfig, rm, nodeName, operatingSystem, internalIP, daemonEndpointPort)
-	case "azure":
-		return azure.NewACIProvider(providerConfig, rm, nodeName, operatingSystem, internalIP, daemonEndpointPort)
-	case "azurebatch":
-		return azurebatch.NewBatchProvider(providerConfig, rm, nodeName, operatingSystem, internalIP, daemonEndpointPort)
-	case "hyper":
-		return hypersh.NewHyperProvider(providerConfig, rm, nodeName, operatingSystem)
-	case "web":
-		return web.NewBrokerProvider(nodeName, operatingSystem, daemonEndpointPort)
-	case "mock":
-		return mock.NewMockProvider(providerConfig, nodeName, operatingSystem, internalIP, daemonEndpointPort)
-	case "huawei":
-		return huawei.NewCCIProvider(providerConfig, rm, nodeName, operatingSystem, internalIP, daemonEndpointPort)
-	case "sfmesh":
-		return sfmesh.NewSFMeshProvider(rm, nodeName, operatingSystem, internalIP, daemonEndpointPort)
-	default:
-		return nil, errors.New("provider is not supported")
-	}
-}

vkubelet/lookup_windows.go

@@ -1,47 +0,0 @@
-package vkubelet
-
-import (
-	"github.com/pkg/errors"
-	"github.com/virtual-kubelet/virtual-kubelet/manager"
-	"github.com/virtual-kubelet/virtual-kubelet/providers/aws"
-	"github.com/virtual-kubelet/virtual-kubelet/providers/azure"
-	"github.com/virtual-kubelet/virtual-kubelet/providers/azurebatch"
-	"github.com/virtual-kubelet/virtual-kubelet/providers/huawei"
-	"github.com/virtual-kubelet/virtual-kubelet/providers/hypersh"
-	"github.com/virtual-kubelet/virtual-kubelet/providers/mock"
-	"github.com/virtual-kubelet/virtual-kubelet/providers/sfmesh"
-	"github.com/virtual-kubelet/virtual-kubelet/providers/web"
-)
-
-// Compile time proof that our implementations meet the Provider interface.
-var _ Provider = (*aws.FargateProvider)(nil)
-var _ Provider = (*azure.ACIProvider)(nil)
-var _ Provider = (*hypersh.HyperProvider)(nil)
-var _ Provider = (*web.BrokerProvider)(nil)
-var _ Provider = (*mock.MockProvider)(nil)
-var _ Provider = (*huawei.CCIProvider)(nil)
-var _ Provider = (*azurebatch.Provider)(nil)
-var _ Provider = (*sfmesh.SFMeshProvider)(nil)
-
-func lookupProvider(provider, providerConfig string, rm *manager.ResourceManager, nodeName, operatingSystem, internalIP string, daemonEndpointPort int32) (Provider, error) {
-	switch provider {
-	case "aws":
-		return aws.NewFargateProvider(providerConfig, rm, nodeName, operatingSystem, internalIP, daemonEndpointPort)
-	case "azure":
-		return azure.NewACIProvider(providerConfig, rm, nodeName, operatingSystem, internalIP, daemonEndpointPort)
-	case "azurebatch":
-		return azurebatch.NewBatchProvider(providerConfig, rm, nodeName, operatingSystem, internalIP, daemonEndpointPort)
-	case "hyper":
-		return hypersh.NewHyperProvider(providerConfig, rm, nodeName, operatingSystem)
-	case "web":
-		return web.NewBrokerProvider(nodeName, operatingSystem, daemonEndpointPort)
-	case "mock":
-		return mock.NewMockProvider(providerConfig, nodeName, operatingSystem, internalIP, daemonEndpointPort)
-	case "huawei":
-		return huawei.NewCCIProvider(providerConfig, rm, nodeName, operatingSystem, internalIP, daemonEndpointPort)
-	case "sfmesh":
-		return sfmesh.NewSFMeshProvider(rm, nodeName, operatingSystem, internalIP, daemonEndpointPort)
-	default:
-		return nil, errors.New("provider not supported")
-	}
-}

vkubelet/provider.go

@@ -1,63 +0,0 @@
-package vkubelet
-
-import (
-	"context"
-	"io"
-	"time"
-
-	"k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/types"
-	"k8s.io/client-go/tools/remotecommand"
-	stats "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
-)
-
-// Provider contains the methods required to implement a virtual-kubelet provider.
-type Provider interface {
-	// CreatePod takes a Kubernetes Pod and deploys it within the provider.
-	CreatePod(ctx context.Context, pod *v1.Pod) error
-
-	// UpdatePod takes a Kubernetes Pod and updates it within the provider.
-	UpdatePod(ctx context.Context, pod *v1.Pod) error
-
-	// DeletePod takes a Kubernetes Pod and deletes it from the provider.
-	DeletePod(ctx context.Context, pod *v1.Pod) error
-
-	// GetPod retrieves a pod by name from the provider (can be cached).
-	GetPod(ctx context.Context, namespace, name string) (*v1.Pod, error)
-
-	// GetContainerLogs retrieves the logs of a container by name from the provider.
-	GetContainerLogs(ctx context.Context, namespace, podName, containerName string, tail int) (string, error)
-
-	// ExecInContainer executes a command in a container in the pod, copying data
-	// between in/out/err and the container's stdin/stdout/stderr.
-	ExecInContainer(name string, uid types.UID, container string, cmd []string, in io.Reader, out, err io.WriteCloser, tty bool, resize <-chan remotecommand.TerminalSize, timeout time.Duration) error
-
-	// GetPodStatus retrieves the status of a pod by name from the provider.
-	GetPodStatus(ctx context.Context, namespace, name string) (*v1.PodStatus, error)
-
-	// GetPods retrieves a list of all pods running on the provider (can be cached).
-	GetPods(context.Context) ([]*v1.Pod, error)
-
-	// Capacity returns a resource list with the capacity constraints of the provider.
-	Capacity(context.Context) v1.ResourceList
-
-	// NodeConditions returns a list of conditions (Ready, OutOfDisk, etc), which is
-	// polled periodically to update the node status within Kubernetes.
-	NodeConditions(context.Context) []v1.NodeCondition
-
-	// NodeAddresses returns a list of addresses for the node status
-	// within Kubernetes.
-	NodeAddresses(context.Context) []v1.NodeAddress
-
-	// NodeDaemonEndpoints returns NodeDaemonEndpoints for the node status
-	// within Kubernetes.
-	NodeDaemonEndpoints(context.Context) *v1.NodeDaemonEndpoints
-
-	// OperatingSystem returns the operating system the provider is for.
-	OperatingSystem() string
-}
-
-// PodMetricsProvider is an optional interface that providers can implement to expose pod stats
-type PodMetricsProvider interface {
-	GetStatsSummary(context.Context) (*stats.Summary, error)
-}

vkubelet/vkubelet.go

@@ -3,9 +3,9 @@ package vkubelet
 import (
 	"context"
 	"fmt"
+	"net"
 	"os"
 	"os/signal"
-	"strconv"
 	"strings"
 	"syscall"
 	"time"
@@ -13,14 +13,13 @@ import (
 	pkgerrors "github.com/pkg/errors"
 	"github.com/virtual-kubelet/virtual-kubelet/log"
 	"github.com/virtual-kubelet/virtual-kubelet/manager"
+	"github.com/virtual-kubelet/virtual-kubelet/providers"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/watch"
 	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/rest"
-	"k8s.io/client-go/tools/clientcmd"
 )
 
 const (
@@ -32,123 +31,73 @@ type Server struct {
 	nodeName        string
 	namespace       string
 	k8sClient       *kubernetes.Clientset
-	taint           corev1.Taint
-	disableTaint    bool
-	provider        Provider
+	taint           *corev1.Taint
+	provider        providers.Provider
 	podWatcher      watch.Interface
 	resourceManager *manager.ResourceManager
 }
 
-func getEnv(key, defaultValue string) string {
-	value, found := os.LookupEnv(key)
-	if found {
-		return value
-	}
-	return defaultValue
+// Config is used to configure a new server.
+type Config struct {
+	APIConfig       APIConfig
+	Client          *kubernetes.Clientset
+	MetricsAddr     string
+	Namespace       string
+	NodeName        string
+	Provider        providers.Provider
+	ResourceManager *manager.ResourceManager
+	Taint           *corev1.Taint
+}
+
+type APIConfig struct {
+	CertPath string
+	KeyPath  string
+	Addr     string
 }
 
 // New creates a new virtual-kubelet server.
-func New(nodeName, operatingSystem, namespace, kubeConfig, provider, providerConfig, taintKey string, disableTaint bool, metricsAddr string) (*Server, error) {
-	var config *rest.Config
-
-	// Check if the kubeConfig file exists.
-	if _, err := os.Stat(kubeConfig); !os.IsNotExist(err) {
-		// Get the kubeconfig from the filepath.
-		config, err = clientcmd.BuildConfigFromFlags("", kubeConfig)
-		if err != nil {
-			return nil, err
-		}
-	} else {
-		// Set to in-cluster config.
-		config, err = rest.InClusterConfig()
-		if err != nil {
-			return nil, err
-		}
+func New(ctx context.Context, cfg Config) (s *Server, retErr error) {
+	s = &Server{
+		namespace:       cfg.Namespace,
+		nodeName:        cfg.NodeName,
+		taint:           cfg.Taint,
+		k8sClient:       cfg.Client,
+		resourceManager: cfg.ResourceManager,
+		provider:        cfg.Provider,
 	}
 
-	if masterURI := os.Getenv("MASTER_URI"); masterURI != "" {
-		config.Host = masterURI
-	}
-
-	clientset, err := kubernetes.NewForConfig(config)
-	if err != nil {
-		return nil, err
-	}
-
-	rm, err := manager.NewResourceManager(clientset)
-	if err != nil {
-		return nil, pkgerrors.Wrap(err, "error creating resource manager")
-	}
-
-	daemonEndpointPortEnv := os.Getenv("KUBELET_PORT")
-	if daemonEndpointPortEnv == "" {
-		daemonEndpointPortEnv = "10250"
-	}
-	i64value, err := strconv.ParseInt(daemonEndpointPortEnv, 10, 32)
-	daemonEndpointPort := int32(i64value)
-
-	internalIP := os.Getenv("VKUBELET_POD_IP")
-
-	var defaultTaintKey string
-	var defaultTaintValue string
-	if taintKey != "" {
-		defaultTaintKey = taintKey
-		defaultTaintValue = ""
-	} else {
-		defaultTaintKey = "virtual-kubelet.io/provider"
-		defaultTaintValue = provider
-	}
-	vkTaintKey := getEnv("VKUBELET_TAINT_KEY", defaultTaintKey)
-	vkTaintValue := getEnv("VKUBELET_TAINT_VALUE", defaultTaintValue)
-	vkTaintEffectEnv := getEnv("VKUBELET_TAINT_EFFECT", "NoSchedule")
-	var vkTaintEffect corev1.TaintEffect
-	switch vkTaintEffectEnv {
-	case "NoSchedule":
-		vkTaintEffect = corev1.TaintEffectNoSchedule
-	case "NoExecute":
-		vkTaintEffect = corev1.TaintEffectNoExecute
-	case "PreferNoSchedule":
-		vkTaintEffect = corev1.TaintEffectPreferNoSchedule
-	default:
-		return nil, pkgerrors.Errorf("taint effect %q is not supported", vkTaintEffectEnv)
-	}
-
-	taint := corev1.Taint{
-		Key:    vkTaintKey,
-		Value:  vkTaintValue,
-		Effect: vkTaintEffect,
-	}
-
-	p, err := lookupProvider(provider, providerConfig, rm, nodeName, operatingSystem, internalIP, daemonEndpointPort)
-	if err != nil {
-		return nil, err
-	}
-
-	s := &Server{
-		namespace:       namespace,
-		nodeName:        nodeName,
-		taint:           taint,
-		disableTaint:    disableTaint,
-		k8sClient:       clientset,
-		resourceManager: rm,
-		provider:        p,
-	}
-
-	ctx := context.TODO()
 	ctx = log.WithLogger(ctx, log.G(ctx))
 
-	if err = s.registerNode(ctx); err != nil {
-		return s, err
+	apiL, err := net.Listen("tcp", cfg.APIConfig.Addr)
+	if err != nil {
+		return nil, pkgerrors.Wrap(err, "error setting up API listener")
 	}
+	defer func() {
+		if retErr != nil {
+			apiL.Close()
+		}
+	}()
+	go KubeletServerStart(cfg.Provider, apiL, cfg.APIConfig.CertPath, cfg.APIConfig.KeyPath)
 
-	go KubeletServerStart(p)
-
-	if metricsAddr != "" {
-		go MetricsServerStart(p, metricsAddr)
+	if cfg.MetricsAddr != "" {
+		metricsL, err := net.Listen("tcp", cfg.MetricsAddr)
+		if err != nil {
+			return nil, pkgerrors.Wrap(err, "error setting up metrics listener")
+		}
+		defer func() {
+			if retErr != nil {
+				metricsL.Close()
+			}
+		}()
+		go MetricsServerStart(cfg.Provider, metricsL)
 	} else {
 		log.G(ctx).Info("Skipping metrics server startup since no address was provided")
 	}
 
+	if err := s.registerNode(ctx); err != nil {
+		return s, err
+	}
+
 	tick := time.Tick(5 * time.Second)
 
 	go func() {
@@ -165,8 +114,8 @@ func New(nodeName, operatingSystem, namespace, kubeConfig, provider, providerCon
 func (s *Server) registerNode(ctx context.Context) error {
 	taints := make([]corev1.Taint, 0)
 
-	if !s.disableTaint {
-		taints = append(taints, s.taint)
+	if s.taint != nil {
+		taints = append(taints, *s.taint)
 	}
 
 	node := &corev1.Node{
@@ -350,10 +299,10 @@ func (s *Server) reconcile(ctx context.Context) {
 		}
 
 		if providerPod == nil &&
-		   pod.DeletionTimestamp == nil &&
-		   pod.Status.Phase != corev1.PodSucceeded &&
-		   pod.Status.Phase != corev1.PodFailed &&
-		   pod.Status.Reason != PodStatusReason_ProviderFailed {
+			pod.DeletionTimestamp == nil &&
+			pod.Status.Phase != corev1.PodSucceeded &&
+			pod.Status.Phase != corev1.PodFailed &&
+			pod.Status.Reason != PodStatusReason_ProviderFailed {
 			logger.Debug("Creating pod")
 			if err := s.createPod(ctx, pod); err != nil {
 				logger.WithError(err).Error("Error creating pod")
@@ -436,8 +385,8 @@ func (s *Server) updatePodStatuses(ctx context.Context) {
 	pods := s.resourceManager.GetPods()
 	for _, pod := range pods {
 		if pod.Status.Phase == corev1.PodSucceeded ||
-		   pod.Status.Phase == corev1.PodFailed ||
-		   pod.Status.Reason == PodStatusReason_ProviderFailed {
+			pod.Status.Phase == corev1.PodFailed ||
+			pod.Status.Reason == PodStatusReason_ProviderFailed {
 			continue
 		}