Add support for tracing via OpenCencus

This adds a few flags for configuring the tracer.
Includes support for jaeger tracing (built into OC).

cencus_jaeger.go

@@ -0,0 +1,36 @@
+// +build !no_jaeger_exporter
+
+package cmd
+
+import (
+	"errors"
+	"os"
+
+	"go.opencensus.io/exporter/jaeger"
+	"go.opencensus.io/trace"
+)
+
+func init() {
+	RegisterTracingExporter("jaeger", NewJaegerExporter)
+}
+
+func NewJaegerExporter(opts TracingExporterOptions) (trace.Exporter, error) {
+	jOpts := jaeger.Options{
+		Endpoint:      os.Getenv("JAEGER_ENDPOINT"),
+		AgentEndpoint: os.Getenv("JAEGER_AGENT_ENDPOINT"),
+		Username:      os.Getenv("JAEGER_USER"),
+		Password:      os.Getenv("JAEGER_PASSWORD"),
+		Process: jaeger.Process{
+			ServiceName: opts.ServiceName,
+		},
+	}
+
+	if jOpts.Endpoint == "" && jOpts.AgentEndpoint == "" {
+		return nil, errors.New("Must specify either JAEGER_ENDPOINT or JAEGER_AGENT_ENDPOINT")
+	}
+
+	for k, v := range opts.Tags {
+		jOpts.Process.Tags = append(jOpts.Process.Tags, jaeger.StringTag(k, v))
+	}
+	return jaeger.NewExporter(jOpts)
+}

census.go

@@ -0,0 +1,51 @@
+package cmd
+
+import (
+	"github.com/pkg/errors"
+	"go.opencensus.io/trace"
+)
+
+var (
+	tracingExporters = make(map[string]TracingExporterInitFunc)
+
+	reservedTagNames = map[string]bool{
+		"operatingSystem": true,
+		"provider":        true,
+		"nodeName":        true,
+	}
+)
+
+// TracingExporterOptions is used to pass options to the configured tracer
+type TracingExporterOptions struct {
+	Tags        map[string]string
+	ServiceName string
+}
+
+// TracingExporterInitFunc is the function that is called to initialize an exporter.
+// This is used when registering an exporter and called when a user specifed they want to use the exporter.
+type TracingExporterInitFunc func(TracingExporterOptions) (trace.Exporter, error)
+
+// RegisterTracingExporter registers a tracing exporter.
+// For a user to select an exporter, it must be registered here.
+func RegisterTracingExporter(name string, f TracingExporterInitFunc) {
+	tracingExporters[name] = f
+}
+
+// GetTracingExporter gets the specified tracing exporter passing in the options to the exporter init function.
+// For an exporter to be availbale here it must be registered with `RegisterTracingExporter`.
+func GetTracingExporter(name string, opts TracingExporterOptions) (trace.Exporter, error) {
+	f, ok := tracingExporters[name]
+	if !ok {
+		return nil, errors.Errorf("tracing exporter %q not found", name)
+	}
+	return f(opts)
+}
+
+// AvailableTraceExporters gets the list of registered exporters
+func AvailableTraceExporters() []string {
+	out := make([]string, 0, len(tracingExporters))
+	for k := range tracingExporters {
+		out = append(out, k)
+	}
+	return out
+}

root.go

@@ -33,6 +33,7 @@ import (
 	"github.com/virtual-kubelet/virtual-kubelet/providers"
 	"github.com/virtual-kubelet/virtual-kubelet/providers/register"
 	vkubelet "github.com/virtual-kubelet/virtual-kubelet/vkubelet"
+	"go.opencensus.io/trace"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/client-go/kubernetes"
 )
@@ -58,6 +59,10 @@ var p providers.Provider
 var rm *manager.ResourceManager
 var apiConfig vkubelet.APIConfig
 
+var userTraceExporters []string
+var userTraceConfig = TracingExporterOptions{Tags: make(map[string]string)}
+var traceSampler string
+
 // RootCmd represents the base command when called without any subcommands
 var RootCmd = &cobra.Command{
 	Use:   "virtual-kubelet",
@@ -94,6 +99,38 @@ func Execute() {
 	}
 }
 
+type mapVar map[string]string
+
+func (mv mapVar) String() string {
+	var s string
+	for k, v := range mv {
+		if s == "" {
+			s = fmt.Sprintf("%s=%v", k, v)
+		} else {
+			s += fmt.Sprintf(", %s=%v", k, v)
+		}
+	}
+	return s
+}
+
+func (mv mapVar) Set(s string) error {
+	split := strings.SplitN(s, "=", 2)
+	if len(split) != 2 {
+		return errors.Errorf("invalid format, must be `key=value`: %s", s)
+	}
+
+	_, ok := mv[split[0]]
+	if ok {
+		return errors.Errorf("duplicate key: %s", split[0])
+	}
+	mv[split[0]] = split[1]
+	return nil
+}
+
+func (mv mapVar) Type() string {
+	return "map"
+}
+
 func init() {
 	cobra.OnInitialize(initConfig)
 
@@ -120,6 +157,11 @@ func init() {
 	RootCmd.PersistentFlags().MarkDeprecated("taint", "Taint key should now be configured using the VK_TAINT_KEY environment variable")
 	RootCmd.PersistentFlags().StringVar(&logLevel, "log-level", "info", `set the log level, e.g. "trace", debug", "info", "warn", "error"`)
 
+	RootCmd.PersistentFlags().StringSliceVar(&userTraceExporters, "trace-exporter", nil, fmt.Sprintf("sets the tracing exporter to use, available exporters: %s", AvailableTraceExporters()))
+	RootCmd.PersistentFlags().StringVar(&userTraceConfig.ServiceName, "trace-service-name", "virtual-kubelet", "sets the name of the service used to register with the trace exporter")
+	RootCmd.PersistentFlags().Var(mapVar(userTraceConfig.Tags), "trace-tag", "add tags to include with traces in key=value form")
+	RootCmd.PersistentFlags().StringVar(&traceSampler, "trace-sample-rate", "", "set probability of tracing samples")
+
 	// Cobra also supports local flags, which will only run
 	// when this action is called directly.
 	// RootCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
@@ -223,6 +265,49 @@ func initConfig() {
 	if err != nil {
 		logger.WithError(err).Fatal("Error reading API config")
 	}
+
+	for k := range userTraceConfig.Tags {
+		if reservedTagNames[k] {
+			logger.WithField("tag", k).Fatal("must not use a reserved tag key")
+		}
+	}
+	userTraceConfig.Tags["operatingSystem"] = operatingSystem
+	userTraceConfig.Tags["provider"] = provider
+	userTraceConfig.Tags["nodeName"] = nodeName
+	for _, e := range userTraceExporters {
+		exporter, err := GetTracingExporter(e, userTraceConfig)
+		if err != nil {
+			log.L.WithError(err).WithField("exporter", e).Fatal("Cannot initialize exporter")
+		}
+		trace.RegisterExporter(exporter)
+	}
+	if len(userTraceExporters) > 0 {
+		var s trace.Sampler
+		switch strings.ToLower(traceSampler) {
+		case "":
+		case "always":
+			s = trace.AlwaysSample()
+		case "never":
+			s = trace.NeverSample()
+		default:
+			rate, err := strconv.Atoi(traceSampler)
+			if err != nil {
+				logger.WithError(err).WithField("rate", traceSampler).Fatal("unsupported trace sample rate, supported values: always, never, or number 0-100")
+			}
+			if rate < 0 || rate > 100 {
+				logger.WithField("rate", traceSampler).Fatal("trace sample rate must not be less than zero or greater than 100")
+			}
+			s = trace.ProbabilitySampler(float64(rate) / 100)
+		}
+
+		if s != nil {
+			trace.ApplyConfig(
+				trace.Config{
+					DefaultSampler: s,
+				},
+			)
+		}
+	}
 }
 
 func getAPIConfig() (vkubelet.APIConfig, error) {