Add OpenTelemetry tracing to minikube

cmd/minikube/cmd/start.go

@@ -60,6 +60,7 @@ import (
 	"k8s.io/minikube/pkg/minikube/out/register"
 	"k8s.io/minikube/pkg/minikube/reason"
 	"k8s.io/minikube/pkg/minikube/style"
+	pkgtrace "k8s.io/minikube/pkg/trace"
 
 	"k8s.io/minikube/pkg/minikube/registry"
 	"k8s.io/minikube/pkg/minikube/translate"
@@ -129,6 +130,10 @@ func runStart(cmd *cobra.Command, args []string) {
 	register.SetEventLogPath(localpath.EventLog(ClusterFlagValue()))
 
 	out.SetJSON(outputFormat == "json")
+	if err := pkgtrace.Initialize(viper.GetString(trace)); err != nil {
+		exit.Message(reason.Usage, "error initializing tracing: {{.Error}}", out.V{"Error": err.Error()})
+	}
+	defer pkgtrace.Cleanup()
 	displayVersion(version.GetVersion())
 
 	// No need to do the update check if no one is going to see it

cmd/minikube/cmd/start_flags.go

@@ -108,6 +108,7 @@ const (
 	kicBaseImage            = "base-image"
 	ports                   = "ports"
 	startNamespace          = "namespace"
+	trace                   = "trace"
 )
 
 var (
@@ -152,6 +153,7 @@ func initMinikubeFlags() {
 	startCmd.Flags().Bool(deleteOnFailure, false, "If set, delete the current cluster if start fails and try again. Defaults to false.")
 	startCmd.Flags().Bool(forceSystemd, false, "If set, force the container runtime to use sytemd as cgroup manager. Currently available for docker and crio. Defaults to false.")
 	startCmd.Flags().StringVarP(&outputFormat, "output", "o", "text", "Format to print stdout in. Options include: [text,json]")
+	startCmd.Flags().StringP(trace, "", "", "Send trace events. Options include: [gcp]")
 }
 
 // initKubernetesFlags inits the commandline flags for Kubernetes related options

go.mod

@@ -3,8 +3,9 @@ module k8s.io/minikube
 go 1.15
 
 require (
-	cloud.google.com/go/storage v1.8.0
+	cloud.google.com/go/storage v1.10.0
 	github.com/Azure/azure-sdk-for-go v42.3.0+incompatible
+	github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v0.13.0
 	github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5 // indirect
 	github.com/Parallels/docker-machine-parallels/v2 v2.0.1
 	github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
@@ -74,21 +75,25 @@ require (
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
 	github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f
 	github.com/zchee/go-vmnet v0.0.0-20161021174912-97ebf9174097
+	go.opencensus.io v0.22.4
+	go.opentelemetry.io/otel v0.13.0
+	go.opentelemetry.io/otel/sdk v0.13.0
 	golang.org/x/build v0.0.0-20190927031335-2835ba2e683f
 	golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a
 	golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6
 	golang.org/x/mod v0.3.0
 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
-	golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a
+	golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208
 	golang.org/x/sys v0.0.0-20200523222454-059865788121
-	golang.org/x/text v0.3.2
-	google.golang.org/api v0.25.0
+	golang.org/x/text v0.3.3
+	google.golang.org/api v0.29.0
 	gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect
 	gopkg.in/yaml.v2 v2.3.0
 	gotest.tools/v3 v3.0.2 // indirect
 	k8s.io/api v0.17.4
 	k8s.io/apimachinery v0.17.4
 	k8s.io/client-go v0.17.4
+	k8s.io/klog v1.0.0
 	k8s.io/klog/v2 v2.4.0
 	k8s.io/kubectl v0.0.0
 	k8s.io/kubernetes v1.18.5

pkg/minikube/out/register/register.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 
 	"k8s.io/klog/v2"
+	"k8s.io/minikube/pkg/trace"
 )
 
 const (
@@ -117,13 +118,16 @@ func (r *Register) currentStep() string {
 
 // SetStep sets the current step
 func (r *Register) SetStep(s RegStep) {
+	defer trace.StartSpan(string(s))
 	if r.first == RegStep("") {
 		_, ok := r.steps[s]
 		if ok {
 			r.first = s
 		} else {
 			klog.Errorf("unexpected first step: %q", r.first)
 		}
+	} else {
+		trace.EndSpan(string(r.current))
 	}
 
 	r.current = s

pkg/trace/gcp.go

@@ -0,0 +1,104 @@
+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package trace
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"go.opentelemetry.io/otel/api/trace"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+	"k8s.io/klog"
+
+	texporter "github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace"
+	"github.com/pkg/errors"
+	"go.opentelemetry.io/otel/api/global"
+)
+
+const (
+	projectEnvVar = "MINIKUBE_GCP_PROJECT_ID"
+	// this is the name of the parent span to help identify it
+	// in the Cloud Trace UI.
+	parentSpanName = "minikube start"
+)
+
+type gcpTracer struct {
+	projectID string
+	parentCtx context.Context
+	trace.Tracer
+	spans   map[string]trace.Span
+	cleanup func()
+}
+
+// StartSpan starts a span for the next step of
+// `minikube start` via the GCP tracer
+func (t *gcpTracer) StartSpan(name string) {
+	_, span := t.Tracer.Start(t.parentCtx, name)
+	t.spans[name] = span
+}
+
+// EndSpan ends the most recent span, indicating
+// that one step of `minikube start` has completed
+func (t *gcpTracer) EndSpan(name string) {
+	span, ok := t.spans[name]
+	if !ok {
+		klog.Warningf("cannot end span %s as it was never started", name)
+		return
+	}
+	span.End()
+}
+
+func (t *gcpTracer) Cleanup() {
+	span, ok := t.spans[parentSpanName]
+	if ok {
+		span.End()
+	}
+	t.cleanup()
+}
+
+func initGCPTracer() (*gcpTracer, error) {
+	projectID := os.Getenv(projectEnvVar)
+	if projectID == "" {
+		return nil, fmt.Errorf("GCP tracer requires a valid GCP project id set via the %s env variable", projectEnvVar)
+	}
+
+	_, flush, err := texporter.InstallNewPipeline(
+		[]texporter.Option{
+			texporter.WithProjectID(projectID),
+		},
+		sdktrace.WithConfig(sdktrace.Config{
+			DefaultSampler: sdktrace.AlwaysSample(),
+		}),
+	)
+	if err != nil {
+		return nil, errors.Wrap(err, "installing pipeline")
+	}
+
+	t := global.Tracer(parentSpanName)
+
+	ctx, span := t.Start(context.Background(), parentSpanName)
+	return &gcpTracer{
+		projectID: projectID,
+		parentCtx: ctx,
+		cleanup:   flush,
+		Tracer:    t,
+		spans: map[string]trace.Span{
+			parentSpanName: span,
+		},
+	}, nil
+}

pkg/trace/trace.go

@@ -0,0 +1,78 @@
+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package trace
+
+import (
+	"fmt"
+
+	"github.com/pkg/errors"
+)
+
+var (
+	tracer minikubeTracer
+)
+
+type minikubeTracer interface {
+	StartSpan(string)
+	EndSpan(string)
+	Cleanup()
+}
+
+// Initialize intializes the global tracer variable
+func Initialize(t string) error {
+	tr, err := getTracer(t)
+	if err != nil {
+		return errors.Wrap(err, "getting tracer")
+	}
+	tracer = tr
+	return nil
+}
+
+func getTracer(t string) (minikubeTracer, error) {
+	switch t {
+	case "gcp":
+		return initGCPTracer()
+	case "":
+		return nil, nil
+	}
+	return nil, fmt.Errorf("%s is not a valid tracer, valid tracers include: [gcp]", t)
+}
+
+// StartSpan starts a span with the given name
+func StartSpan(name string) {
+	if tracer == nil {
+		return
+	}
+	tracer.StartSpan(name)
+}
+
+// EndSpan ends a span with the given name
+func EndSpan(name string) {
+	if tracer == nil {
+		return
+	}
+	tracer.EndSpan(name)
+}
+
+// Cleanup is responsible for trace related cleanup,
+// such as flushing all data
+func Cleanup() {
+	if tracer == nil {
+		return
+	}
+	tracer.Cleanup()
+}

site/content/en/docs/commands/start.md

@@ -87,6 +87,7 @@ minikube start [flags]
       --preload                           If set, download tarball of preloaded images if available to improve start time. Defaults to true. (default true)
       --registry-mirror strings           Registry mirrors to pass to the Docker daemon
       --service-cluster-ip-range string   The CIDR to be used for service cluster IPs. (default "10.96.0.0/12")
+      --trace string                      Send trace events. Options include: [gcp]
       --uuid string                       Provide VM UUID to restore MAC address (hyperkit driver only)
       --vm                                Filter to use only VM Drivers
       --vm-driver driver                  DEPRECATED, use driver instead.

site/content/en/docs/tutorials/telemetry.md

@@ -0,0 +1,24 @@
+---
+title: "Telemetry"
+linkTitle: "telemetry"
+weight: 1
+date: 2020-11-24
+---
+
+## Overview
+
+minikube provides telemetry suppport via [OpenTelemetry tracing](https://opentelemetry.io/about/) to collect trace data for `minikube start`.
+
+Currently, minikube supports the following exporters for tracing data:
+- [Stackdriver](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/master/exporter/stackdriverexporter)
+
+To collect trace data with minikube and the Stackdriver exporter, run:
+
+```
+MINIKUBE_GCP_PROJECT_ID=<project ID> minikube start --output json --trace gcp
+```
+
+## Contributing
+There are many exporters available via [OpenTelemetry community contributions](https://github.com/open-telemetry/opentelemetry-collector-contrib).
+
+If you would like to see additional exporters, please create an [issue](https://github.com/kubernetes/minikube/issues) or refer to our [contribution][https://minikube.sigs.k8s.io/docs/contrib/] guidelines and submit a pull request. Thank you!