feat(devstack): propagate observability context

devnet-sdk/devstack/devtest/package.go

@@ -7,6 +7,7 @@ import (
 	"sync"
 
 	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/trace"
 
 	"github.com/ethereum/go-ethereum/log"
@@ -107,7 +108,7 @@ func (t *implP) Logger() Logger {
 }
 
 func (t *implP) Tracer() trace.Tracer {
-	return nil
+	return otel.Tracer(t.Name())
 }
 
 func (t *implP) Ctx() context.Context {
@@ -161,8 +162,8 @@ func (t *implP) _PackageOnly() {
 	panic("do not use - this method only forces the interface to be unique")
 }
 
-func NewP(logger log.Logger, onFail func()) P {
-	ctx, cancel := context.WithCancel(context.Background())
+func NewP(ctx context.Context, logger log.Logger, onFail func()) P {
+	ctx, cancel := context.WithCancel(ctx)
 	out := &implP{
 		scopeName: "pkg",
 		logger:    &pkgLogger{logger},

devnet-sdk/devstack/devtest/testing.go

@@ -16,6 +16,12 @@ import (
 
 const ExpectPreconditionsMet = "DEVNET_EXPECT_PRECONDITIONS_MET"
 
+var (
+	// RootContext is the context that is used for the root of the test suite.
+	// It should be set for good before any tests are run.
+	RootContext = context.Background()
+)
+
 type T interface {
 	CommonT
 
@@ -130,7 +136,7 @@ func (t *testingT) Run(name string, fn func(T)) {
 		ctx, cancel := context.WithCancel(t.ctx)
 		subGoT.Cleanup(cancel)
 
-		tracer := otel.GetTracerProvider().Tracer(baseName + "::" + name)
+		tracer := otel.Tracer(baseName + "::" + name)
 		ctx, span := tracer.Start(ctx, name)
 		subGoT.Cleanup(func() {
 			span.End()
@@ -205,10 +211,10 @@ var _ T = (*testingT)(nil)
 
 // SerialT wraps around a test-logger and turns it into a T for devstack testing.
 func SerialT(t *testing.T) T {
-	ctx, cancel := context.WithCancel(context.Background())
+	ctx, cancel := context.WithCancel(RootContext)
 	t.Cleanup(cancel)
 
-	tracer := otel.GetTracerProvider().Tracer(t.Name())
+	tracer := otel.Tracer(t.Name())
 	ctx, span := tracer.Start(ctx, t.Name())
 	t.Cleanup(func() {
 		span.End()

devnet-sdk/devstack/example/example_test.go

@@ -27,26 +27,7 @@ func TestExample2(gt *testing.T) {
 	sys.Supervisor.VerifySyncStatus(dsl.WithAllLocalUnsafeHeadsAdvancedBy(4))
 }
 
-func TestExampleTxs(gt *testing.T) {
-	t := devtest.ParallelT(gt)
-	sys := SimpleInterop(t)
-	require := t.Require()
-
-	pre := eth.OneEther
-	alice := sys.FunderA.NewFundedEOA(pre)
-
-	bob := sys.Wallet.NewEOA(sys.L2ELA)
-	bob.VerifyBalanceExact(eth.ZeroWei)
-
-	transferred := eth.GWei(42)
-	tx := alice.Transfer(bob.Address(), transferred)
-	require.Equal(params.TxGas, tx.Included.Value().GasUsed, "transfers cost 21k gas")
-
-	alice.VerifyBalanceLessThan(pre.Sub(transferred)) // less than, because of the tx fee
-	bob.VerifyBalanceExact(transferred)
-}
-
-func TestExampleTracing(gt *testing.T) {
+func TestExampleTxsTracing(gt *testing.T) {
 	t := devtest.ParallelT(gt)
 	ctx := t.Ctx()
 	require := t.Require()

devnet-sdk/devstack/presets/orchestrator.go

@@ -1,13 +1,15 @@
 package presets
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"runtime/debug"
 	"sync/atomic"
 	"testing"
 
 	"github.com/ethereum/go-ethereum/log"
+	"go.opentelemetry.io/otel"
 
 	"github.com/ethereum-optimism/optimism/devnet-sdk/devstack/devtest"
 	"github.com/ethereum-optimism/optimism/devnet-sdk/devstack/stack"
@@ -39,7 +41,7 @@ func DoMain(m *testing.M, opts ...stack.Option) {
 		defer func() {
 			if x := recover(); x != nil {
 				_, _ = fmt.Fprintf(os.Stderr, "Panic during test Main: %v\n", x)
-				_, _ = fmt.Fprintf(os.Stderr, "Stacktrace from panic: \n"+string(debug.Stack()))
+				_, _ = fmt.Fprint(os.Stderr, "Stacktrace from panic: \n"+string(debug.Stack()))
 
 				failed.Store(true)
 			}
@@ -53,14 +55,18 @@ func DoMain(m *testing.M, opts ...stack.Option) {
 			Pid:    false,
 		})
 
-		otelShutdown, err := telemetry.SetupOpenTelemetry("devstack")
+		ctx, otelShutdown, err := telemetry.SetupOpenTelemetry(context.Background())
 		if err != nil {
 			logger.Warn("Failed to setup OpenTelemetry", "error", err)
 		} else {
 			defer otelShutdown()
 		}
 
-		p := devtest.NewP(logger, func() {
+		ctx, run := otel.Tracer("run").Start(ctx, "test suite")
+		defer run.End()
+
+		devtest.RootContext = ctx
+		p := devtest.NewP(ctx, logger, func() {
 			debug.PrintStack()
 			failed.Store(true)
 			panic("setup fail")
@@ -75,7 +81,7 @@ func DoMain(m *testing.M, opts ...stack.Option) {
 		// TODO(#15139): set log-level filter, reduce noise
 		//log.SetDefault(t.Log.New("logger", "global"))
 
-		initOrchestrator(p, opts...)
+		initOrchestrator(ctx, p, opts...)
 
 		errCode = m.Run()
 		return
@@ -84,7 +90,10 @@ func DoMain(m *testing.M, opts ...stack.Option) {
 	os.Exit(code)
 }
 
-func initOrchestrator(p devtest.P, opts ...stack.Option) {
+func initOrchestrator(ctx context.Context, p devtest.P, opts ...stack.Option) {
+	ctx, span := p.Tracer().Start(ctx, "initializing orchestrator")
+	defer span.End()
+
 	lockedOrchestrator.Lock()
 	defer lockedOrchestrator.Unlock()
 	if lockedOrchestrator.Value != nil {
@@ -95,10 +104,13 @@ func initOrchestrator(p devtest.P, opts ...stack.Option) {
 		p.Logger().Warn("Selecting sysgo as default devstack orchestrator")
 		kind = "sysgo"
 	}
+
 	switch kind {
 	case "sysgo":
+		p.Logger().WithContext(ctx).Info("initializing sysgo orchestrator")
 		lockedOrchestrator.Value = sysgo.NewOrchestrator(p)
 	case "syskt":
+		p.Logger().WithContext(ctx).Info("initializing sysext orchestrator")
 		lockedOrchestrator.Value = sysext.NewOrchestrator(p)
 	default:
 		p.Logger().Crit("Unknown devstack backend", "kind", kind)

devnet-sdk/devstack/sysgo/control_plane_test.go

@@ -25,10 +25,14 @@ func TestControlPlane(gt *testing.T) {
 
 	logger := testlog.Logger(gt, log.LevelInfo)
 
-	p := devtest.NewP(logger, func() {
-		gt.Helper()
-		gt.FailNow()
-	})
+	p := devtest.NewP(
+		context.Background(),
+		logger,
+		func() {
+			gt.Helper()
+			gt.FailNow()
+		},
+	)
 	gt.Cleanup(p.Close)
 
 	orch := NewOrchestrator(p)

devnet-sdk/devstack/sysgo/sync_test.go

@@ -24,7 +24,7 @@ func TestL2CLResync(gt *testing.T) {
 
 	logger := testlog.Logger(gt, log.LevelInfo)
 
-	p := devtest.NewP(logger, func() {
+	p := devtest.NewP(context.Background(), logger, func() {
 		gt.Helper()
 		gt.FailNow()
 	})
@@ -125,7 +125,7 @@ func TestL2CLSyncP2P(gt *testing.T) {
 
 	logger := testlog.Logger(gt, log.LevelInfo)
 
-	p := devtest.NewP(logger, func() {
+	p := devtest.NewP(context.Background(), logger, func() {
 		gt.Helper()
 		gt.FailNow()
 	})
@@ -241,7 +241,7 @@ func TestUnsafeChainUnknownToL2CL(gt *testing.T) {
 
 	logger := testlog.Logger(gt, log.LevelInfo)
 
-	p := devtest.NewP(logger, func() {
+	p := devtest.NewP(context.Background(), logger, func() {
 		gt.Helper()
 		gt.FailNow()
 	})

devnet-sdk/devstack/sysgo/system_test.go

@@ -20,7 +20,7 @@ func TestSystem(gt *testing.T) {
 
 	logger := testlog.Logger(gt, log.LevelInfo)
 
-	p := devtest.NewP(logger, func() {
+	p := devtest.NewP(context.Background(), logger, func() {
 		gt.Helper()
 		gt.FailNow()
 	})

devnet-sdk/devstack/telemetry/carrier.go

@@ -0,0 +1,55 @@
+package telemetry
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"go.opentelemetry.io/otel/propagation"
+)
+
+const CarrierEnvVarPrefix = "OTEL_DEVSTACK_PROPAGATOR_CARRIER_"
+
+// keep in sync with textPropagator() below
+var defaultPropagators = []string{
+	"tracecontext",
+	"baggage",
+}
+
+func textPropagator() propagation.TextMapPropagator {
+	return propagation.NewCompositeTextMapPropagator(
+		// keep in sync with propagators above
+		propagation.TraceContext{},
+		propagation.Baggage{},
+	)
+}
+
+func InstrumentEnvironment(ctx context.Context, env []string) []string {
+	propagator := textPropagator()
+	carrier := propagation.MapCarrier{}
+	propagator.Inject(ctx, carrier)
+
+	for k, v := range carrier {
+		env = append(env, fmt.Sprintf("%s%s=%s", CarrierEnvVarPrefix, k, v))
+	}
+
+	return env
+}
+
+func ExtractEnvironment(ctx context.Context, env []string) (context.Context, error) {
+	carrier := propagation.MapCarrier{}
+	// Reconstruct the carrier from the environment variables
+	for _, e := range env {
+		if strings.HasPrefix(e, CarrierEnvVarPrefix) {
+			parts := strings.SplitN(e, "=", 2)
+			if len(parts) == 2 {
+				key := strings.TrimPrefix(parts[0], CarrierEnvVarPrefix)
+				value := parts[1]
+				carrier.Set(key, value)
+			}
+		}
+	}
+
+	ctx = textPropagator().Extract(ctx, carrier)
+	return ctx, nil
+}

devnet-sdk/devstack/telemetry/init.go

@@ -1,23 +1,52 @@
 package telemetry
 
 import (
+	"context"
 	"os"
 
 	"github.com/honeycombio/otel-config-go/otelconfig"
 )
 
-func SetupOpenTelemetry(svc string) (func(), error) {
+const (
+	serviceNameEnvVar    = "OTEL_SERVICE_NAME"
+	serviceVersionEnvVar = "OTEL_SERVICE_VERSION"
+)
+
+var (
+	serviceName    = envOrDefault(serviceNameEnvVar, "devstack")
+	serviceVersion = envOrDefault(serviceVersionEnvVar, "0.0.0")
+)
+
+func envOrDefault(key, def string) string {
+	if v, ok := os.LookupEnv(key); ok {
+		return v
+	}
+	return def
+}
+
+func SetupOpenTelemetry(ctx context.Context, opts ...otelconfig.Option) (context.Context, func(), error) {
 	// do not use localhost:4317 by default, we want telemetry to be opt-in and
 	// explicit.
 	if os.Getenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT") == "" {
-		return func() {}, nil
+		return ctx, func() {}, nil
 	}
 
-	otelShutdown, err := otelconfig.ConfigureOpenTelemetry(
-		otelconfig.WithServiceName(svc),
-	)
+	opts = append([]otelconfig.Option{
+		otelconfig.WithServiceName(serviceName),
+		otelconfig.WithServiceVersion(serviceVersion),
+		otelconfig.WithPropagators(defaultPropagators),
+	}, opts...)
+	otelShutdown, err := otelconfig.ConfigureOpenTelemetry(opts...)
 	if err != nil {
-		return nil, err
+		return ctx, nil, err
 	}
-	return otelShutdown, nil
+
+	// If the environment contains carrier information, extract it.
+	// This is useful for test runner / test communication for example.
+	ctx, err = ExtractEnvironment(ctx, os.Environ())
+	if err != nil {
+		return ctx, nil, err
+	}
+
+	return ctx, otelShutdown, nil
 }