feat: add sampling support for tracing

apps/engineering/content/architecture/services/api/config.mdx

@@ -165,6 +165,25 @@ These options configure analytics storage and observability for the Unkey API.
   ```
 </Property>
 
+<Property name="--otel-trace-sampling-rate | UNKEY_OTEL_TRACE_SAMPLING_RATE" type="float64" defaultValue={0.25} required={false}>
+  Sets the sampling rate for OpenTelemetry traces as a decimal value between 0.0 and 1.0. This controls what percentage of traces will be collected and exported, helping to balance observability needs with performance and cost considerations.
+
+  - 0.0 means no traces are sampled (0%)
+  - 0.25 means 25% of traces are sampled (default)
+  - 1.0 means all traces are sampled (100%)
+
+  Lower sampling rates reduce overhead and storage costs but provide less visibility. Higher rates give more comprehensive data but increase resource usage and costs.
+
+  This setting only takes effect when OpenTelemetry is enabled with `--otel=true`.
+
+  **Examples:**
+  - `--otel-trace-sampling-rate=0.1` - Sample 10% of traces
+  - `--otel-trace-sampling-rate=0.25` - Sample 25% of traces (default)
+  - `--otel-trace-sampling-rate=1.0` - Sample all traces
+
+  **Environment variable:** `UNKEY_OTEL_TRACE_SAMPLING_RATE`
+</Property>
+
 <Property name="--color | UNKEY_COLOR" type="boolean" defaultValue={false} required={false}>
   Enable ANSI color codes in log output. When enabled, log output will include ANSI color escape sequences to highlight different log levels, timestamps, and other components of the log messages.
 

go/apps/api/config.go

@@ -68,7 +68,8 @@ type Config struct {
 	// --- OpenTelemetry configuration ---
 
 	// OtelOtlpEndpoint specifies the OpenTelemetry collector endpoint for metrics, traces, and logs
-	OtelEnabled bool
+	OtelEnabled           bool
+	OtelTraceSamplingRate float64
 
 	Clock clock.Clock
 }

go/apps/api/run.go

@@ -44,10 +44,11 @@ func Run(ctx context.Context, cfg Config) error {
 
 	if cfg.OtelEnabled {
 		grafanaErr := otel.InitGrafana(ctx, otel.Config{
-			Application: "api",
-			Version:     version.Version,
-			InstanceID:  cfg.ClusterInstanceID,
-			CloudRegion: cfg.Region,
+			Application:     "api",
+			Version:         version.Version,
+			InstanceID:      cfg.ClusterInstanceID,
+			CloudRegion:     cfg.Region,
+			TraceSampleRate: cfg.OtelTraceSamplingRate,
 		},
 			shutdowns,
 		)

go/cmd/api/main.go

@@ -317,6 +317,29 @@ Examples:
 			Sources:  cli.EnvVars("UNKEY_OTEL"),
 			Required: false,
 		},
+		&cli.FloatFlag{
+			Name: "otel-trace-sampling-rate",
+			Usage: `Sets the sampling rate for OpenTelemetry traces as a value between 0.0 and 1.0.
+This controls what percentage of traces will be collected and exported, helping to balance
+observability needs with performance and cost considerations.
+
+- 0.0 means no traces are sampled (0%)
+- 0.25 means 25% of traces are sampled (default)
+- 1.0 means all traces are sampled (100%)
+
+Lower sampling rates reduce overhead and storage costs but provide less visibility.
+Higher rates give more comprehensive data but increase resource usage and costs.
+
+This setting only takes effect when OpenTelemetry is enabled with --otel=true.
+
+Examples:
+  --otel-trace-sampling-rate=0.1   # Sample 10% of traces
+  --otel-trace-sampling-rate=0.25  # Sample 25% of traces (default)
+  --otel-trace-sampling-rate=1.0   # Sample all traces`,
+			Sources:  cli.EnvVars("UNKEY_OTEL_TRACE_SAMPLING_RATE"),
+			Value:    0.25,
+			Required: false,
+		},
 	},
 
 	Action: action,
@@ -342,7 +365,8 @@ func action(ctx context.Context, cmd *cli.Command) error {
 		ClickhouseURL: cmd.String("clickhouse-url"),
 
 		// OpenTelemetry configuration
-		OtelEnabled: cmd.Bool("otel"),
+		OtelEnabled:           cmd.Bool("otel"),
+		OtelTraceSamplingRate: cmd.Float("otel-trace-sampling-rate"),
 
 		// Cluster
 		ClusterEnabled:                     cmd.Bool("cluster"),

go/pkg/otel/grafana.go

@@ -43,6 +43,15 @@ type Config struct {
 	// Version is the current version of your application, allowing you to correlate
 	// behavior changes with specific releases.
 	Version string
+
+	// TraceSampleRate controls what percentage of traces are sampled.
+	// Values range from 0.0 to 1.0, where:
+	// - 1.0 means all traces are sampled (100%)
+	// - 0.25 means 25% of traces are sampled (the default if not specified)
+	// - 0.0 means no traces are sampled (0%)
+	//
+	// As long as the sampling rate is greater than 0.0, all errors will be sampled.
+	TraceSampleRate float64
 }
 
 // InitGrafana initializes the global tracer and metric providers for OpenTelemetry,
@@ -81,6 +90,7 @@ func InitGrafana(ctx context.Context, config Config, shutdowns *shutdown.Shutdow
 	// Create a resource with common attributes
 	res, err := resource.New(ctx,
 		resource.WithAttributes(
+			semconv.ServiceNamespace(config.Application),
 			semconv.ServiceName(config.Application),
 			semconv.ServiceVersion(config.Version),
 			semconv.ServiceInstanceID(config.InstanceID),
@@ -125,6 +135,7 @@ func InitGrafana(ctx context.Context, config Config, shutdowns *shutdown.Shutdow
 	// Initialize trace exporter with configuration matching the old implementation
 	traceExporter, err := otlptracehttp.New(ctx,
 		otlptracehttp.WithCompression(otlptracehttp.GzipCompression),
+
 	//	otlptracehttp.WithInsecure(), // For local development
 	)
 	if err != nil {
@@ -134,10 +145,28 @@ func InitGrafana(ctx context.Context, config Config, shutdowns *shutdown.Shutdow
 	// Register shutdown function for trace exporter
 	shutdowns.RegisterCtx(traceExporter.Shutdown)
 
+	var sampler trace.Sampler
+
+	// Configure the sampler
+	if config.TraceSampleRate >= 1.0 {
+		sampler = trace.AlwaysSample()
+	} else if config.TraceSampleRate <= 0.0 {
+		sampler = trace.NeverSample()
+	} else {
+		sampler = trace.ParentBased(
+			trace.TraceIDRatioBased(config.TraceSampleRate),
+			trace.WithRemoteParentSampled(trace.AlwaysSample()),
+			trace.WithRemoteParentNotSampled(trace.TraceIDRatioBased(config.TraceSampleRate)),
+			trace.WithLocalParentSampled(trace.AlwaysSample()),
+			trace.WithLocalParentNotSampled(trace.TraceIDRatioBased(config.TraceSampleRate)),
+		)
+	}
+
 	// Create and register trace provider with the same batch settings as the old code
 	traceProvider := trace.NewTracerProvider(
 		trace.WithBatcher(traceExporter),
 		trace.WithResource(res),
+		trace.WithSampler(sampler),
 	)
 
 	// Register shutdown function for trace provider