diff --git a/apps/engineering/content/architecture/services/api/config.mdx b/apps/engineering/content/architecture/services/api/config.mdx
index 74e76c4a01..0fb3a10fea 100644
--- a/apps/engineering/content/architecture/services/api/config.mdx
+++ b/apps/engineering/content/architecture/services/api/config.mdx
@@ -165,6 +165,25 @@ These options configure analytics storage and observability for the Unkey API.
```
+
+ Sets the sampling rate for OpenTelemetry traces as a decimal value between 0.0 and 1.0. This controls what percentage of traces will be collected and exported, helping to balance observability needs with performance and cost considerations.
+
+ - 0.0 means no traces are sampled (0%)
+ - 0.25 means 25% of traces are sampled (default)
+ - 1.0 means all traces are sampled (100%)
+
+ Lower sampling rates reduce overhead and storage costs but provide less visibility. Higher rates give more comprehensive data but increase resource usage and costs.
+
+ This setting only takes effect when OpenTelemetry is enabled with `--otel=true`.
+
+ **Examples:**
+ - `--otel-trace-sampling-rate=0.1` - Sample 10% of traces
+ - `--otel-trace-sampling-rate=0.25` - Sample 25% of traces (default)
+ - `--otel-trace-sampling-rate=1.0` - Sample all traces
+
+ **Environment variable:** `UNKEY_OTEL_TRACE_SAMPLING_RATE`
+
+
Enable ANSI color codes in log output. When enabled, log output will include ANSI color escape sequences to highlight different log levels, timestamps, and other components of the log messages.
diff --git a/go/apps/api/config.go b/go/apps/api/config.go
index 4987f62f43..e23e15cdef 100644
--- a/go/apps/api/config.go
+++ b/go/apps/api/config.go
@@ -68,7 +68,8 @@ type Config struct {
// --- OpenTelemetry configuration ---
// OtelOtlpEndpoint specifies the OpenTelemetry collector endpoint for metrics, traces, and logs
- OtelEnabled bool
+ OtelEnabled bool
+ OtelTraceSamplingRate float64
Clock clock.Clock
}
diff --git a/go/apps/api/run.go b/go/apps/api/run.go
index 9aabf8721d..e6155f4826 100644
--- a/go/apps/api/run.go
+++ b/go/apps/api/run.go
@@ -44,10 +44,11 @@ func Run(ctx context.Context, cfg Config) error {
if cfg.OtelEnabled {
grafanaErr := otel.InitGrafana(ctx, otel.Config{
- Application: "api",
- Version: version.Version,
- InstanceID: cfg.ClusterInstanceID,
- CloudRegion: cfg.Region,
+ Application: "api",
+ Version: version.Version,
+ InstanceID: cfg.ClusterInstanceID,
+ CloudRegion: cfg.Region,
+ TraceSampleRate: cfg.OtelTraceSamplingRate,
},
shutdowns,
)
diff --git a/go/cmd/api/main.go b/go/cmd/api/main.go
index 4a4cf8ca1d..46841c6e07 100644
--- a/go/cmd/api/main.go
+++ b/go/cmd/api/main.go
@@ -317,6 +317,29 @@ Examples:
Sources: cli.EnvVars("UNKEY_OTEL"),
Required: false,
},
+ &cli.FloatFlag{
+ Name: "otel-trace-sampling-rate",
+ Usage: `Sets the sampling rate for OpenTelemetry traces as a value between 0.0 and 1.0.
+This controls what percentage of traces will be collected and exported, helping to balance
+observability needs with performance and cost considerations.
+
+- 0.0 means no traces are sampled (0%)
+- 0.25 means 25% of traces are sampled (default)
+- 1.0 means all traces are sampled (100%)
+
+Lower sampling rates reduce overhead and storage costs but provide less visibility.
+Higher rates give more comprehensive data but increase resource usage and costs.
+
+This setting only takes effect when OpenTelemetry is enabled with --otel=true.
+
+Examples:
+ --otel-trace-sampling-rate=0.1 # Sample 10% of traces
+ --otel-trace-sampling-rate=0.25 # Sample 25% of traces (default)
+ --otel-trace-sampling-rate=1.0 # Sample all traces`,
+ Sources: cli.EnvVars("UNKEY_OTEL_TRACE_SAMPLING_RATE"),
+ Value: 0.25,
+ Required: false,
+ },
},
Action: action,
@@ -342,7 +365,8 @@ func action(ctx context.Context, cmd *cli.Command) error {
ClickhouseURL: cmd.String("clickhouse-url"),
// OpenTelemetry configuration
- OtelEnabled: cmd.Bool("otel"),
+ OtelEnabled: cmd.Bool("otel"),
+ OtelTraceSamplingRate: cmd.Float("otel-trace-sampling-rate"),
// Cluster
ClusterEnabled: cmd.Bool("cluster"),
diff --git a/go/pkg/otel/grafana.go b/go/pkg/otel/grafana.go
index e5fab657d6..f162d339fd 100644
--- a/go/pkg/otel/grafana.go
+++ b/go/pkg/otel/grafana.go
@@ -43,6 +43,15 @@ type Config struct {
// Version is the current version of your application, allowing you to correlate
// behavior changes with specific releases.
Version string
+
+ // TraceSampleRate controls what percentage of traces are sampled.
+ // Values range from 0.0 to 1.0, where:
+ // - 1.0 means all traces are sampled (100%)
+ // - 0.25 means 25% of traces are sampled (the default if not specified)
+ // - 0.0 means no traces are sampled (0%)
+ //
+ // As long as the sampling rate is greater than 0.0, all errors will be sampled.
+ TraceSampleRate float64
}
// InitGrafana initializes the global tracer and metric providers for OpenTelemetry,
@@ -81,6 +90,7 @@ func InitGrafana(ctx context.Context, config Config, shutdowns *shutdown.Shutdow
// Create a resource with common attributes
res, err := resource.New(ctx,
resource.WithAttributes(
+ semconv.ServiceNamespace(config.Application),
semconv.ServiceName(config.Application),
semconv.ServiceVersion(config.Version),
semconv.ServiceInstanceID(config.InstanceID),
@@ -125,6 +135,7 @@ func InitGrafana(ctx context.Context, config Config, shutdowns *shutdown.Shutdow
// Initialize trace exporter with configuration matching the old implementation
traceExporter, err := otlptracehttp.New(ctx,
otlptracehttp.WithCompression(otlptracehttp.GzipCompression),
+
// otlptracehttp.WithInsecure(), // For local development
)
if err != nil {
@@ -134,10 +145,28 @@ func InitGrafana(ctx context.Context, config Config, shutdowns *shutdown.Shutdow
// Register shutdown function for trace exporter
shutdowns.RegisterCtx(traceExporter.Shutdown)
+ var sampler trace.Sampler
+
+ // Configure the sampler
+ if config.TraceSampleRate >= 1.0 {
+ sampler = trace.AlwaysSample()
+ } else if config.TraceSampleRate <= 0.0 {
+ sampler = trace.NeverSample()
+ } else {
+ sampler = trace.ParentBased(
+ trace.TraceIDRatioBased(config.TraceSampleRate),
+ trace.WithRemoteParentSampled(trace.AlwaysSample()),
+ trace.WithRemoteParentNotSampled(trace.TraceIDRatioBased(config.TraceSampleRate)),
+ trace.WithLocalParentSampled(trace.AlwaysSample()),
+ trace.WithLocalParentNotSampled(trace.TraceIDRatioBased(config.TraceSampleRate)),
+ )
+ }
+
// Create and register trace provider with the same batch settings as the old code
traceProvider := trace.NewTracerProvider(
trace.WithBatcher(traceExporter),
trace.WithResource(res),
+ trace.WithSampler(sampler),
)
// Register shutdown function for trace provider