diff --git a/apps/agent/pkg/gossip/membership_test.go b/apps/agent/pkg/gossip/membership_test.go deleted file mode 100644 index 818e3b1036..0000000000 --- a/apps/agent/pkg/gossip/membership_test.go +++ /dev/null @@ -1,256 +0,0 @@ -package gossip - -import ( - "context" - "fmt" - "sync" - - // "sync" - "testing" - "time" - - "github.com/stretchr/testify/require" - "github.com/unkeyed/unkey/apps/agent/pkg/logging" - "github.com/unkeyed/unkey/apps/agent/pkg/port" -) - -var CLUSTER_SIZES = []int{3, 9, 36} - -func TestJoin2Nodes(t *testing.T) { - - freePort := port.New() - - m1, err := New(Config{ - NodeId: "node_1", - RpcAddr: fmt.Sprintf("http://localhost:%d", freePort.Get()), - Logger: logging.New(&logging.Config{Debug: true}), - }) - require.NoError(t, err) - - require.NoError(t, err) - - go NewClusterServer(m1, logging.New(&logging.Config{Debug: true})).Serve() - - m2, err := New(Config{ - NodeId: "node_2", - RpcAddr: fmt.Sprintf("http://localhost:%d", freePort.Get()), - Logger: logging.New(&logging.Config{Debug: true}), - }) - require.NoError(t, err) - - go NewClusterServer(m2, logging.New(&logging.Config{Debug: true})).Serve() - - t.Logf("m1 addr: %s", m1.RpcAddr()) - - err = m2.Join(context.Background(), m1.RpcAddr()) - require.NoError(t, err) - require.Eventually(t, func() bool { - members := m2.Members() - require.NoError(t, err) - return len(members) == 2 - - }, 10*time.Second, 100*time.Millisecond) - -} - -func TestMembers_returns_all_members(t *testing.T) { - - for _, clusterSize := range CLUSTER_SIZES { - - t.Run(fmt.Sprintf("cluster size %d", clusterSize), func(t *testing.T) { - - nodes := runMany(t, clusterSize) - for _, m := range nodes { - require.Eventually(t, func() bool { - return len(m.Members()) == clusterSize - }, time.Minute, 100*time.Millisecond) - - } - }) - } - -} - -// Test whether nodes correctly emit a join event when they join the cluster -// When a node joins, a listener to the `JoinEvents` topic should receive a message with the member that jioned. -func TestJoin_emits_join_event(t *testing.T) { - for _, clusterSize := range CLUSTER_SIZES { - t.Run(fmt.Sprintf("cluster size %d", clusterSize), func(t *testing.T) { - freePort := port.New() - members := make([]*cluster, clusterSize) - var err error - for i := 0; i < clusterSize; i++ { - members[i], err = New(Config{ - NodeId: fmt.Sprintf("node_%d", i), - RpcAddr: fmt.Sprintf("http://localhost:%d", freePort.Get()), - Logger: logging.New(&logging.Config{Debug: true}), - }) - require.NoError(t, err) - go NewClusterServer(members[i], logging.New(&logging.Config{Debug: true})).Serve() - } - - joinEvents := members[0].SubscribeJoinEvents("test") - joinMu := sync.RWMutex{} - join := make(map[string]bool) - - go func() { - for event := range joinEvents { - joinMu.Lock() - join[event.NodeId] = true - joinMu.Unlock() - } - }() - - rpcAddrs := make([]string, 0) - for _, m := range members { - err := m.Join(context.Background(), rpcAddrs...) - require.NoError(t, err) - rpcAddrs = append(rpcAddrs, m.RpcAddr()) - } - - for _, n := range members[1:] { - require.Eventually(t, func() bool { - joinMu.RLock() - t.Logf("joins: %+v", join) - ok := join[n.self.NodeId] - joinMu.RUnlock() - return ok - }, 30*time.Second, 100*time.Millisecond) - } - }) - } -} - -// Test whether nodes correctly emit a leave event when they leave the cluster -// When a node leaves, a listener to the `LeaveEvents` topic should receive a message with the member that left. -func TestLeave_emits_leave_event(t *testing.T) { - for _, clusterSize := range CLUSTER_SIZES { - t.Run(fmt.Sprintf("cluster size %d", clusterSize), func(t *testing.T) { - - nodes := runMany(t, clusterSize) - - leaveEvents := nodes[0].SubscribeLeaveEvents("test") - leftMu := sync.RWMutex{} - left := make(map[string]bool) - - go func() { - for event := range leaveEvents { - leftMu.Lock() - left[event.NodeId] = true - leftMu.Unlock() - } - }() - - for _, n := range nodes[1:] { - err := n.Shutdown(context.Background()) - require.NoError(t, err) - } - - for _, n := range nodes[1:] { - require.Eventually(t, func() bool { - leftMu.RLock() - t.Log(left) - l := left[n.self.NodeId] - leftMu.RUnlock() - return l - }, 30*time.Second, 100*time.Millisecond) - } - }) - } -} - -func TestUncleanShutdown(t *testing.T) { - t.Skip("WIP") - for _, clusterSize := range CLUSTER_SIZES { - - t.Run(fmt.Sprintf("cluster size %d", clusterSize), func(t *testing.T) { - - freePort := port.New() - - gossipFactor := 3 - if clusterSize < gossipFactor { - gossipFactor = clusterSize - 1 - } - - members := make([]*cluster, clusterSize) - srvs := make([]*clusterServer, clusterSize) - for i := 0; i < clusterSize; i++ { - c, err := New(Config{ - NodeId: fmt.Sprintf("node_%d", i), - RpcAddr: fmt.Sprintf("http://localhost:%d", freePort.Get()), - Logger: logging.New(&logging.Config{Debug: true}), - GossipFactor: gossipFactor, - }) - require.NoError(t, err) - - srv := NewClusterServer(c, logging.New(&logging.Config{Debug: true})) - go srv.Serve() - members[i] = c - srvs[i] = srv - } - - rpcAddrs := make([]string, 0) - for _, m := range members { - err := m.Join(context.Background(), rpcAddrs...) - require.NoError(t, err) - rpcAddrs = append(rpcAddrs, m.self.RpcAddr) - } - - for _, m := range members { - require.Eventually(t, func() bool { - return len(m.Members()) == clusterSize - }, 5*time.Second, 100*time.Millisecond) - } - - srvs[0]._testSimulateFailure() - - for _, m := range members[1:] { - require.Eventually(t, func() bool { - t.Logf("members: %+v", m.Members()) - return len(m.Members()) == clusterSize-1 - }, 10*time.Second, 100*time.Millisecond) - } - - }) - } -} - -func runMany(t *testing.T, n int) []*cluster { - freePort := port.New() - - gossipFactor := 3 - if n < gossipFactor { - gossipFactor = n - 1 - } - - members := make([]*cluster, n) - for i := 0; i < n; i++ { - c, err := New(Config{ - NodeId: fmt.Sprintf("node_%d", i), - RpcAddr: fmt.Sprintf("http://localhost:%d", freePort.Get()), - Logger: logging.New(&logging.Config{Debug: true}), - GossipFactor: gossipFactor, - }) - require.NoError(t, err) - - members[i] = c - - srv := NewClusterServer(c, logging.New(&logging.Config{Debug: true})) - go srv.Serve() - } - - rpcAddrs := make([]string, 0) - for _, m := range members { - err := m.Join(context.Background(), rpcAddrs...) - require.NoError(t, err) - rpcAddrs = append(rpcAddrs, m.self.RpcAddr) - } - - for _, m := range members { - require.Eventually(t, func() bool { - return len(m.Members()) == n - }, 5*time.Second, 100*time.Millisecond) - } - return members - -} diff --git a/apps/engineering/content/architecture/services/api/config.mdx b/apps/engineering/content/architecture/services/api/config.mdx index 5e55045f46..4b8a8c82d5 100644 --- a/apps/engineering/content/architecture/services/api/config.mdx +++ b/apps/engineering/content/architecture/services/api/config.mdx @@ -86,8 +86,6 @@ These options control the fundamental behavior of the API server. **Examples:** - `--http-port=7070` - Default port - - `--http-port=8080` - Common alternative for local development - - `--http-port=80` - Standard HTTP port (requires root privileges on Unix systems) @@ -110,9 +108,8 @@ The Unkey API requires a MySQL database for storing keys and configuration. For For production use, ensure the database has proper backup procedures in place. Unkey is using [PlanetScale](https://planetscale.com/) **Examples:** - - `--database-primary=mysql://root:password@localhost:3306/unkey` - - `--database-primary=mysql://user:password@mysql.example.com:3306/unkey?tls=true` - - `--database-primary=mysql://unkey:password@mysql.default.svc.cluster.local:3306/unkey` + - `--database-primary=mysql://root:password@localhost:3306/unkey?parseTime=true` - Local MySQL for development + - `--database-primary=mysql://username:pscale_pw_...@aws.connect.psdb.cloud/unkey?sslmode=require` - PlanetScale connection @@ -123,8 +120,8 @@ The Unkey API requires a MySQL database for storing keys and configuration. For Unkey is using [PlanetScales](https://planetscale.com/) global read replica endpoint. **Examples:** - - `--database-readonly-replica=mysql://readonly:password@replica.mysql.example.com:3306/unkey?tls=true` - - `--database-readonly-replica=mysql://readonly:password@mysql-replica.default.svc.cluster.local:3306/unkey` + - `--database-readonly-replica=mysql://root:password@localhost:3306/unkey?parseTime=true` - Local MySQL for development + - `--database-readonly-replica=mysql://username:pscale_pw_...@aws.connect.psdb.cloud/unkey?sslmode=require` - PlanetScale connection ## Analytics & Monitoring @@ -143,15 +140,29 @@ These options configure analytics storage and observability for the Unkey API. - `--clickhouse-url=clickhouse://default:password@clickhouse.default.svc.cluster.local:9000/unkey?secure=true` - - OpenTelemetry collector endpoint for metrics, traces, and logs. When provided, the Unkey API will send telemetry data (metrics, traces, and logs) to this endpoint using the OTLP protocol. + + Enable OpenTelemetry. The Unkey API will collect and export telemetry data (metrics, traces, and logs) using the OpenTelemetry protocol. - This should be specified as a host and optional port without scheme or path. The implementation is configured for HTTP protocol by default. + When this flag is set to true, the following standard OpenTelemetry environment variables are used to configure the exporter: + + - `OTEL_EXPORTER_OTLP_ENDPOINT`: The URL of your OpenTelemetry collector + - `OTEL_EXPORTER_OTLP_PROTOCOL`: The protocol to use (http/protobuf or grpc) + - `OTEL_EXPORTER_OTLP_HEADERS`: Headers for authentication (e.g., "authorization=Bearer \") + + Using these standard variables ensures compatibility with OpenTelemetry documentation and tools. For detailed configuration information, see the [official OpenTelemetry documentation](https://grafana.com/docs/grafana-cloud/send-data/otlp/send-data-otlp/). **Examples:** - - `--otel-otlp-endpoint=localhost:4317` - Local collector - - `--otel-otlp-endpoint=otlp-gateway-prod-us-east-0.grafana.net:443` - Grafana Cloud - - `--otel-otlp-endpoint=api.honeycomb.io:443` - Honeycomb.io + + ```bash + # Enable OpenTelemetry + export UNKEY_OTEL=true + export OTEL_EXPORTER_OTLP_ENDPOINT="https://otlp-gateway-prod-us-east-0.grafana.net/otlp" + export OTEL_EXPORTER_OTLP_PROTOCOL="http/protobuf" + export OTEL_EXPORTER_OTLP_HEADERS="authorization=Basic ..." + + # Or as command-line flags + unkey api --otel=true" + ``` diff --git a/deployment/docker-compose.yaml b/deployment/docker-compose.yaml index d1660433b4..ab0f5cb0c7 100644 --- a/deployment/docker-compose.yaml +++ b/deployment/docker-compose.yaml @@ -52,7 +52,6 @@ services: - mysql - redis - clickhouse - - tempo environment: UNKEY_HTTP_PORT: 7070 UNKEY_CLUSTER: true @@ -62,7 +61,6 @@ services: UNKEY_CLUSTER_DISCOVERY_REDIS_URL: "redis://redis:6379" UNKEY_DATABASE_PRIMARY_DSN: "mysql://unkey:password@tcp(mysql:3900)/unkey?parseTime=true" UNKEY_CLICKHOUSE_URL: "clickhouse://default:password@clickhouse:9000" - UNKEY_OTEL_OTLP_ENDPOINT: "otel-collector:4318" # Point directly to Tempo redis: image: redis:latest @@ -170,73 +168,9 @@ services: - agent - clickhouse - chproxy - otel-collector: - image: otel/opentelemetry-collector-contrib:latest - container_name: otel-collector - command: ["--config=/etc/otel-collector-config.yaml"] - volumes: - - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml - ports: - - "4318:4318" # OTLP HTTP - depends_on: - - prometheus - - tempo - - loki - - grafana - prometheus: - image: prom/prometheus:latest - container_name: prometheus - command: - - "--config.file=/etc/prometheus/config.yaml" - - "--storage.tsdb.path=/prometheus" - - "--web.enable-lifecycle" - - "--web.enable-remote-write-receiver" # Add this flag - ports: - - 9090:9090 - volumes: - - ./prometheus/config.yaml:/etc/prometheus/config.yaml - - prometheus:/prometheus - - grafana: - image: grafana/grafana-oss:latest - container_name: grafana - ports: - - 3000:3000 - environment: - - GF_SECURITY_ADMIN_USER=admin - - GF_SECURITY_ADMIN_PASSWORD=grafana - - GF_INSTALL_PLUGINS=grafana-clock-panel,grafana-simple-json-datasource - volumes: - - ./grafana/provisioning:/etc/grafana/provisioning - - grafana:/var/lib/grafana - - # Tempo for distributed tracing - tempo: - image: grafana/tempo:latest - container_name: tempo - command: ["-config.file=/etc/tempo.yaml"] - volumes: - - ./tempo/config.yaml:/etc/tempo.yaml - - tempo:/tmp/tempo - ports: - - "3200:3200" # tempo - - # Loki for logs - loki: - container_name: loki - image: grafana/loki:latest - ports: - - "3100:3100" - command: -config.file=/etc/loki/config.yaml - volumes: - - ./loki/config.yaml:/etc/loki/config.yaml volumes: mysql: - grafana: - tempo: - loki: clickhouse: clickhouse-keeper: s3: - prometheus: diff --git a/deployment/grafana/grafana.yaml b/deployment/grafana/grafana.yaml deleted file mode 100644 index d442154352..0000000000 --- a/deployment/grafana/grafana.yaml +++ /dev/null @@ -1,28 +0,0 @@ -apiVersion: 1 - -auth: - basic: - enabled: false - anonymous: - enabled: true - org_name: Local - org_role: Admin -datasources: -- name: Prometheus - type: prometheus - url: http://prometheus:9090 - isDefault: true - access: proxy - editable: true -- name: ClickHouse - type: grafana-clickhouse-datasource - jsonData: - defaultDatabase: database - port: 9000 - host: clickhouse - username: default - isDefault: true - editable: true - tlsSkipVerify: false - secureJsonData: - password: password diff --git a/deployment/grafana/provisioning/datasources/datasources.yaml b/deployment/grafana/provisioning/datasources/datasources.yaml deleted file mode 100644 index 49e26498e7..0000000000 --- a/deployment/grafana/provisioning/datasources/datasources.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: 1 - -datasources: - - name: Prometheus - type: prometheus - access: proxy - url: http://prometheus:9090 - isDefault: true - - - name: Tempo - type: tempo - access: proxy - url: http://tempo:3200 - uid: tempo - - - name: Loki - type: loki - access: proxy - url: http://loki:3100 - uid: loki diff --git a/deployment/loki/config.yaml b/deployment/loki/config.yaml deleted file mode 100644 index 3d55b8c4c1..0000000000 --- a/deployment/loki/config.yaml +++ /dev/null @@ -1,62 +0,0 @@ -auth_enabled: false - -server: - http_listen_port: 3100 - grpc_listen_port: 9096 - log_level: debug - grpc_server_max_concurrent_streams: 1000 - -common: - instance_addr: 127.0.0.1 - path_prefix: /tmp/loki - storage: - filesystem: - chunks_directory: /tmp/loki/chunks - rules_directory: /tmp/loki/rules - replication_factor: 1 - ring: - kvstore: - store: inmemory - -query_range: - results_cache: - cache: - embedded_cache: - enabled: true - max_size_mb: 100 - -limits_config: - metric_aggregation_enabled: true - -schema_config: - configs: - - from: 2020-10-24 - store: tsdb - object_store: filesystem - schema: v13 - index: - prefix: index_ - period: 24h - -pattern_ingester: - enabled: true - metric_aggregation: - loki_address: localhost:3100 - -ruler: - alertmanager_url: http://localhost:9093 - -frontend: - encoding: protobuf -# By default, Loki will send anonymous, but uniquely-identifiable usage and configuration -# analytics to Grafana Labs. These statistics are sent to https://stats.grafana.org/ -# -# Statistics help us better understand how Loki is used, and they show us performance -# levels for most users. This helps us prioritize features and documentation. -# For more information on what's sent, look at -# https://github.com/grafana/loki/blob/main/pkg/analytics/stats.go -# Refer to the buildReport method to see what goes into a report. -# -# If you would like to disable reporting, uncomment the following lines: -#analytics: -# reporting_enabled: false diff --git a/deployment/otel-collector-config.yaml b/deployment/otel-collector-config.yaml deleted file mode 100644 index ceba3adffc..0000000000 --- a/deployment/otel-collector-config.yaml +++ /dev/null @@ -1,44 +0,0 @@ -receivers: - otlp: - protocols: - http: - endpoint: "0.0.0.0:4318" - grpc: - endpoint: "0.0.0.0:4317" - -processors: - batch: - send_batch_size: 10000 - timeout: 5s - -exporters: - # For traces - send to Tempo - otlp/tempo: - endpoint: "tempo:4317" - tls: - insecure: true - - # For metrics - send to Prometheus - prometheusremotewrite: - endpoint: "http://prometheus:9090/api/v1/write" - tls: - insecure: true - - # Debug output for troubleshooting - debug: - verbosity: detailed - -service: - pipelines: - traces: - receivers: [otlp] - processors: [batch] - exporters: [otlp/tempo, debug] - metrics: - receivers: [otlp] - processors: [batch] - exporters: [prometheusremotewrite, debug] - logs: - receivers: [otlp] - processors: [batch] - exporters: [debug] diff --git a/deployment/prometheus/config.yaml b/deployment/prometheus/config.yaml deleted file mode 100644 index c852f72a24..0000000000 --- a/deployment/prometheus/config.yaml +++ /dev/null @@ -1,20 +0,0 @@ -global: - scrape_interval: 15s - evaluation_interval: 15s - -scrape_configs: - - job_name: "prometheus" - static_configs: - - targets: ["localhost:9090"] - - - job_name: "tempo" - static_configs: - - targets: ["tempo:3200"] - - - job_name: "loki" - static_configs: - - targets: ["loki:3100"] - - - job_name: "otel-collector" - static_configs: - - targets: ["otel-collector:8889"] # If your collector exposes metrics diff --git a/deployment/tempo/config.yaml b/deployment/tempo/config.yaml deleted file mode 100644 index d5c30614f5..0000000000 --- a/deployment/tempo/config.yaml +++ /dev/null @@ -1,24 +0,0 @@ -server: - http_listen_port: 3200 - -distributor: - receivers: - otlp: - protocols: - http: - endpoint: "0.0.0.0:4318" - grpc: - endpoint: "0.0.0.0:4317" - -storage: - trace: - backend: local - local: - path: /tmp/tempo - -ingester: - max_block_duration: 5m - -compactor: - compaction: - block_retention: 24h diff --git a/go/apps/api/cancel_test.go b/go/apps/api/cancel_test.go index 274257818b..ecb52e2ebe 100644 --- a/go/apps/api/cancel_test.go +++ b/go/apps/api/cancel_test.go @@ -41,7 +41,7 @@ func TestContextCancellation(t *testing.T) { ClickhouseURL: "", DatabasePrimary: dbDsn, DatabaseReadonlyReplica: "", - OtelOtlpEndpoint: "", + OtelEnabled: false, } // Create a channel to receive the result of the Run function diff --git a/go/apps/api/config.go b/go/apps/api/config.go index 1850c64f75..8d1a8260e6 100644 --- a/go/apps/api/config.go +++ b/go/apps/api/config.go @@ -68,7 +68,7 @@ type Config struct { // --- OpenTelemetry configuration --- // OtelOtlpEndpoint specifies the OpenTelemetry collector endpoint for metrics, traces, and logs - OtelOtlpEndpoint string + OtelEnabled bool Clock clock.Clock } diff --git a/go/apps/api/routes/services.go b/go/apps/api/routes/services.go index 19df927896..069f392e8b 100644 --- a/go/apps/api/routes/services.go +++ b/go/apps/api/routes/services.go @@ -6,7 +6,7 @@ import ( "github.com/unkeyed/unkey/go/internal/services/ratelimit" "github.com/unkeyed/unkey/go/pkg/clickhouse/schema" "github.com/unkeyed/unkey/go/pkg/db" - "github.com/unkeyed/unkey/go/pkg/logging" + "github.com/unkeyed/unkey/go/pkg/otel/logging" "github.com/unkeyed/unkey/go/pkg/zen/validation" ) diff --git a/go/apps/api/routes/v2_ratelimit_delete_override/handler.go b/go/apps/api/routes/v2_ratelimit_delete_override/handler.go index 80971b5b89..bc99347330 100644 --- a/go/apps/api/routes/v2_ratelimit_delete_override/handler.go +++ b/go/apps/api/routes/v2_ratelimit_delete_override/handler.go @@ -14,7 +14,7 @@ import ( "github.com/unkeyed/unkey/go/pkg/auditlog" "github.com/unkeyed/unkey/go/pkg/db" "github.com/unkeyed/unkey/go/pkg/fault" - "github.com/unkeyed/unkey/go/pkg/logging" + "github.com/unkeyed/unkey/go/pkg/otel/logging" "github.com/unkeyed/unkey/go/pkg/rbac" "github.com/unkeyed/unkey/go/pkg/uid" "github.com/unkeyed/unkey/go/pkg/zen" diff --git a/go/apps/api/routes/v2_ratelimit_get_override/handler.go b/go/apps/api/routes/v2_ratelimit_get_override/handler.go index 0b2db23e25..c7f25f943b 100644 --- a/go/apps/api/routes/v2_ratelimit_get_override/handler.go +++ b/go/apps/api/routes/v2_ratelimit_get_override/handler.go @@ -12,7 +12,7 @@ import ( "github.com/unkeyed/unkey/go/internal/services/permissions" "github.com/unkeyed/unkey/go/pkg/db" "github.com/unkeyed/unkey/go/pkg/fault" - "github.com/unkeyed/unkey/go/pkg/logging" + "github.com/unkeyed/unkey/go/pkg/otel/logging" "github.com/unkeyed/unkey/go/pkg/rbac" "github.com/unkeyed/unkey/go/pkg/zen" ) diff --git a/go/apps/api/routes/v2_ratelimit_limit/handler.go b/go/apps/api/routes/v2_ratelimit_limit/handler.go index 66f40eec29..076d989cff 100644 --- a/go/apps/api/routes/v2_ratelimit_limit/handler.go +++ b/go/apps/api/routes/v2_ratelimit_limit/handler.go @@ -11,7 +11,7 @@ import ( "github.com/unkeyed/unkey/go/internal/services/ratelimit" "github.com/unkeyed/unkey/go/pkg/db" "github.com/unkeyed/unkey/go/pkg/fault" - "github.com/unkeyed/unkey/go/pkg/logging" + "github.com/unkeyed/unkey/go/pkg/otel/logging" "github.com/unkeyed/unkey/go/pkg/rbac" "github.com/unkeyed/unkey/go/pkg/zen" ) diff --git a/go/apps/api/routes/v2_ratelimit_limit/simulation_test.gox b/go/apps/api/routes/v2_ratelimit_limit/simulation_test.gox index 2888a2a1ef..b805ce105e 100644 --- a/go/apps/api/routes/v2_ratelimit_limit/simulation_test.gox +++ b/go/apps/api/routes/v2_ratelimit_limit/simulation_test.gox @@ -13,7 +13,7 @@ import ( "github.com/unkeyed/unkey/go/pkg/clock" "github.com/unkeyed/unkey/go/pkg/cluster" "github.com/unkeyed/unkey/go/pkg/db" - "github.com/unkeyed/unkey/go/pkg/logging" + "github.com/unkeyed/unkey/go/pkg/otel/logging" "github.com/unkeyed/unkey/go/pkg/sim" "github.com/unkeyed/unkey/go/pkg/testutil" "github.com/unkeyed/unkey/go/pkg/uid" diff --git a/go/apps/api/routes/v2_ratelimit_set_override/handler.go b/go/apps/api/routes/v2_ratelimit_set_override/handler.go index f4d7ebca1c..42336e227d 100644 --- a/go/apps/api/routes/v2_ratelimit_set_override/handler.go +++ b/go/apps/api/routes/v2_ratelimit_set_override/handler.go @@ -12,7 +12,7 @@ import ( "github.com/unkeyed/unkey/go/internal/services/permissions" "github.com/unkeyed/unkey/go/pkg/db" "github.com/unkeyed/unkey/go/pkg/fault" - "github.com/unkeyed/unkey/go/pkg/logging" + "github.com/unkeyed/unkey/go/pkg/otel/logging" "github.com/unkeyed/unkey/go/pkg/rbac" "github.com/unkeyed/unkey/go/pkg/uid" "github.com/unkeyed/unkey/go/pkg/zen" diff --git a/go/apps/api/run.go b/go/apps/api/run.go index 7adb5c74df..ade875d4c2 100644 --- a/go/apps/api/run.go +++ b/go/apps/api/run.go @@ -22,9 +22,9 @@ import ( "github.com/unkeyed/unkey/go/pkg/cluster" "github.com/unkeyed/unkey/go/pkg/db" "github.com/unkeyed/unkey/go/pkg/discovery" - "github.com/unkeyed/unkey/go/pkg/logging" "github.com/unkeyed/unkey/go/pkg/membership" "github.com/unkeyed/unkey/go/pkg/otel" + "github.com/unkeyed/unkey/go/pkg/otel/logging" "github.com/unkeyed/unkey/go/pkg/shutdown" "github.com/unkeyed/unkey/go/pkg/version" "github.com/unkeyed/unkey/go/pkg/zen" @@ -43,7 +43,21 @@ func Run(ctx context.Context, cfg Config) error { clk := clock.New() - logger := logging.New(logging.Config{Development: true, NoColor: true}). + if cfg.OtelEnabled { + grafanaErr := otel.InitGrafana(ctx, otel.Config{ + Application: "api", + Version: version.Version, + NodeID: cfg.ClusterNodeID, + CloudRegion: cfg.Region, + }, + shutdowns, + ) + if grafanaErr != nil { + return fmt.Errorf("unable to init grafana: %w", grafanaErr) + } + } + + logger := logging.New(). With( slog.String("nodeId", cfg.ClusterNodeID), slog.String("platform", cfg.Platform), @@ -61,21 +75,6 @@ func Run(ctx context.Context, cfg Config) error { } }() - if cfg.OtelOtlpEndpoint != "" { - grafanaErr := otel.InitGrafana(ctx, otel.Config{ - GrafanaEndpoint: cfg.OtelOtlpEndpoint, - Application: "api", - Version: version.Version, - NodeID: cfg.ClusterNodeID, - CloudRegion: cfg.Region, - }, - shutdowns, - ) - if grafanaErr != nil { - return fmt.Errorf("unable to init grafana: %w", grafanaErr) - } - } - db, err := db.New(db.Config{ PrimaryDSN: cfg.DatabasePrimary, ReadOnlyDSN: cfg.DatabaseReadonlyReplica, diff --git a/go/apps/api/run_test.go b/go/apps/api/run_test.go index a9beca6773..7ad492795e 100644 --- a/go/apps/api/run_test.go +++ b/go/apps/api/run_test.go @@ -52,7 +52,7 @@ func TestClusterFormation(t *testing.T) { ClickhouseURL: "", DatabasePrimary: dbDsn, DatabaseReadonlyReplica: "", - OtelOtlpEndpoint: "", + OtelEnabled: false, } joinAddrs = append(joinAddrs, fmt.Sprintf("localhost:%d", gossipPort)) diff --git a/go/cmd/api/main.go b/go/cmd/api/main.go index 6746c0dada..5adf6cb266 100644 --- a/go/cmd/api/main.go +++ b/go/cmd/api/main.go @@ -47,9 +47,7 @@ In containerized environments, ensure this port is properly exposed. The default port is 7070 if not specified. Examples: - --http-port=7070 # Default port - --http-port=8080 # Common alternative for local development - --http-port=80 # Standard HTTP port (requires root privileges on Unix systems)`, + --http-port=7070 # Default port`, Sources: cli.EnvVars("UNKEY_HTTP_PORT"), Value: 7070, Required: false, @@ -158,8 +156,7 @@ In containerized environments, ensure this port is properly exposed between cont For security, this port should typically not be exposed to external networks. Examples: - --cluster-rpc-port=7071 # Default RPC port - --cluster-rpc-port=9000 # Alternative port if 7071 is unavailable`, + --cluster-rpc-port=7071 # Default RPC port`, Sources: cli.EnvVars("UNKEY_CLUSTER_RPC_PORT"), Value: 7071, Required: false, @@ -177,8 +174,7 @@ In containerized environments, ensure this port is properly exposed between cont For security, this port should typically not be exposed to external networks. Examples: - --cluster-gossip-port=7072 # Default gossip port - --cluster-gossip-port=9001 # Alternative port if 7072 is unavailable`, + --cluster-gossip-port=7072 # Default gossip port`, Sources: cli.EnvVars("UNKEY_CLUSTER_GOSSIP_PORT"), Value: 7072, Required: false, @@ -277,9 +273,8 @@ The connection string must be a valid MySQL connection string with all necessary parameters, including SSL mode for secure connections. Examples: - --database-primary=mysql://root:password@localhost:3306/unkey - --database-primary=mysql://user:password@mysql.example.com:3306/unkey?tls=true - --database-primary=mysql://unkey:password@mysql.default.svc.cluster.local:3306/unkey`, + --database-primary=mysql://root:password@localhost:3306/unkey?parseTime=true + --database-primary=mysql://username:pscale_pw_...@aws.connect.psdb.cloud/unkey?sslmode=require`, Sources: cli.EnvVars("UNKEY_DATABASE_PRIMARY_DSN"), Required: true, }, @@ -296,36 +291,34 @@ In AWS, this could be an RDS read replica. In other environments, it could be a MySQL replica configured with binary log replication. Examples: - --database-readonly-replica=mysql://readonly:password@replica.mysql.example.com:3306/unkey?tls=true - --database-readonly-replica=mysql://readonly:password@mysql-replica.default.svc.cluster.local:3306/unkey`, + --database-readonly-replica=mysql://root:password@localhost:3306/unkey?parseTime=true + --database-readonly-replica=mysql://username:pscale_pw_...@aws.connect.psdb.cloud/unkey?sslmode=require`, Sources: cli.EnvVars("UNKEY_DATABASE_READONLY_DSN"), Required: false, }, // OpenTelemetry configuration - &cli.StringFlag{ - Name: "otel-otlp-endpoint", - Usage: `OpenTelemetry collector endpoint for metrics, traces, and logs. -Specified as host:port (without scheme or path) - -When provided, the Unkey API will send telemetry data (metrics, traces, and logs) -to this endpoint using the OTLP protocol. This enables comprehensive observability -for production deployments. + &cli.BoolFlag{ + Name: "otel", + Usage: `Enable OpenTelemetry tracing and metrics. +When enabled, the Unkey API will collect and export telemetry data (metrics, traces, and logs) +using the OpenTelemetry protocol. This provides comprehensive observability for production deployments. -The endpoint should be an OpenTelemetry collector capable of receiving OTLP data. -The implementation is currently configured for Grafana Cloud integration but is -compatible with any OTLP-compliant collector. +When this flag is set to true, the following standard OpenTelemetry environment variables are used: +- OTEL_EXPORTER_OTLP_ENDPOINT: The URL of your OpenTelemetry collector +- OTEL_EXPORTER_OTLP_PROTOCOL: The protocol to use (http/protobuf or grpc) +- OTEL_EXPORTER_OTLP_HEADERS: Headers for authentication (e.g., "authorization=Bearer ") -Enabling telemetry is highly recommended for production deployments to monitor -performance, detect issues, and troubleshoot problems. +For more information on these variables, see: +https://grafana.com/docs/grafana-cloud/send-data/otlp/send-data-otlp/ Examples: - --otel-otlp-endpoint=http://localhost:4317 # Local collector - --otel-otlp-endpoint=https://otlp.grafana-cloud.example.com # Grafana Cloud - --otel-otlp-endpoint=https://api.honeycomb.io:443 # Honeycomb.io`, - Sources: cli.EnvVars("UNKEY_OTEL_OTLP_ENDPOINT"), + --otel=true # Enable OpenTelemetry with environment variable configuration + --otel=false # Disable OpenTelemetry (default)`, + Sources: cli.EnvVars("UNKEY_OTEL"), Required: false, }, }, + Action: action, } @@ -349,7 +342,7 @@ func action(ctx context.Context, cmd *cli.Command) error { ClickhouseURL: cmd.String("clickhouse-url"), // OpenTelemetry configuration - OtelOtlpEndpoint: cmd.String("otel-otlp-endpoint"), + OtelEnabled: cmd.Bool("otel"), // Cluster ClusterEnabled: cmd.Bool("cluster"), diff --git a/go/cmd/quotacheck/main.go b/go/cmd/quotacheck/main.go index 0d0b16411c..cf41faff8d 100644 --- a/go/cmd/quotacheck/main.go +++ b/go/cmd/quotacheck/main.go @@ -12,7 +12,7 @@ import ( "github.com/unkeyed/unkey/go/pkg/clickhouse" "github.com/unkeyed/unkey/go/pkg/db" - "github.com/unkeyed/unkey/go/pkg/logging" + "github.com/unkeyed/unkey/go/pkg/otel/logging" "golang.org/x/text/language" "golang.org/x/text/message" "golang.org/x/text/number" @@ -56,7 +56,7 @@ func run(ctx context.Context, cmd *cli.Command) error { year, month, _ := time.Now().Date() - logger := logging.New(logging.Config{Development: true, NoColor: false}) + logger := logging.New() slackWebhookURL := cmd.String("slack-webhook-url") diff --git a/go/go.mod b/go/go.mod index a7fd1e2c23..fa84a65c54 100644 --- a/go/go.mod +++ b/go/go.mod @@ -24,21 +24,24 @@ require ( github.com/stretchr/testify v1.10.0 github.com/unkeyed/unkey/apps/agent v0.0.0-20250305080604-6976ad945f11 github.com/urfave/cli/v3 v3.0.0-beta1 + go.opentelemetry.io/contrib/bridges/otelslog v0.10.0 go.opentelemetry.io/contrib/instrumentation/runtime v0.59.0 - go.opentelemetry.io/otel v1.34.0 + go.opentelemetry.io/contrib/processors/minsev v0.8.0 + go.opentelemetry.io/otel v1.35.0 + go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.11.0 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.34.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.34.0 - go.opentelemetry.io/otel/metric v1.34.0 - go.opentelemetry.io/otel/sdk v1.34.0 + go.opentelemetry.io/otel/metric v1.35.0 + go.opentelemetry.io/otel/sdk v1.35.0 + go.opentelemetry.io/otel/sdk/log v0.11.0 go.opentelemetry.io/otel/sdk/metric v1.34.0 - go.opentelemetry.io/otel/trace v1.34.0 + go.opentelemetry.io/otel/trace v1.35.0 golang.org/x/text v0.22.0 google.golang.org/protobuf v1.36.5 ) require ( - cel.dev/expr v0.19.0 // indirect + cel.dev/expr v0.19.1 // indirect dario.cat/mergo v1.0.1 // indirect filippo.io/edwards25519 v1.1.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect @@ -143,6 +146,8 @@ require ( github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xeipuuv/gojsonschema v1.2.0 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0 // indirect + go.opentelemetry.io/otel/log v0.11.0 // indirect go.opentelemetry.io/proto/otlp v1.5.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect @@ -154,9 +159,9 @@ require ( golang.org/x/sync v0.11.0 // indirect golang.org/x/sys v0.30.0 // indirect golang.org/x/tools v0.30.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20250207221924-e9438ea467c6 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250207221924-e9438ea467c6 // indirect - google.golang.org/grpc v1.70.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250218202821-56aae31c358a // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a // indirect + google.golang.org/grpc v1.71.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go/go.sum b/go/go.sum index 2c52d5a19d..0713e584e1 100644 --- a/go/go.sum +++ b/go/go.sum @@ -1,5 +1,5 @@ -cel.dev/expr v0.19.0 h1:lXuo+nDhpyJSpWxpPVi5cPUwzKb+dsdOiw6IreM5yt0= -cel.dev/expr v0.19.0/go.mod h1:MrpN08Q+lEBs+bGYdLxxHkZoUSsCp0nSKTs0nTymJgw= +cel.dev/expr v0.19.1 h1:NciYrtDRIR0lNCnH1LFJegdjspNx9fI59O7TWcua/W4= +cel.dev/expr v0.19.1/go.mod h1:MrpN08Q+lEBs+bGYdLxxHkZoUSsCp0nSKTs0nTymJgw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= connectrpc.com/connect v1.18.1 h1:PAg7CjSAGvscaf6YZKUefjoih5Z/qYkyaTrBW8xvYPw= connectrpc.com/connect v1.18.1/go.mod h1:0292hj1rnx8oFrStN7cB4jjVBeqs+Yx5yDIC2prWDO8= @@ -181,8 +181,8 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -463,26 +463,36 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/bridges/otelslog v0.10.0 h1:lRKWBp9nWoBe1HKXzc3ovkro7YZSb72X2+3zYNxfXiU= +go.opentelemetry.io/contrib/bridges/otelslog v0.10.0/go.mod h1:D+iyUv/Wxbw5LUDO5oh7x744ypftIryiWjoj42I6EKs= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0 h1:CV7UdSGJt/Ao6Gp4CXckLxVRRsRgDHoI8XjbL3PDl8s= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0/go.mod h1:FRmFuRJfag1IZ2dPkHnEoSFVgTVPUd2qf5Vi69hLb8I= go.opentelemetry.io/contrib/instrumentation/runtime v0.59.0 h1:rfi2MMujBc4yowE0iHckZX4o4jg6SA67EnFVL8ldVvU= go.opentelemetry.io/contrib/instrumentation/runtime v0.59.0/go.mod h1:IO/gfPEcQYpOpPxn1OXFp1DvRY0viP8ONMedXLjjHIU= -go.opentelemetry.io/otel v1.34.0 h1:zRLXxLCgL1WyKsPVrgbSdMN4c0FMkDAskSTQP+0hdUY= -go.opentelemetry.io/otel v1.34.0/go.mod h1:OWFPOQ+h4G8xpyjgqo4SxJYdDQ/qmRH+wivy7zzx9oI= +go.opentelemetry.io/contrib/processors/minsev v0.8.0 h1:/i0gaV0Z174Twy1/NfgQoE+oQvFVbQItNl8UMwe62Jc= +go.opentelemetry.io/contrib/processors/minsev v0.8.0/go.mod h1:5siKBWhXmdM2gNh8KHZ4b97bdS4MYhqPJEEu6JtHciw= +go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ= +go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.11.0 h1:C/Wi2F8wEmbxJ9Kuzw/nhP+Z9XaHYMkyDmXy6yR2cjw= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.11.0/go.mod h1:0Lr9vmGKzadCTgsiBydxr6GEZ8SsZ7Ks53LzjWG5Ar4= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.34.0 h1:opwv08VbCZ8iecIWs+McMdHRcAXzjAeda3uG2kI/hcA= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.34.0/go.mod h1:oOP3ABpW7vFHulLpE8aYtNBodrHhMTrvfxUXGvqm7Ac= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0 h1:OeNbIYk/2C15ckl7glBlOBp5+WlYsOElzTNmiPW/x60= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0/go.mod h1:7Bept48yIeqxP2OZ9/AqIpYS94h2or0aB4FypJTc8ZM= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.34.0 h1:BEj3SPM81McUZHYjRS5pEgNgnmzGJ5tRpU5krWnV8Bs= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.34.0/go.mod h1:9cKLGBDzI/F3NoHLQGm4ZrYdIHsvGt6ej6hUowxY0J4= -go.opentelemetry.io/otel/metric v1.34.0 h1:+eTR3U0MyfWjRDhmFMxe2SsW64QrZ84AOhvqS7Y+PoQ= -go.opentelemetry.io/otel/metric v1.34.0/go.mod h1:CEDrp0fy2D0MvkXE+dPV7cMi8tWZwX3dmaIhwPOaqHE= -go.opentelemetry.io/otel/sdk v1.34.0 h1:95zS4k/2GOy069d321O8jWgYsW3MzVV+KuSPKp7Wr1A= -go.opentelemetry.io/otel/sdk v1.34.0/go.mod h1:0e/pNiaMAqaykJGKbi+tSjWfNNHMTxoC9qANsCzbyxU= +go.opentelemetry.io/otel/log v0.11.0 h1:c24Hrlk5WJ8JWcwbQxdBqxZdOK7PcP/LFtOtwpDTe3Y= +go.opentelemetry.io/otel/log v0.11.0/go.mod h1:U/sxQ83FPmT29trrifhQg+Zj2lo1/IPN1PF6RTFqdwc= +go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M= +go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE= +go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY= +go.opentelemetry.io/otel/sdk v1.35.0/go.mod h1:+ga1bZliga3DxJ3CQGg3updiaAJoNECOgJREo9KHGQg= +go.opentelemetry.io/otel/sdk/log v0.11.0 h1:7bAOpjpGglWhdEzP8z0VXc4jObOiDEwr3IYbhBnjk2c= +go.opentelemetry.io/otel/sdk/log v0.11.0/go.mod h1:dndLTxZbwBstZoqsJB3kGsRPkpAgaJrWfQg3lhlHFFY= go.opentelemetry.io/otel/sdk/metric v1.34.0 h1:5CeK9ujjbFVL5c1PhLuStg1wxA7vQv7ce1EK0Gyvahk= go.opentelemetry.io/otel/sdk/metric v1.34.0/go.mod h1:jQ/r8Ze28zRKoNRdkjCZxfs6YvBTG1+YIqyFVFYec5w= -go.opentelemetry.io/otel/trace v1.34.0 h1:+ouXS2V8Rd4hp4580a8q23bg0azF2nI8cqLYnC8mh/k= -go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE= +go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs= +go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= go.opentelemetry.io/proto/otlp v1.5.0 h1:xJvq7gMzB31/d406fB8U5CBdyQGw4P399D1aQWU/3i4= go.opentelemetry.io/proto/otlp v1.5.0/go.mod h1:keN8WnHxOy8PG0rQZjJJ5A2ebUoafqWp0eVQ4yIXvJ4= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= @@ -596,12 +606,12 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/genproto/googleapis/api v0.0.0-20250207221924-e9438ea467c6 h1:L9JNMl/plZH9wmzQUHleO/ZZDSN+9Gh41wPczNy+5Fk= -google.golang.org/genproto/googleapis/api v0.0.0-20250207221924-e9438ea467c6/go.mod h1:iYONQfRdizDB8JJBybql13nArx91jcUk7zCXEsOofM4= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250207221924-e9438ea467c6 h1:2duwAxN2+k0xLNpjnHTXoMUgnv6VPSp5fiqTuwSxjmI= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250207221924-e9438ea467c6/go.mod h1:8BS3B93F/U1juMFq9+EDk+qOT5CO1R9IzXxG3PTqiRk= -google.golang.org/grpc v1.70.0 h1:pWFv03aZoHzlRKHWicjsZytKAiYCtNS0dHbXnIdq7jQ= -google.golang.org/grpc v1.70.0/go.mod h1:ofIJqVKDXx/JiXrwr2IG4/zwdH9txy3IlF40RmcJSQw= +google.golang.org/genproto/googleapis/api v0.0.0-20250218202821-56aae31c358a h1:nwKuGPlUAt+aR+pcrkfFRrTU1BVrSmYyYMxYbUIVHr0= +google.golang.org/genproto/googleapis/api v0.0.0-20250218202821-56aae31c358a/go.mod h1:3kWAYMk1I75K4vykHtKt2ycnOgpA6974V7bREqbsenU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a h1:51aaUVRocpvUOSQKM6Q7VuoaktNIaMCLuhZB6DKksq4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a/go.mod h1:uRxBH1mhmO8PGhU89cMcHaXKZqO+OfakD8QQO0oYwlQ= +google.golang.org/grpc v1.71.0 h1:kF77BGdPTQ4/JZWMlb9VpJ5pa25aqvVqogsxNHHdeBg= +google.golang.org/grpc v1.71.0/go.mod h1:H0GRtasmQOh9LkFoCPDu3ZrwUtD1YGE+b2vYBYd/8Ec= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/go/internal/services/keys/service.go b/go/internal/services/keys/service.go index 204cde9ffc..b9d5058c46 100644 --- a/go/internal/services/keys/service.go +++ b/go/internal/services/keys/service.go @@ -2,7 +2,7 @@ package keys import ( "github.com/unkeyed/unkey/go/pkg/db" - "github.com/unkeyed/unkey/go/pkg/logging" + "github.com/unkeyed/unkey/go/pkg/otel/logging" ) type Config struct { diff --git a/go/internal/services/permissions/service.go b/go/internal/services/permissions/service.go index fb8c1903ec..c859abf2f1 100644 --- a/go/internal/services/permissions/service.go +++ b/go/internal/services/permissions/service.go @@ -2,7 +2,7 @@ package permissions import ( "github.com/unkeyed/unkey/go/pkg/db" - "github.com/unkeyed/unkey/go/pkg/logging" + "github.com/unkeyed/unkey/go/pkg/otel/logging" "github.com/unkeyed/unkey/go/pkg/rbac" ) diff --git a/go/internal/services/ratelimit/sliding_window.go b/go/internal/services/ratelimit/sliding_window.go index fb2474929b..ed870a5027 100644 --- a/go/internal/services/ratelimit/sliding_window.go +++ b/go/internal/services/ratelimit/sliding_window.go @@ -12,7 +12,7 @@ import ( "github.com/unkeyed/unkey/go/pkg/circuitbreaker" "github.com/unkeyed/unkey/go/pkg/clock" "github.com/unkeyed/unkey/go/pkg/cluster" - "github.com/unkeyed/unkey/go/pkg/logging" + "github.com/unkeyed/unkey/go/pkg/otel/logging" "github.com/unkeyed/unkey/go/pkg/otel/tracing" "go.opentelemetry.io/otel/attribute" ) diff --git a/go/pkg/cache/cache.go b/go/pkg/cache/cache.go index a195413df2..842ef0aed1 100644 --- a/go/pkg/cache/cache.go +++ b/go/pkg/cache/cache.go @@ -11,7 +11,7 @@ import ( "github.com/panjf2000/ants" "github.com/unkeyed/unkey/go/pkg/clock" "github.com/unkeyed/unkey/go/pkg/fault" - "github.com/unkeyed/unkey/go/pkg/logging" + "github.com/unkeyed/unkey/go/pkg/otel/logging" "github.com/unkeyed/unkey/go/pkg/otel/metrics" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/metric" diff --git a/go/pkg/cache/cache_test.go b/go/pkg/cache/cache_test.go index bdfc164f2c..b0f57b5e97 100644 --- a/go/pkg/cache/cache_test.go +++ b/go/pkg/cache/cache_test.go @@ -10,7 +10,7 @@ import ( "github.com/unkeyed/unkey/go/pkg/cache" "github.com/unkeyed/unkey/go/pkg/clock" - "github.com/unkeyed/unkey/go/pkg/logging" + "github.com/unkeyed/unkey/go/pkg/otel/logging" ) func TestWriteRead(t *testing.T) { diff --git a/go/pkg/cache/simulation_test.go b/go/pkg/cache/simulation_test.go index 8555e0da16..60bfb5e4ce 100644 --- a/go/pkg/cache/simulation_test.go +++ b/go/pkg/cache/simulation_test.go @@ -10,7 +10,7 @@ import ( "github.com/stretchr/testify/require" "github.com/unkeyed/unkey/go/pkg/cache" "github.com/unkeyed/unkey/go/pkg/clock" - "github.com/unkeyed/unkey/go/pkg/logging" + "github.com/unkeyed/unkey/go/pkg/otel/logging" "github.com/unkeyed/unkey/go/pkg/sim" ) diff --git a/go/pkg/circuitbreaker/lib.go b/go/pkg/circuitbreaker/lib.go index d335affd29..05c0f41565 100644 --- a/go/pkg/circuitbreaker/lib.go +++ b/go/pkg/circuitbreaker/lib.go @@ -8,7 +8,7 @@ import ( "time" "github.com/unkeyed/unkey/go/pkg/clock" - "github.com/unkeyed/unkey/go/pkg/logging" + "github.com/unkeyed/unkey/go/pkg/otel/logging" "github.com/unkeyed/unkey/go/pkg/otel/tracing" ) diff --git a/go/pkg/clickhouse/client.go b/go/pkg/clickhouse/client.go index f04d8405b2..6eefb80c9d 100644 --- a/go/pkg/clickhouse/client.go +++ b/go/pkg/clickhouse/client.go @@ -9,7 +9,7 @@ import ( "github.com/unkeyed/unkey/go/pkg/batch" "github.com/unkeyed/unkey/go/pkg/clickhouse/schema" "github.com/unkeyed/unkey/go/pkg/fault" - "github.com/unkeyed/unkey/go/pkg/logging" + "github.com/unkeyed/unkey/go/pkg/otel/logging" "github.com/unkeyed/unkey/go/pkg/retry" ) @@ -101,7 +101,7 @@ func New(config Config) (*Clickhouse, error) { FlushInterval: time.Second, Consumers: 4, Flush: func(ctx context.Context, rows []schema.ApiRequestV1) { - table := "raw_api_requests_v1" + table := "metrics.raw_api_requests_v1" err := flush(ctx, conn, table, rows) if err != nil { config.Logger.Error("failed to flush batch", @@ -120,7 +120,7 @@ func New(config Config) (*Clickhouse, error) { FlushInterval: time.Second, Consumers: 4, Flush: func(ctx context.Context, rows []schema.KeyVerificationRequestV1) { - table := "raw_key_verifications_v1" + table := "verifications.raw_key_verifications_v1" err := flush(ctx, conn, table, rows) if err != nil { config.Logger.Error("failed to flush batch", diff --git a/go/pkg/cluster/cluster.go b/go/pkg/cluster/cluster.go index 366537ea2f..9bd731f391 100644 --- a/go/pkg/cluster/cluster.go +++ b/go/pkg/cluster/cluster.go @@ -5,8 +5,8 @@ import ( "fmt" "github.com/unkeyed/unkey/go/pkg/events" - "github.com/unkeyed/unkey/go/pkg/logging" "github.com/unkeyed/unkey/go/pkg/membership" + "github.com/unkeyed/unkey/go/pkg/otel/logging" "github.com/unkeyed/unkey/go/pkg/otel/metrics" "github.com/unkeyed/unkey/go/pkg/ring" "go.opentelemetry.io/otel/attribute" diff --git a/go/pkg/cluster/cluster_test.go b/go/pkg/cluster/cluster_test.go index ecdf4bfa99..d92151a696 100644 --- a/go/pkg/cluster/cluster_test.go +++ b/go/pkg/cluster/cluster_test.go @@ -8,8 +8,8 @@ import ( "github.com/stretchr/testify/require" "github.com/unkeyed/unkey/go/pkg/discovery" - "github.com/unkeyed/unkey/go/pkg/logging" "github.com/unkeyed/unkey/go/pkg/membership" + "github.com/unkeyed/unkey/go/pkg/otel/logging" "github.com/unkeyed/unkey/go/pkg/port" ) diff --git a/go/pkg/db/database.go b/go/pkg/db/database.go index d2cebe33aa..ae373d08c1 100644 --- a/go/pkg/db/database.go +++ b/go/pkg/db/database.go @@ -7,7 +7,7 @@ import ( _ "github.com/go-sql-driver/mysql" "github.com/unkeyed/unkey/go/pkg/fault" - "github.com/unkeyed/unkey/go/pkg/logging" + "github.com/unkeyed/unkey/go/pkg/otel/logging" ) // Config defines the parameters needed to establish database connections. diff --git a/go/pkg/discovery/redis.go b/go/pkg/discovery/redis.go index d62c969216..71014ced8f 100644 --- a/go/pkg/discovery/redis.go +++ b/go/pkg/discovery/redis.go @@ -6,7 +6,7 @@ import ( "time" "github.com/redis/go-redis/v9" - "github.com/unkeyed/unkey/go/pkg/logging" + "github.com/unkeyed/unkey/go/pkg/otel/logging" "github.com/unkeyed/unkey/go/pkg/retry" ) diff --git a/go/pkg/membership/logger.go b/go/pkg/membership/logger.go index eda00f8830..c7ae8673f2 100644 --- a/go/pkg/membership/logger.go +++ b/go/pkg/membership/logger.go @@ -3,7 +3,7 @@ package membership import ( "bytes" - "github.com/unkeyed/unkey/go/pkg/logging" + "github.com/unkeyed/unkey/go/pkg/otel/logging" ) // logger implements io.Writer interface to integrate memberlist's logging system diff --git a/go/pkg/membership/membership_test.go b/go/pkg/membership/membership_test.go index 285da43a35..783d9f40ed 100644 --- a/go/pkg/membership/membership_test.go +++ b/go/pkg/membership/membership_test.go @@ -8,8 +8,8 @@ import ( "github.com/stretchr/testify/require" "github.com/unkeyed/unkey/go/pkg/discovery" - "github.com/unkeyed/unkey/go/pkg/logging" "github.com/unkeyed/unkey/go/pkg/membership" + "github.com/unkeyed/unkey/go/pkg/otel/logging" "github.com/unkeyed/unkey/go/pkg/port" "github.com/unkeyed/unkey/go/pkg/retry" ) diff --git a/go/pkg/membership/serf.go b/go/pkg/membership/serf.go index 5d153b4c32..265a7fd171 100644 --- a/go/pkg/membership/serf.go +++ b/go/pkg/membership/serf.go @@ -12,7 +12,7 @@ import ( "github.com/unkeyed/unkey/go/pkg/discovery" "github.com/unkeyed/unkey/go/pkg/events" "github.com/unkeyed/unkey/go/pkg/fault" - "github.com/unkeyed/unkey/go/pkg/logging" + "github.com/unkeyed/unkey/go/pkg/otel/logging" "github.com/unkeyed/unkey/go/pkg/retry" ) diff --git a/go/pkg/otel/grafana.go b/go/pkg/otel/grafana.go index 649544f8df..a6320ba507 100644 --- a/go/pkg/otel/grafana.go +++ b/go/pkg/otel/grafana.go @@ -5,12 +5,18 @@ import ( "fmt" "time" + "github.com/unkeyed/unkey/go/pkg/otel/logging" "github.com/unkeyed/unkey/go/pkg/otel/metrics" "github.com/unkeyed/unkey/go/pkg/otel/tracing" "github.com/unkeyed/unkey/go/pkg/shutdown" + "github.com/unkeyed/unkey/go/pkg/version" + "go.opentelemetry.io/contrib/bridges/otelslog" "go.opentelemetry.io/contrib/instrumentation/runtime" + "go.opentelemetry.io/contrib/processors/minsev" + sdklog "go.opentelemetry.io/otel/sdk/log" "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp" "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp" "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" "go.opentelemetry.io/otel/sdk/metric" @@ -30,10 +36,6 @@ type Config struct { // which helps with identifying regional performance patterns or issues. CloudRegion string - // GrafanaEndpoint is the URL endpoint where telemetry data will be sent. - // For Grafana Cloud, this looks like "https://otlp-gateway-{your-stack-id}.grafana.net/otlp" - GrafanaEndpoint string - // Application is the name of your application, used to identify the source of telemetry data. // This appears in Grafana dashboards and alerts. Application string @@ -89,12 +91,41 @@ func InitGrafana(ctx context.Context, config Config, shutdowns *shutdown.Shutdow return fmt.Errorf("failed to create resource: %w", err) } + // Configure OTLP log handler + logExporter, err := otlploghttp.New(ctx, + otlploghttp.WithCompression(otlploghttp.GzipCompression), + ) + if err != nil { + return fmt.Errorf("failed to create log exporter: %w", err) + } + shutdowns.RegisterCtx(logExporter.Shutdown) + + var processor sdklog.Processor = sdklog.NewBatchProcessor(logExporter, sdklog.WithExportBufferSize(512)) + + processor = minsev.NewLogProcessor(processor, minsev.SeverityInfo) + shutdowns.RegisterCtx(processor.Shutdown) + + // if config.LogDebug { + // processor = minsev.NewLogProcessor(processor, minsev.SeverityDebug) + // } + + logProvider := sdklog.NewLoggerProvider( + sdklog.WithResource(res), + sdklog.WithProcessor(processor), + ) + shutdowns.RegisterCtx(logProvider.Shutdown) + + logging.SetHandler(otelslog.NewHandler( + config.Application, + otelslog.WithLoggerProvider(logProvider), + otelslog.WithVersion(version.Version), + otelslog.WithSource(true), + )) + // Initialize trace exporter with configuration matching the old implementation traceExporter, err := otlptracehttp.New(ctx, - otlptracehttp.WithEndpoint(config.GrafanaEndpoint), otlptracehttp.WithCompression(otlptracehttp.GzipCompression), - otlptracehttp.WithInsecure(), // For local development - + // otlptracehttp.WithInsecure(), // For local development ) if err != nil { return fmt.Errorf("failed to create trace exporter: %w", err) @@ -118,10 +149,8 @@ func InitGrafana(ctx context.Context, config Config, shutdowns *shutdown.Shutdow // Initialize metrics exporter with configuration matching the old implementation metricExporter, err := otlpmetrichttp.New(ctx, - otlpmetrichttp.WithEndpoint(config.GrafanaEndpoint), otlpmetrichttp.WithCompression(otlpmetrichttp.GzipCompression), - otlpmetrichttp.WithInsecure(), // For local development - + // otlpmetrichttp.WithInsecure(), // For local development ) if err != nil { return fmt.Errorf("failed to create metric exporter: %w", err) diff --git a/go/pkg/logging/doc.go b/go/pkg/otel/logging/doc.go similarity index 100% rename from go/pkg/logging/doc.go rename to go/pkg/otel/logging/doc.go diff --git a/go/pkg/logging/interface.go b/go/pkg/otel/logging/interface.go similarity index 100% rename from go/pkg/logging/interface.go rename to go/pkg/otel/logging/interface.go diff --git a/go/pkg/logging/noop.go b/go/pkg/otel/logging/noop.go similarity index 100% rename from go/pkg/logging/noop.go rename to go/pkg/otel/logging/noop.go diff --git a/go/pkg/logging/slog.go b/go/pkg/otel/logging/slog.go similarity index 72% rename from go/pkg/logging/slog.go rename to go/pkg/otel/logging/slog.go index a73c8470c5..3020818cd1 100644 --- a/go/pkg/logging/slog.go +++ b/go/pkg/otel/logging/slog.go @@ -9,17 +9,19 @@ import ( "github.com/lmittmann/tint" ) -// Config defines the configuration options for creating a logger. -type Config struct { - // Development enables human-readable logging with additional details - // that are helpful during development. When false, logs are formatted - // as JSON for easier machine processing. - Development bool - - // NoColor disables ANSI color codes in the development output format. - // This is useful for environments where colors are not supported or - // when redirecting logs to files. - NoColor bool +var handler slog.Handler + +func init() { + handler = tint.NewHandler(os.Stdout, &tint.Options{ + AddSource: false, + Level: slog.LevelDebug, + ReplaceAttr: nil, + TimeFormat: time.StampMilli, + NoColor: false, + }) +} +func SetHandler(h slog.Handler) { + handler = h } // logger implements the Logger interface using Go's standard slog package. @@ -44,27 +46,10 @@ type logger struct { // prodLogger := logging.New(logging.Config{ // Development: false, // }) -func New(cfg Config) Logger { - var handler slog.Handler - switch { - case cfg.Development && !cfg.NoColor: - // Colored, human-readable format for development with terminal colors - handler = tint.NewHandler(os.Stdout, &tint.Options{ - AddSource: false, - Level: slog.LevelInfo, - ReplaceAttr: nil, - TimeFormat: time.StampMilli, - NoColor: false, - }) - case cfg.Development: - // Plain text format for development without colors - handler = slog.NewTextHandler(os.Stdout, nil) - default: - // JSON format for production environments - handler = slog.NewJSONHandler(os.Stdout, nil) - } +func New() Logger { l := slog.New(handler) + return &logger{ logger: l, } diff --git a/go/pkg/ring/ring.go b/go/pkg/ring/ring.go index efc864bcca..2fbc2e528f 100644 --- a/go/pkg/ring/ring.go +++ b/go/pkg/ring/ring.go @@ -11,7 +11,7 @@ import ( "sync" "github.com/unkeyed/unkey/go/pkg/fault" - "github.com/unkeyed/unkey/go/pkg/logging" + "github.com/unkeyed/unkey/go/pkg/otel/logging" ) // Node represents an individual entity in the ring, usually a service instance diff --git a/go/pkg/ring/ring_test.go b/go/pkg/ring/ring_test.go index 164163c606..93165094f5 100644 --- a/go/pkg/ring/ring_test.go +++ b/go/pkg/ring/ring_test.go @@ -8,7 +8,7 @@ import ( "github.com/gonum/stat" "github.com/segmentio/ksuid" "github.com/stretchr/testify/require" - "github.com/unkeyed/unkey/go/pkg/logging" + "github.com/unkeyed/unkey/go/pkg/otel/logging" ) // we don't need tags for this test. diff --git a/go/pkg/sim/simulation.go b/go/pkg/sim/simulation.go index f695f13f09..3ae8cf647d 100644 --- a/go/pkg/sim/simulation.go +++ b/go/pkg/sim/simulation.go @@ -7,7 +7,7 @@ import ( "time" "github.com/unkeyed/unkey/go/pkg/clock" - "github.com/unkeyed/unkey/go/pkg/logging" + "github.com/unkeyed/unkey/go/pkg/otel/logging" ) type Validator[S any] func(*S) error @@ -66,7 +66,7 @@ func New[State any](seed Seed, fns ...apply[State]) *Simulation[State] { Errors: []error{}, applied: 0, eventStats: make(map[string]int), // Initialize event stats map - logger: logging.New(logging.Config{NoColor: false, Development: true}), + logger: logging.New(), validators: []Validator[State]{}, } diff --git a/go/pkg/testutil/http.go b/go/pkg/testutil/http.go index caab7bc614..44e85dabe0 100644 --- a/go/pkg/testutil/http.go +++ b/go/pkg/testutil/http.go @@ -18,7 +18,7 @@ import ( "github.com/unkeyed/unkey/go/pkg/cluster" "github.com/unkeyed/unkey/go/pkg/db" "github.com/unkeyed/unkey/go/pkg/hash" - "github.com/unkeyed/unkey/go/pkg/logging" + "github.com/unkeyed/unkey/go/pkg/otel/logging" "github.com/unkeyed/unkey/go/pkg/testutil/containers" "github.com/unkeyed/unkey/go/pkg/uid" "github.com/unkeyed/unkey/go/pkg/zen" @@ -53,7 +53,7 @@ type Harness struct { func NewHarness(t *testing.T) *Harness { clk := clock.NewTestClock() - logger := logging.New(logging.Config{Development: true, NoColor: false}) + logger := logging.New() cont := containers.New(t) diff --git a/go/pkg/zen/README.md b/go/pkg/zen/README.md index b7dcd5a7e7..fd45d99665 100644 --- a/go/pkg/zen/README.md +++ b/go/pkg/zen/README.md @@ -38,7 +38,7 @@ import ( "net/http" "github.com/unkeyed/unkey/go/pkg/zen" - "github.com/unkeyed/unkey/go/pkg/logging" + "github.com/unkeyed/unkey/go/pkg/otel/logging" "github.com/unkeyed/unkey/go/pkg/zen/validation" "github.com/unkeyed/unkey/go/pkg/fault" ) diff --git a/go/pkg/zen/middleware_errors.go b/go/pkg/zen/middleware_errors.go index c87d0f1474..121f1332b1 100644 --- a/go/pkg/zen/middleware_errors.go +++ b/go/pkg/zen/middleware_errors.go @@ -6,7 +6,7 @@ import ( "github.com/unkeyed/unkey/go/api" "github.com/unkeyed/unkey/go/pkg/fault" - "github.com/unkeyed/unkey/go/pkg/logging" + "github.com/unkeyed/unkey/go/pkg/otel/logging" ) // WithErrorHandling returns middleware that translates errors into appropriate diff --git a/go/pkg/zen/middleware_logger.go b/go/pkg/zen/middleware_logger.go index 892c90a8f0..f48d3d81f5 100644 --- a/go/pkg/zen/middleware_logger.go +++ b/go/pkg/zen/middleware_logger.go @@ -5,7 +5,7 @@ import ( "log/slog" "time" - "github.com/unkeyed/unkey/go/pkg/logging" + "github.com/unkeyed/unkey/go/pkg/otel/logging" ) // WithLogging returns middleware that logs information about each request. diff --git a/go/pkg/zen/server.go b/go/pkg/zen/server.go index 26002830d3..54c91d18f8 100644 --- a/go/pkg/zen/server.go +++ b/go/pkg/zen/server.go @@ -8,7 +8,7 @@ import ( "time" "github.com/unkeyed/unkey/go/pkg/fault" - "github.com/unkeyed/unkey/go/pkg/logging" + "github.com/unkeyed/unkey/go/pkg/otel/logging" ) // Server manages HTTP server configuration, route registration, and lifecycle.