feat: api sends metrics and traces to local otel

deployment/docker-compose.yaml

@@ -52,6 +52,7 @@ services:
       - mysql
       - redis
       - clickhouse
+      - tempo
     environment:
       UNKEY_HTTP_PORT: 7070
       UNKEY_CLUSTER: true
@@ -61,6 +62,7 @@ services:
       UNKEY_CLUSTER_DISCOVERY_REDIS_URL: "redis://redis:6379"
       UNKEY_DATABASE_PRIMARY_DSN: "mysql://unkey:password@tcp(mysql:3900)/unkey?parseTime=true"
       UNKEY_CLICKHOUSE_URL: "clickhouse://default:password@clickhouse:9000"
+      UNKEY_OTEL_OTLP_ENDPOINT: "otel-collector:4318" # Point directly to Tempo
 
   redis:
     image: redis:latest
@@ -168,33 +170,72 @@ services:
       - agent
       - clickhouse
       - chproxy
+  otel-collector:
+    image: otel/opentelemetry-collector-contrib:latest
+    container_name: otel-collector
+    command: ["--config=/etc/otel-collector-config.yaml"]
+    volumes:
+      - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml
+    ports:
+      - "4318:4318" # OTLP HTTP
+    depends_on:
+      - prometheus
+      - tempo
+      - loki
+      - grafana
+  prometheus:
+    image: prom/prometheus:latest
+    container_name: prometheus
+    command:
+      - "--config.file=/etc/prometheus/config.yaml"
+      - "--storage.tsdb.path=/prometheus"
+      - "--web.enable-lifecycle"
+      - "--web.enable-remote-write-receiver" # Add this flag
+    ports:
+      - 9090:9090
+    volumes:
+      - ./prometheus/config.yaml:/etc/prometheus/config.yaml
+      - prometheus:/prometheus
 
-  # prometheus:
-  #     image: prom/prometheus
-  #     container_name: prometheus
-  #     command:
-  #       - '--config.file=/etc/prometheus/prometheus.yaml'
-  #     ports:
-  #       - 9090:9090
-  #     restart: unless-stopped
-  #     volumes:
-  #       - ./prometheus:/etc/prometheus
-  #       - prometheus:/prometheus
+  grafana:
+    image: grafana/grafana-oss:latest
+    container_name: grafana
+    ports:
+      - 3000:3000
+    environment:
+      - GF_SECURITY_ADMIN_USER=admin
+      - GF_SECURITY_ADMIN_PASSWORD=grafana
+      - GF_INSTALL_PLUGINS=grafana-clock-panel,grafana-simple-json-datasource
+    volumes:
+      - ./grafana/provisioning:/etc/grafana/provisioning
+      - grafana:/var/lib/grafana
+
+  # Tempo for distributed tracing
+  tempo:
+    image: grafana/tempo:latest
+    container_name: tempo
+    command: ["-config.file=/etc/tempo.yaml"]
+    volumes:
+      - ./tempo/config.yaml:/etc/tempo.yaml
+      - tempo:/tmp/tempo
+    ports:
+      - "3200:3200" # tempo
+
+  # Loki for logs
+  loki:
+    container_name: loki
+    image: grafana/loki:latest
+    ports:
+      - "3100:3100"
+    command: -config.file=/etc/loki/config.yaml
+    volumes:
+      - ./loki/config.yaml:/etc/loki/config.yaml
 
-  # grafana:
-  #   image: grafana/grafana
-  #   container_name: grafana
-  #   ports:
-  #     - 4000:3000
-  #   restart: unless-stopped
-  #   environment:
-  #         - GF_SECURITY_ADMIN_USER=admin
-  #         - GF_SECURITY_ADMIN_PASSWORD=grafana
-  #   volumes:
-  #     - ./grafana:/etc/grafana/provisioning/datasources
 volumes:
   mysql:
   grafana:
+  tempo:
+  loki:
   clickhouse:
   clickhouse-keeper:
   s3:

deployment/grafana/provisioning/datasources/datasources.yaml

@@ -0,0 +1,20 @@
+apiVersion: 1
+
+datasources:
+  - name: Prometheus
+    type: prometheus
+    access: proxy
+    url: http://prometheus:9090
+    isDefault: true
+
+  - name: Tempo
+    type: tempo
+    access: proxy
+    url: http://tempo:3200
+    uid: tempo
+
+  - name: Loki
+    type: loki
+    access: proxy
+    url: http://loki:3100
+    uid: loki

deployment/loki/config.yaml

@@ -0,0 +1,62 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+  grpc_listen_port: 9096
+  log_level: debug
+  grpc_server_max_concurrent_streams: 1000
+
+common:
+  instance_addr: 127.0.0.1
+  path_prefix: /tmp/loki
+  storage:
+    filesystem:
+      chunks_directory: /tmp/loki/chunks
+      rules_directory: /tmp/loki/rules
+  replication_factor: 1
+  ring:
+    kvstore:
+      store: inmemory
+
+query_range:
+  results_cache:
+    cache:
+      embedded_cache:
+        enabled: true
+        max_size_mb: 100
+
+limits_config:
+  metric_aggregation_enabled: true
+
+schema_config:
+  configs:
+    - from: 2020-10-24
+      store: tsdb
+      object_store: filesystem
+      schema: v13
+      index:
+        prefix: index_
+        period: 24h
+
+pattern_ingester:
+  enabled: true
+  metric_aggregation:
+    loki_address: localhost:3100
+
+ruler:
+  alertmanager_url: http://localhost:9093
+
+frontend:
+  encoding: protobuf
+# By default, Loki will send anonymous, but uniquely-identifiable usage and configuration
+# analytics to Grafana Labs. These statistics are sent to https://stats.grafana.org/
+#
+# Statistics help us better understand how Loki is used, and they show us performance
+# levels for most users. This helps us prioritize features and documentation.
+# For more information on what's sent, look at
+# https://github.com/grafana/loki/blob/main/pkg/analytics/stats.go
+# Refer to the buildReport method to see what goes into a report.
+#
+# If you would like to disable reporting, uncomment the following lines:
+#analytics:
+#  reporting_enabled: false

deployment/otel-collector-config.yaml

@@ -0,0 +1,44 @@
+receivers:
+  otlp:
+    protocols:
+      http:
+        endpoint: "0.0.0.0:4318"
+      grpc:
+        endpoint: "0.0.0.0:4317"
+
+processors:
+  batch:
+    send_batch_size: 10000
+    timeout: 5s
+
+exporters:
+  # For traces - send to Tempo
+  otlp/tempo:
+    endpoint: "tempo:4317"
+    tls:
+      insecure: true
+
+  # For metrics - send to Prometheus
+  prometheusremotewrite:
+    endpoint: "http://prometheus:9090/api/v1/write"
+    tls:
+      insecure: true
+
+  # Debug output for troubleshooting
+  debug:
+    verbosity: detailed
+
+service:
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [otlp/tempo, debug]
+    metrics:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [prometheusremotewrite, debug]
+    logs:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [debug]

deployment/prometheus/config.yaml

@@ -0,0 +1,20 @@
+global:
+  scrape_interval: 15s
+  evaluation_interval: 15s
+
+scrape_configs:
+  - job_name: "prometheus"
+    static_configs:
+      - targets: ["localhost:9090"]
+
+  - job_name: "tempo"
+    static_configs:
+      - targets: ["tempo:3200"]
+
+  - job_name: "loki"
+    static_configs:
+      - targets: ["loki:3100"]
+
+  - job_name: "otel-collector"
+    static_configs:
+      - targets: ["otel-collector:8889"] # If your collector exposes metrics

deployment/tempo/config.yaml

@@ -0,0 +1,24 @@
+server:
+  http_listen_port: 3200
+
+distributor:
+  receivers:
+    otlp:
+      protocols:
+        http:
+          endpoint: "0.0.0.0:4318"
+        grpc:
+          endpoint: "0.0.0.0:4317"
+
+storage:
+  trace:
+    backend: local
+    local:
+      path: /tmp/tempo
+
+ingester:
+  max_block_duration: 5m
+
+compactor:
+  compaction:
+    block_retention: 24h

go/apps/api/run.go

@@ -62,15 +62,18 @@ func Run(ctx context.Context, cfg Config) error {
 	}()
 
 	if cfg.OtelOtlpEndpoint != "" {
-		shutdownOtel, grafanaErr := otel.InitGrafana(ctx, otel.Config{
+		grafanaErr := otel.InitGrafana(ctx, otel.Config{
 			GrafanaEndpoint: cfg.OtelOtlpEndpoint,
 			Application:     "api",
 			Version:         version.Version,
-		})
+			NodeID:          cfg.ClusterNodeID,
+			CloudRegion:     cfg.Region,
+		},
+			shutdowns,
+		)
 		if grafanaErr != nil {
 			return fmt.Errorf("unable to init grafana: %w", grafanaErr)
 		}
-		shutdowns.RegisterCtx(shutdownOtel...)
 	}
 
 	db, err := db.New(db.Config{