feat: Revisit OTEL & setup Grafana + Prometheus + metrics dashboard (#192)

* build: Set up otel-collector with prometheus and grafana

* fix: Marshal event with telemetry data

* fix: Event lifecycle metrics

* chore: Include destination type attribute in metrics

* chore: Grafana otel-collector accept traces & logs

* build: Grafana provisioning

* chore: Rename otel-collector-config.yaml file

* chore: Rename data source file to avoid prom linter

* build: Delivery success rate panel

Author: alexluong

Makefile

@@ -20,6 +20,12 @@ up/mqs:
 down/mqs:
 	docker-compose -f build/dev/mqs/compose.yml down
 
+up/grafana:
+	docker-compose -f build/dev/grafana/compose.yml up -d
+
+down/grafana:
+	docker-compose -f build/dev/grafana/compose.yml down
+
 up/uptrace:
 	docker-compose -f build/dev/uptrace/compose.yml up -d
 

build/dev/compose.yml

@@ -110,19 +110,6 @@ services:
       timeout: 1s
       retries: 30
 
-  otel-collector:
-    image: otel/opentelemetry-collector-contrib
-    volumes:
-      - ./otel-collector.yaml:/etc/otelcol-contrib/config.yaml
-    ports:
-      - 1888:1888 # pprof extension
-      - 8888:8888 # Prometheus metrics exposed by the Collector
-      - 8889:8889 # Prometheus exporter metrics
-      - 13133:13133 # health_check extension
-      - 4317:4317 # OTLP gRPC receiver
-      - 4318:4318 # OTLP http receiver
-      - 55679:55679 # zpages extension
-
 volumes:
   redis:
     driver: local

build/dev/grafana/compose.yml

@@ -0,0 +1,43 @@
+name: "outpost-grafana"
+
+services:
+  otel-collector:
+    image: otel/opentelemetry-collector-contrib:latest
+    command: ["--config=/etc/otel-collector-config.yaml"]
+    volumes:
+      - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml
+    ports:
+      - "4317:4317" # OTLP gRPC
+      - "4318:4318" # OTLP HTTP
+      - "8888:8888" # Telemetry metrics
+      - "8889:8889" # Prometheus metrics
+
+  prometheus:
+    image: prom/prometheus:latest
+    volumes:
+      - ./prometheus.yml:/etc/prometheus/prometheus.yml
+    ports:
+      - "9090:9090"
+
+  grafana:
+    image: grafana/grafana:latest
+    ports:
+      - "3000:3000"
+    environment:
+      - GF_SECURITY_ADMIN_USER=admin
+      - GF_SECURITY_ADMIN_PASSWORD=admin
+      - GF_AUTH_ANONYMOUS_ENABLED=true
+      - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
+      - GF_AUTH_DISABLE_LOGIN_FORM=true
+    volumes:
+      - ./provisioning/datasources:/etc/grafana/provisioning/datasources
+      - ./provisioning/dashboards:/etc/grafana/provisioning/dashboards
+      - ./dashboards:/etc/grafana/dashboards
+
+volumes:
+  grafana-data:
+
+networks:
+  default:
+    name: outpost
+    external: true