Open Source APM/Observability 운영 경로 설계 및 구현

backend/main.py

@@ -9,6 +9,9 @@
 from api.emails import router as emails_router
 from api.tenant_config import router as tenant_config_router
 from services.imap_worker import ImapSyncWorker
+from prometheus_fastapi_instrumentator import Instrumentator
+from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
+from core.config import settings
 
 imap_worker = ImapSyncWorker()
 
@@ -24,7 +27,42 @@ async def lifespan(app: FastAPI):
         await imap_worker.stop()
 
 
-app = FastAPI(title="AI Email Client API", lifespan=lifespan)
+app = FastAPI(
+    title="Naruon Backend",
+    version="0.1.0",
+    lifespan=lifespan,
+)
+
+# OpenTelemetry Tracing Setup (if enabled)
+if os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT"):
+    from opentelemetry import trace
+    from opentelemetry.sdk.trace import TracerProvider
+    from opentelemetry.sdk.trace.export import BatchSpanProcessor
+    from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
+    from opentelemetry.sdk.resources import Resource, SERVICE_NAME
+
+    resource = Resource(attributes={
+        SERVICE_NAME: "naruon-backend"
+    })
+
+    trace_provider = TracerProvider(resource=resource)
+    processor = BatchSpanProcessor(OTLPSpanExporter())
+    trace_provider.add_span_processor(processor)
+    trace.set_tracer_provider(trace_provider)
+
+# Instrument Prometheus Metrics
+Instrumentator().instrument(app).expose(app, include_in_schema=False, should_gzip=True)
+
+# Instrument OpenTelemetry
+FastAPIInstrumentor.instrument_app(app)
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["http://localhost:3000", "http://127.0.0.1:3000", "http://localhost:8000", "http://127.0.0.1:8000"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
 
 app.include_router(search_router)
 app.include_router(llm_router)
@@ -33,6 +71,7 @@ async def lifespan(app: FastAPI):
 app.include_router(emails_router)
 app.include_router(tenant_config_router)
 
+
 app.add_middleware(
     CORSMiddleware,
     allow_origins=["http://localhost:3000"],

backend/pytest.ini

@@ -3,3 +3,6 @@ asyncio_default_fixture_loop_scope = function
 filterwarnings =
     ignore:You are using a Python version.*which Google will stop supporting.*:FutureWarning
     ignore:Unclosed <MemoryObject:ResourceWarning
+    ignore:pkg_resources is deprecated:UserWarning
+    ignore:pkg_resources is deprecated:DeprecationWarning
+    ignore:Deprecated call to .pkg_resources.declare_namespace:DeprecationWarning

backend/requirements.txt

@@ -18,3 +18,9 @@ google-auth-oauthlib==1.2.0
 email-validator==2.1.0
 cryptography==47.0.0
 python-multipart==0.0.9
+prometheus-fastapi-instrumentator==7.0.0
+opentelemetry-api==1.25.0
+opentelemetry-sdk==1.25.0
+opentelemetry-instrumentation-fastapi==0.46b0
+opentelemetry-exporter-otlp==1.25.0
+setuptools<71.0.0

backend/tests/test_apm_observability.py

@@ -0,0 +1,22 @@
+import pytest
+import subprocess
+from pathlib import Path
+
+def test_observability_compose_file_exists():
+    assert Path("../docker-compose.observability.yml").exists()
+
+def test_observability_provisioning_exists():
+    assert Path("../observability/grafana/provisioning/datasources/datasources.yaml").exists()
+    assert Path("../observability/prometheus.yml").exists()
+    assert Path("../observability/tempo.yaml").exists()
+
+def test_backend_exposes_metrics_endpoint():
+    from main import app
+    from fastapi.testclient import TestClient
+
+    with TestClient(app, headers={"X-User-Id": "testuser"}) as client:
+        # Before instrumentation, it might be 404, but after it should be 200
+        response = client.get("/metrics")
+        # Ensure it is added as part of the APM implementation
+        assert response.status_code == 200
+        assert "python_info" in response.text or "process_" in response.text

docker-compose.observability.yml

@@ -0,0 +1,35 @@
+services:
+  prometheus:
+    image: prom/prometheus:v2.51.2
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+    ports:
+      - "9090:9090"
+    volumes:
+      - ./observability/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+
+  grafana:
+    image: grafana/grafana:11.5.0
+    ports:
+      - "3000:3000"
+    environment:
+      - GF_AUTH_ANONYMOUS_ENABLED=true
+      - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
+    volumes:
+      - ./observability/grafana/provisioning:/etc/grafana/provisioning
+
+  loki:
+    image: grafana/loki:2.9.8
+    ports:
+      - "3100:3100"
+    command: -config.file=/etc/loki/local-config.yaml
+
+  tempo:
+    image: grafana/tempo:2.4.1
+    command: [ "-config.file=/etc/tempo.yaml" ]
+    volumes:
+      - ./observability/tempo.yaml:/etc/tempo.yaml:ro
+    ports:
+      - "3200:3200"
+      - "4317:4317"  # otlp grpc
+      - "4318:4318"  # otlp http

docs/plans/2026-05-11-apm-observability-implementation.md

@@ -0,0 +1,23 @@
+# Open Source APM & Observability Implementation Plan
+
+## Overview
+Implement OpenTelemetry, Prometheus, Grafana, Loki, and Tempo for Open Source Application Performance Monitoring (APM).
+
+## Reference
+- Target Issue: #135 (Closed for design, but we will reopen/implement)
+
+## Architecture
+- **Grafana**: Dashboards and visualization.
+- **Prometheus**: Metrics backend.
+- **Loki**: Log aggregation.
+- **Tempo**: Distributed tracing.
+- **Alloy / OpenTelemetry Collector**: Receive traces/metrics from FastAPI/Next.js and route to backends.
+
+## Tasks
+1. Create `docker-compose.observability.yml` to provision Grafana, Prometheus, Loki, and Tempo.
+2. Update backend FastAPI application to include OpenTelemetry instrumentation.
+   - `opentelemetry-instrumentation-fastapi`
+   - `opentelemetry-exporter-otlp`
+3. Update frontend Next.js to expose basic metrics or tracing if applicable.
+4. Set up Grafana provisioning (Datasources for Prometheus, Loki, Tempo).
+5. Verify via live tests that the stack comes up and endpoints return 200 OK.

observability/grafana/provisioning/datasources/datasources.yaml

@@ -0,0 +1,15 @@
+apiVersion: 1
+datasources:
+  - name: Prometheus
+    type: prometheus
+    access: proxy
+    url: http://prometheus:9090
+    isDefault: true
+  - name: Tempo
+    type: tempo
+    access: proxy
+    url: http://tempo:3200
+  - name: Loki
+    type: loki
+    access: proxy
+    url: http://loki:3100

observability/prometheus.yml

@@ -0,0 +1,7 @@
+global:
+  scrape_interval: 15s
+
+scrape_configs:
+  - job_name: 'backend'
+    static_configs:
+      - targets: ['backend:8000']

observability/tempo.yaml

@@ -0,0 +1,28 @@
+server:
+  http_listen_port: 3200
+
+distributor:
+  receivers:
+    otlp:
+      protocols:
+        http:
+        grpc:
+
+ingester:
+  max_block_bytes: 1000000
+  max_block_duration: 5m
+
+compactor:
+  compaction:
+    compaction_window: 1h
+    max_block_bytes: 100000000
+    block_retention: 1h
+    compacted_block_retention: 10m
+
+storage:
+  trace:
+    backend: local
+    wal:
+      path: /tmp/tempo/wal
+    local:
+      path: /tmp/tempo/blocks