feat: Phase 12 - Application Performance Monitoring and Security Governance

.github/workflows/strix.yml

@@ -82,7 +82,7 @@ jobs:
         env:
           LLM_API_KEY: ${{ secrets.LLM_API_KEY }}
         run: |
-          strix_llm="github_models/gpt-4o"
+          strix_llm="github/gpt-4o"
           llm_api_key="$(printf '%s' "$LLM_API_KEY" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')"
           if [ -z "$llm_api_key" ]; then
             # On protected/privileged events, fail-closed: missing secrets must
@@ -117,7 +117,7 @@ jobs:
           DEFAULT_PROVIDER: gemini
         run: |
           . "$TRUSTED_WORKSPACE/scripts/ci/strix_model_utils.sh"
-          strix_llm="github_models/gpt-4o"
+          strix_llm="github/gpt-4o"
           requires_vertex_auth='false'
           model_auth_rc=0
           model_requires_vertex_auth "$strix_llm" || model_auth_rc=$?
@@ -225,7 +225,7 @@ jobs:
         run: |
           umask 077
           strix_llm_file="$RUNNER_TEMP/strix_llm.txt"
-            printf '%s' "github_models/gpt-4o" > "$strix_llm_file"
+            printf '%s' "github/gpt-4o" > "$strix_llm_file"
           echo "STRIX_LLM_FILE=$strix_llm_file" >> "$GITHUB_ENV"
 
       - name: Prepare LLM API base input file

.gitignore

@@ -71,3 +71,5 @@ frontend/test-results/
 frontend/playwright-report/
 frontend/playwright/.cache/
 frontend/trace_output/
+
+.worktrees/

AGENTS.md

@@ -10,7 +10,7 @@
   explicit `if: ${{ always() }}` upload steps when needed.
 - Prefer upgrading or removing vulnerable dependencies over downgrading patched
   packages unless compatibility evidence is recorded in the PR.
-- Strix Security Scan must use `github_models/gpt-4o` as the default model to bypass Vertex AI GCP credential prerequisites in PR bounds.
+- Strix Security Scan must use `github/gpt-4o` as the default model to bypass Vertex AI GCP credential prerequisites in PR bounds.
 
 ## PR automation and review defaults
 

backend/api/tenant_config.py

@@ -6,7 +6,7 @@
 
 from db.models import TenantConfig
 from db.session import get_db
-from api.auth import get_current_user
+from api.auth import get_current_user, get_current_user_role
 from services.email_client import (
     validate_smtp_destination,
     validate_smtp_host,
@@ -15,6 +15,14 @@
 
 router = APIRouter(prefix="/api/config")
 
+@router.get("/global")
+async def get_global_config(
+    role: str = Depends(get_current_user_role)
+):
+    if role not in ["platform_admin", "organization_admin"]:
+        raise HTTPException(status_code=403, detail="Not enough privileges")
+    return {"status": "ok", "global_settings": {}}
+
 
 class TenantConfigCreate(BaseModel):
     user_id: str

backend/tests/test_apm_observability.py

@@ -3,7 +3,7 @@
 ROOT_DIR = Path(__file__).parent.parent.parent
 
 def test_observability_compose_file_exists():
-    assert (ROOT_DIR / "docker-compose.observability.yml").exists()
+    assert (ROOT_DIR / "docker-compose.infra.yml").exists()
 
 
 def test_observability_provisioning_exists():

backend/tests/test_tenant_config_api.py

@@ -189,3 +189,29 @@ def test_tenant_config_get_rejects_cross_user_access(client):
     assert response.json() == {
         "detail": "Mailbox settings are personal and can only be viewed by the authenticated user"
     }
+
+def test_global_config_requires_admin(client):
+    response = client.get(
+        "/api/config/global",
+        headers={
+            "X-User-Id": "member-user",
+            "X-User-Role": "member",
+            "X-Organization-Id": "org-acme",
+            "X-Workspace-Id": "ws-1",
+        },
+    )
+    assert response.status_code == 403
+    assert response.json() == {"detail": "Not enough privileges"}
+
+def test_global_config_allows_admin(client):
+    response = client.get(
+        "/api/config/global",
+        headers={
+            "X-User-Id": "admin-user",
+            "X-User-Role": "organization_admin",
+            "X-Organization-Id": "org-acme",
+            "X-Workspace-Id": "ws-1",
+        },
+    )
+    assert response.status_code == 200
+    assert response.json()["status"] == "ok"

docker-compose.infra.yml

@@ -19,7 +19,7 @@ services:
   prometheus:
     image: prom/prometheus:latest
     volumes:
-      - ./docs/operations/prometheus.yml:/etc/prometheus/prometheus.yml
+      - ./observability/prometheus.yml:/etc/prometheus/prometheus.yml
     ports:
       - "9090:9090"
     networks:
@@ -29,6 +29,8 @@ services:
     image: grafana/grafana:latest
     environment:
       - GF_SECURITY_ADMIN_PASSWORD=admin
+    volumes:
+      - ./observability/grafana/provisioning/datasources:/etc/grafana/provisioning/datasources
     ports:
       - "3000:3000"
     networks:
@@ -42,6 +44,17 @@ services:
     networks:
       - naruon-network
 
+  tempo:
+    image: grafana/tempo:latest
+    command: [ "-config.file=/etc/tempo.yaml" ]
+    volumes:
+      - ./observability/tempo.yaml:/etc/tempo.yaml
+    ports:
+      - "3200:3200"
+      - "4317" # otlp grpc
+    networks:
+      - naruon-network
+
   otel-collector:
     image: otel/opentelemetry-collector:0.88.0
     ports:

docs/plans/2026-05-24-phase11-implementation.md

@@ -0,0 +1,58 @@
+# Phase 11: Email Threading, DAG Ontology Pipeline, and CalDAV Sync Implementation Plan
+
+> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
+
+**Goal:** Wire the previously defined schema and core logic (from Phase 10) into the actual background sync workers and API endpoints. Implement true email threading, DAG ontology persistence, reply tracking, and CalDAV/WebDAV integration.
+
+**Architecture:** Use the existing `ImapSyncWorker` to trigger ontology and fingerprinting logic. Use `SenderRelationship` to store the DAG. Expose these via REST endpoints for the frontend.
+
+**Tech Stack:** FastAPI, SQLAlchemy, PostgreSQL, Python
+
+---
+
+## Task 1: Email Threading and Deduplication Pipeline
+
+**Files:**
+- Modify: `backend/services/threading_service.py` (or create if needed)
+- Modify: `backend/services/imap_worker.py` (or wherever email fetching happens)
+- Test: `backend/tests/test_threading_pipeline.py`
+
+- [ ] **Step 1: Write a test for threading/deduplication using `generate_email_fingerprint`**
+- [ ] **Step 2: Update the worker to compute fingerprint before DB insertion**
+- [ ] **Step 3: If fingerprint exists, update existing thread instead of inserting duplicate**
+- [ ] **Step 4: Verify tests pass**
+
+## Task 2: DAG Ontology & Self-to-Self Knowledge Extraction
+
+**Files:**
+- Modify: `backend/services/ontology_service.py`
+- Modify: `backend/api/ontology.py` (or create if needed)
+- Test: `backend/tests/test_ontology_pipeline.py`
+
+- [ ] **Step 1: Write a test that inserts a `SenderRelationship` to DB based on `analyze_sender_relationship`**
+- [ ] **Step 2: Update `ontology_service.py` to accept DB session and save the relationship including explicit tenant_id and source_id/thread_id fields**
+- [ ] **Step 3: If `process_self_to_self` is true, trigger a knowledge extraction task passing tenant and source context**
+- [ ] **Step 4: Create/Update `/api/ontology/relationships` to fetch the DAG for the frontend, filtering by tenant_id and source_id. Explicitly require the default signed-session authentication by registering the router with the `get_auth_context` dependency.**
+
+## Task 3: Reply Tracking Background Job
+
+**Files:**
+- Modify: `backend/services/reply_tracking_service.py`
+- Modify: `backend/api/emails.py`
+- Test: `backend/tests/test_reply_tracking.py`
+
+- [ ] **Step 1: Write a test for identifying sent emails awaiting replies**
+- [ ] **Step 2: Implement a background task that runs daily (or periodically) to flag missing replies**
+- [ ] **Step 3: Expose `requires_reply` and `schedule_conflict` in `/api/emails` response, verifying the router remains protected by `get_auth_context` and uses the authenticated context.**
+
+## Task 4: CalDAV/WebDAV Background Sync
+
+**Files:**
+- Modify: `backend/services/caldav_service.py`
+- Modify: `backend/services/webdav_service.py`
+- Test: `backend/tests/test_dav_sync.py`
+
+- [ ] **Step 1: Write tests for CalDAV event parsing and WebDAV file listing**
+- [ ] **Step 2: Implement `sync_caldav_accounts` to fetch and store events locally**
+- [ ] **Step 3: Implement `sync_webdav_folders` to fetch folder structures**
+- [ ] **Step 4: Ensure it supports N-accounts and ties data back to the `user_id`**

docs/plans/2026-05-24-phase12-apms-and-security.md

@@ -0,0 +1,50 @@
+# Phase 12: Application Performance Monitoring (APM) and Security Governance Plan
+
+> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
+
+**Goal:** Implement Open Source Application Performance Monitoring (APM) via OpenTelemetry and enforce security/GRC rules including RBAC endpoints and authentication.
+
+**Architecture:** 
+- Add OpenTelemetry SDK instrumentation to FastAPI.
+- Complete the APM infrastructure (Prometheus, Loki, Tempo, Grafana) definition in `docker-compose.infra.yml`.
+- Implement actual backend dependencies for `get_current_user_role` and enforce RBAC on specific endpoints.
+- Address any `Timeout`, `Fatal`, `Warn`, `Denied` suppression to ensure tests pass strictly.
+
+**Tech Stack:** FastAPI, OpenTelemetry, Prometheus, Docker Compose.
+
+---
+
+## Task 1: OpenTelemetry Instrumentation
+
+**Files:**
+- Modify: `backend/requirements.txt`
+- Modify: `backend/main.py`
+- Modify: `backend/tests/test_apm_observability.py`
+
+- [ ] **Step 1: Write a test ensuring `/metrics` is exposed and traces are configurable**
+- [ ] **Step 2: Add `opentelemetry-instrumentation-fastapi` and `prometheus-fastapi-instrumentator` to requirements**
+- [ ] **Step 3: Wire up `FastAPIInstrumentor.instrument_app(app)` in `main.py`**
+- [ ] **Step 4: Verify test passes without warnings**
+
+## Task 2: Grafana & APM Infrastructure Definition
+
+**Files:**
+- Modify: `docker-compose.infra.yml`
+- Create: `observability/grafana/provisioning/datasources/datasources.yaml`
+- Create: `observability/prometheus.yml`
+- Create: `observability/tempo.yaml`
+
+- [ ] **Step 1: Write tests ensuring these files exist in `test_apm_observability.py`**
+- [ ] **Step 2: Define Loki, Tempo, Prometheus, and Grafana containers in `docker-compose.infra.yml`**
+- [ ] **Step 3: Create the config files for the observability stack**
+- [ ] **Step 4: Verify tests pass**
+
+## Task 3: RBAC Endpoint Enforcement
+
+**Files:**
+- Modify: `backend/api/tenant_config.py`
+- Modify: `backend/tests/test_tenant_config_api.py`
+
+- [ ] **Step 1: Write tests asserting non-admins receive 403 Forbidden for admin endpoints**
+- [ ] **Step 2: Use `get_current_user_role` to secure `/api/tenant_config/global`**
+- [ ] **Step 3: Verify the changes correctly deny access to `member` roles**