diff --git a/AGENTS.md b/AGENTS.md index 4570211cb..99fe9dddf 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -104,6 +104,11 @@ server-authoritative source selection and provenance. Do not wire browser actions back to legacy `/api/calendar/sync` unless a trusted backend credential dependency and source-owner contract are explicitly in scope. +- Calendar writeback source selection must resolve through opaque + `calendar_writeback_sources.source_uid` registry rows, not sequential CalDAV + or WebDAV account ids. Browser-visible source ids must not reveal account + primary keys, and provider mutations remain future work until connector + execution can enforce capability, consent, and ETag/If-Match checks. - Self-sent knowledge extraction must first prove true self-to-self addressing, stay idempotent per source email, preserve email/thread provenance, and store only plain-text task titles. Do not create unlinked knowledge tasks from raw diff --git a/README.md b/README.md index 9c64e2a57..49a8925a9 100644 --- a/README.md +++ b/README.md @@ -253,9 +253,11 @@ public task ids instead of exposing database integer surrogates. The new Calendar actions in `EmailDetail` now request `/api/calendar/writeback-intent` for each extracted execution item and display the selected trusted source -provenance. The browser no longer claims `/api/calendar/sync` success from the -mail-detail action path; direct provider writes stay deferred until connector and -source registry work can enforce ETag/If-Match and owner capability checks. +provenance. Calendar source selection now reads opaque +`calendar_writeback_sources.source_uid` rows instead of exposing sequential +CalDAV account ids. The browser no longer claims `/api/calendar/sync` success +from the mail-detail action path; direct provider writes stay deferred until +connector execution can enforce ETag/If-Match and owner capability checks. ## Operations and release docs diff --git a/backend/api/calendar.py b/backend/api/calendar.py index 99bcf659f..094fb3597 100644 --- a/backend/api/calendar.py +++ b/backend/api/calendar.py @@ -3,12 +3,16 @@ from fastapi import APIRouter, Depends, HTTPException from pydantic import BaseModel, ConfigDict +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession from api.auth import ( AuthContext, get_auth_context, is_system_admin_role, is_tenant_admin_role, ) +from db.models import CalendarWritebackSource +from db.session import get_db from services.calendar_service import create_calendar_event, validate_calendar_todo_text from services.exceptions import CalendarServiceError, UnsafeCalendarTodoError @@ -26,7 +30,7 @@ class WritebackSource(BaseModel): provider: str protocol: Literal["caldav", "carddav", "webdav", "local"] owner_id: str - organization_id: str + organization_id: str | None capabilities: list[str] writeback_enabled: bool etag: str | None = None @@ -54,6 +58,54 @@ class WritebackIntentResponse(BaseModel): CUSTOMER_OWNED_PROTOCOLS = {"caldav", "carddav", "webdav"} +def _registry_capabilities(source: CalendarWritebackSource) -> list[str]: + capabilities = ["read"] + if source.writeback_enabled: + capabilities.extend(["write", "etag"]) + return capabilities + + +def _writeback_source_from_registry( + registry_source: CalendarWritebackSource, +) -> WritebackSource: + return WritebackSource( + source_id=registry_source.source_uid, + provider=registry_source.provider_name, + protocol=registry_source.source_protocol, + owner_id=registry_source.user_id, + organization_id=registry_source.organization_id, + capabilities=_registry_capabilities(registry_source), + writeback_enabled=bool(registry_source.writeback_enabled), + etag=registry_source.etag_value, + ) + + +def _registry_scope_statement(auth_context: AuthContext): + statement = ( + select(CalendarWritebackSource) + .where(CalendarWritebackSource.source_protocol == "caldav") + .order_by( + CalendarWritebackSource.created_at.asc(), + CalendarWritebackSource.source_uid.asc(), + ) + ) + if is_system_admin_role(auth_context.role): + return statement + if is_tenant_admin_role(auth_context.role) and auth_context.organization_id: + return statement.where( + CalendarWritebackSource.organization_id == auth_context.organization_id + ) + organization_filter = ( + CalendarWritebackSource.organization_id == auth_context.organization_id + if auth_context.organization_id is not None + else CalendarWritebackSource.organization_id.is_(None) + ) + return statement.where( + CalendarWritebackSource.user_id == auth_context.user_id, + organization_filter, + ) + + def _has_writeback_capability(source: WritebackSource) -> bool: return ( source.writeback_enabled @@ -64,16 +116,19 @@ def _has_writeback_capability(source: WritebackSource) -> bool: async def get_writeback_sources( auth_context: AuthContext = Depends(get_auth_context), + db: AsyncSession = Depends(get_db), ) -> tuple[WritebackSource, ...]: """ Return server-authoritative writeback sources for the authenticated user. - The current slice has no persisted connector/source registry yet, so the - production default is intentionally empty. Tests may override this dependency - with fixture-owned sources, and the future connector registry should replace - this placeholder with a database-backed lookup scoped by `auth_context`. + This currently resolves persisted CalDAV source-registry rows only. Provider + mutations are still out of scope: the endpoint returns intent metadata and + ETag/If-Match requirements so later connector execution can fail closed. """ - return () + result = await db.execute(_registry_scope_statement(auth_context)) + return tuple( + _writeback_source_from_registry(source) for source in result.scalars().all() + ) async def get_calendar_user_token( diff --git a/backend/db/models.py b/backend/db/models.py index 802dbe52a..6532b3562 100644 --- a/backend/db/models.py +++ b/backend/db/models.py @@ -376,6 +376,33 @@ class CaldavAccount(Base): ) +class CalendarWritebackSource(Base): + __tablename__ = "calendar_writeback_sources" + + source_uid: Mapped[str] = mapped_column(String, primary_key=True) + user_id: Mapped[str] = mapped_column(String, index=True, nullable=False) + organization_id: Mapped[str | None] = mapped_column(String, index=True, nullable=True) + workspace_id: Mapped[str] = mapped_column(String, index=True, nullable=False) + account_ref: Mapped[str | None] = mapped_column(String, nullable=True) + provider_name: Mapped[str] = mapped_column(String, nullable=False) + source_protocol: Mapped[str] = mapped_column(String, nullable=False) + source_host: Mapped[str] = mapped_column(String, nullable=False) + writeback_enabled: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False) + etag_value: Mapped[str | None] = mapped_column(String, nullable=True) + created_at: Mapped[datetime.datetime] = mapped_column( + DateTime(timezone=True), + default=lambda: datetime.datetime.now(datetime.timezone.utc), + ) + __table_args__ = ( + Index( + "ix_calendar_writeback_sources_scope", + "user_id", + "organization_id", + "source_protocol", + ), + ) + + class ReplyTracker(Base): __tablename__ = "reply_trackers" diff --git a/backend/scripts/bootstrap_db.py b/backend/scripts/bootstrap_db.py index f2eac2d06..e77df7f13 100644 --- a/backend/scripts/bootstrap_db.py +++ b/backend/scripts/bootstrap_db.py @@ -30,6 +30,21 @@ def schema_backfill_sql(): text( "ALTER TABLE llm_providers ADD COLUMN IF NOT EXISTS organization_id varchar" ), + text( + "CREATE TABLE IF NOT EXISTS calendar_writeback_sources (" + "source_uid varchar PRIMARY KEY, " + "user_id varchar NOT NULL, " + "organization_id varchar, " + "workspace_id varchar NOT NULL, " + "account_ref varchar, " + "provider_name varchar NOT NULL, " + "source_protocol varchar NOT NULL, " + "source_host varchar NOT NULL, " + "writeback_enabled boolean NOT NULL DEFAULT false, " + "etag_value varchar, " + "created_at timestamptz DEFAULT CURRENT_TIMESTAMP" + ")" + ), text("ALTER TABLE tenant_configs ADD COLUMN IF NOT EXISTS pop3_username varchar"), text("ALTER TABLE tenant_configs ADD COLUMN IF NOT EXISTS pop3_password varchar"), text("ALTER TABLE emails ADD COLUMN IF NOT EXISTS in_reply_to varchar"), @@ -61,6 +76,11 @@ def schema_backfill_sql(): "CREATE INDEX IF NOT EXISTS ix_llm_providers_organization_id " "ON llm_providers (organization_id)" ), + text( + "CREATE INDEX IF NOT EXISTS ix_calendar_writeback_sources_scope " + "ON calendar_writeback_sources " + "(user_id, organization_id, source_protocol)" + ), text("ALTER TABLE emails DROP CONSTRAINT IF EXISTS emails_message_id_key"), text( "ALTER TABLE sender_relationships " diff --git a/backend/tests/test_bootstrap_db.py b/backend/tests/test_bootstrap_db.py index 88f6a0863..7634581f7 100644 --- a/backend/tests/test_bootstrap_db.py +++ b/backend/tests/test_bootstrap_db.py @@ -1,5 +1,5 @@ from scripts.bootstrap_db import schema_backfill_sql -from db.models import SenderRelationship +from db.models import CalendarWritebackSource, SenderRelationship def test_schema_backfill_adds_threading_columns_for_existing_tables(monkeypatch): @@ -98,6 +98,25 @@ def test_schema_backfill_adds_threading_columns_for_existing_tables(monkeypatch) in statement for statement in statements ) + assert any( + "create table if not exists calendar_writeback_sources" in statement + for statement in statements + ) + assert any("source_uid varchar primary key" in statement for statement in statements) + assert any("workspace_id varchar not null" in statement for statement in statements) + assert any("provider_name varchar not null" in statement for statement in statements) + assert any("source_protocol varchar not null" in statement for statement in statements) + assert any("source_host varchar not null" in statement for statement in statements) + assert any( + "writeback_enabled boolean not null default false" in statement + for statement in statements + ) + assert any("etag_value varchar" in statement for statement in statements) + assert any( + "create index if not exists ix_calendar_writeback_sources_scope" + in statement + for statement in statements + ) assert any( "create index if not exists ix_llm_providers_organization_id" in statement for statement in statements @@ -193,3 +212,23 @@ def test_sender_relationship_model_declares_source_unique_index(): assert "sender_email" in expression_text assert "source_message_id" in expression_text assert "source_thread_id" in expression_text + + +def test_calendar_writeback_source_model_uses_two_word_names(): + assert CalendarWritebackSource.__tablename__ == "calendar_writeback_sources" + column_names = {column.name for column in CalendarWritebackSource.__table__.columns} + + assert column_names == { + "source_uid", + "user_id", + "organization_id", + "workspace_id", + "account_ref", + "provider_name", + "source_protocol", + "source_host", + "writeback_enabled", + "etag_value", + "created_at", + } + assert all("_" in column_name for column_name in column_names) diff --git a/backend/tests/test_calendar_api.py b/backend/tests/test_calendar_api.py index f9181ecb7..c65b26e64 100644 --- a/backend/tests/test_calendar_api.py +++ b/backend/tests/test_calendar_api.py @@ -1,9 +1,20 @@ +import uuid +from unittest.mock import patch, AsyncMock + +import asyncpg +import httpx import pytest from fastapi.testclient import TestClient -from main import app -from unittest.mock import patch, AsyncMock +from sqlalchemy import text +from sqlalchemy.exc import OperationalError +from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine + from api import calendar as calendar_api from api.calendar import WritebackSource +from core.config import settings +from db.models import CalendarWritebackSource +from db.session import get_db +from main import app from services.exceptions import CalendarServiceError pytestmark = pytest.mark.usefixtures("dev_auth_dependency_overrides") @@ -40,6 +51,57 @@ async def token_override() -> dict[str, str]: app.dependency_overrides.pop(calendar_api.get_calendar_user_token, None) +class FakeScalarResult: + def __init__(self, sources: list[CalendarWritebackSource]): + self._sources = sources + + def all(self) -> list[CalendarWritebackSource]: + return self._sources + + +class FakeExecuteResult: + def __init__(self, sources: list[CalendarWritebackSource]): + self._sources = sources + + def scalars(self) -> FakeScalarResult: + return FakeScalarResult(self._sources) + + +class FakeCalendarRegistrySession: + def __init__(self, sources: list[CalendarWritebackSource]): + self.sources = sources + self.statement_text = "" + + async def execute(self, statement): + self.statement_text = str(statement) + return FakeExecuteResult(self.sources) + + +def _calendar_writeback_source( + *, + source_uid: str = "caldav_src_fastmail_primary", + user_id: str = "testuser", + organization_id: str | None = "org-acme", + workspace_id: str = "workspace-org-acme", + provider_name: str = "Fastmail", + source_protocol: str = "caldav", + writeback_enabled: bool = True, + etag_value: str | None = "etag-caldav-1", +) -> CalendarWritebackSource: + return CalendarWritebackSource( + source_uid=source_uid, + user_id=user_id, + organization_id=organization_id, + workspace_id=workspace_id, + account_ref="caldav-account-ref", + provider_name=provider_name, + source_protocol=source_protocol, + source_host="caldav.fastmail.example", + writeback_enabled=writeback_enabled, + etag_value=etag_value, + ) + + @patch("api.calendar.create_calendar_event", new_callable=AsyncMock) def test_calendar_sync_endpoint_success(mock_create, calendar_user_token_override): # Setup mock @@ -551,3 +613,200 @@ def test_calendar_writeback_rejects_same_owner_cross_org_source( assert response.json() == { "detail": "No customer-owned writeback source is available" } + + +def test_calendar_writeback_sources_use_db_backed_caldav_registry(): + fake_session = FakeCalendarRegistrySession( + [ + _calendar_writeback_source( + source_uid="caldav_src_fastmail_primary", + provider_name="Fastmail", + etag_value="etag-db-42", + ) + ] + ) + + async def override_db(): + yield fake_session + + app.dependency_overrides[get_db] = override_db + try: + response = workspace_client.post( + "/api/calendar/writeback-intent", + json={"action": "update", "summary": "Launch review"}, + ) + finally: + app.dependency_overrides.pop(get_db, None) + + assert response.status_code == 200, response.text + body = response.json() + assert body["target_source_id"] == "caldav_src_fastmail_primary" + assert body["protocol"] == "caldav" + assert body["if_match"] == "etag-db-42" + assert body["provenance"]["source_provider"] == "Fastmail" + assert "calendar_writeback_sources.organization_id" in fake_session.statement_text + assert "calendar_writeback_sources.user_id" in fake_session.statement_text + assert "calendar_writeback_sources.source_protocol" in fake_session.statement_text + + +def test_calendar_writeback_db_registry_rejects_cross_org_rows(): + fake_session = FakeCalendarRegistrySession( + [ + _calendar_writeback_source( + source_uid="caldav_src_rival_primary", + organization_id="org-rival", + provider_name="Rival CalDAV", + ) + ] + ) + + async def override_db(): + yield fake_session + + app.dependency_overrides[get_db] = override_db + try: + response = workspace_client.post( + "/api/calendar/writeback-intent", + json={"action": "create", "summary": "Launch review"}, + ) + finally: + app.dependency_overrides.pop(get_db, None) + + assert response.status_code == 422 + assert response.json() == { + "detail": "No customer-owned writeback source is available" + } + + +@pytest.mark.asyncio +@pytest.mark.postgres +async def test_calendar_writeback_intent_real_postgres_smoke(): + source_uid = f"caldav_src_{uuid.uuid4().hex[:24]}" + user_id = f"caldav-smoke-{uuid.uuid4().hex[:12]}" + organization_id = "org-caldav-smoke" + + engine = create_async_engine(settings.DATABASE_URL) + try: + async with engine.begin() as conn: + await conn.execute(text("SELECT 1")) + await conn.execute( + text( + """ + CREATE TABLE IF NOT EXISTS calendar_writeback_sources ( + source_uid VARCHAR PRIMARY KEY, + user_id VARCHAR NOT NULL, + organization_id VARCHAR, + workspace_id VARCHAR NOT NULL, + account_ref VARCHAR, + provider_name VARCHAR NOT NULL, + source_protocol VARCHAR NOT NULL, + source_host VARCHAR NOT NULL, + writeback_enabled BOOLEAN NOT NULL DEFAULT false, + etag_value VARCHAR, + created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP + ) + """ + ) + ) + await conn.execute( + text( + """ + DELETE FROM calendar_writeback_sources + WHERE source_uid = :source_uid + """ + ), + {"source_uid": source_uid}, + ) + await conn.execute( + text( + """ + INSERT INTO calendar_writeback_sources ( + source_uid, + user_id, + organization_id, + workspace_id, + account_ref, + provider_name, + source_protocol, + source_host, + writeback_enabled, + etag_value + ) + VALUES ( + :source_uid, + :user_id, + :organization_id, + :workspace_id, + :account_ref, + :provider_name, + :source_protocol, + :source_host, + :writeback_enabled, + :etag_value + ) + """ + ), + { + "source_uid": source_uid, + "user_id": user_id, + "organization_id": organization_id, + "workspace_id": f"workspace-{organization_id}", + "account_ref": "caldav-smoke-account", + "provider_name": "Smoke CalDAV", + "source_protocol": "caldav", + "source_host": "caldav-smoke.example", + "writeback_enabled": True, + "etag_value": "etag-smoke", + }, + ) + except ( + ConnectionRefusedError, + OSError, + OperationalError, + asyncpg.CannotConnectNowError, + asyncpg.InvalidAuthorizationSpecificationError, + asyncpg.InvalidCatalogNameError, + asyncpg.InvalidPasswordError, + ): + await engine.dispose() + pytest.skip("PostgreSQL smoke path unavailable") + except Exception: + await engine.dispose() + raise + + session_factory = async_sessionmaker(engine, expire_on_commit=False) + + async def override_real_db(): + async with session_factory() as session: + yield session + + app.dependency_overrides[get_db] = override_real_db + try: + transport = httpx.ASGITransport(app=app) + async with httpx.AsyncClient( + transport=transport, + base_url="http://testserver", + headers={"X-User-Id": user_id, "X-Organization-Id": organization_id}, + ) as client: + response = await client.post( + "/api/calendar/writeback-intent", + json={"action": "update", "summary": "Smoke update"}, + ) + finally: + app.dependency_overrides.pop(get_db, None) + async with engine.begin() as conn: + await conn.execute( + text( + "DELETE FROM calendar_writeback_sources " + "WHERE source_uid = :source_uid" + ), + {"source_uid": source_uid}, + ) + await engine.dispose() + + assert response.status_code == 200, response.text + body = response.json() + assert body["target_source_id"] == source_uid + assert body["protocol"] == "caldav" + assert body["if_match"] == "etag-smoke" + assert body["provenance"]["source_provider"] == "Smoke CalDAV" diff --git a/docs/operations/source-of-truth-and-writeback-sovereignty.md b/docs/operations/source-of-truth-and-writeback-sovereignty.md index 8e83d5424..a34ab301d 100644 --- a/docs/operations/source-of-truth-and-writeback-sovereignty.md +++ b/docs/operations/source-of-truth-and-writeback-sovereignty.md @@ -42,11 +42,13 @@ claims. Implemented slices currently cover signed task/mail provenance, dedupe/threading contracts, source-linked ticket tasks, self-sent knowledge capture into idempotent ticket tasks, deterministic sender ontology action hints, generic -WebDAV writeback intent, and self-sent knowledge WebDAV/Notes materialization -intent. Real CalDAV/WebDAV mutation, ETag-aware WebDAV/Notes provider execution -for synthesized knowledge, POP3 runtime sync, durable reply-tracking -notifications, and source-id filtered sender graph views remain episode work and -must preserve this registry boundary. +WebDAV writeback intent, self-sent knowledge WebDAV/Notes materialization +intent, and DB-backed CalDAV intent source selection through opaque +`calendar_writeback_sources.source_uid` rows. Real CalDAV/WebDAV mutation, +ETag-aware WebDAV/Notes provider execution for synthesized knowledge, POP3 +runtime sync, durable reply-tracking notifications, and source-id filtered +sender graph views remain episode work and must preserve this registry +boundary. ## Policy and audit requirements diff --git a/docs/plans/2026-05-19-north-star-gap-closure.md b/docs/plans/2026-05-19-north-star-gap-closure.md index cfebcc3bf..4d77eae6a 100644 --- a/docs/plans/2026-05-19-north-star-gap-closure.md +++ b/docs/plans/2026-05-19-north-star-gap-closure.md @@ -40,6 +40,10 @@ connector, and PR governance is metadata-only. requirement and uses only an explicitly named `STRIX_OPENAI_API_KEY` OpenAI Platform credential. It fails closed instead of routing scanner traffic through GitHub Models, `github.token`, Gemini, GPT-4o, or GPT-4.1. +- [x] CalDAV source registry: `/api/calendar/writeback-intent` now resolves + DB-backed `calendar_writeback_sources` rows with opaque `source_uid` values + instead of exposing sequential CalDAV account ids or accepting browser-supplied + source metadata. ## Verification evidence @@ -92,7 +96,8 @@ connector, and PR governance is metadata-only. - Replace the HMAC bridge with a verified OIDC provider integration while keeping signed, server-verifiable session claims. - Implement provider writes for CalDAV/CardDAV/WebDAV with ETag/If-Match conflict - handling and source-level audit trails. + handling and source-level audit trails after connector execution can enforce + source capability and consent. - Add OpenTelemetry instrumentation and dashboards for connector heartbeat, sync lag, provider throttling, writeback conflicts, and AI action audit events. diff --git a/docs/plans/2026-05-24-architecture-implementation.md b/docs/plans/2026-05-24-architecture-implementation.md index cfa6bec73..240918d4d 100644 --- a/docs/plans/2026-05-24-architecture-implementation.md +++ b/docs/plans/2026-05-24-architecture-implementation.md @@ -35,6 +35,6 @@ **Files:** - Create: `backend/api/dav_sync.py` -- [ ] **Step 1: Add `/api/calendar/writeback-intent` with provenance** -- [ ] **Step 2: Support ETag / If-Match collision checks** +- [x] **Step 1: Add `/api/calendar/writeback-intent` with provenance** +- [x] **Step 2: Support ETag / If-Match collision checks** - [ ] **Step 3: Write DAG/Ontology placeholder processing** diff --git a/docs/plans/2026-05-27-caldav-writeback-source-registry.md b/docs/plans/2026-05-27-caldav-writeback-source-registry.md new file mode 100644 index 000000000..b4fcca9ef --- /dev/null +++ b/docs/plans/2026-05-27-caldav-writeback-source-registry.md @@ -0,0 +1,29 @@ +# CalDAV Writeback Source Registry Slice + +## Goal + +Close the production placeholder in `/api/calendar/writeback-intent` by reading +server-authoritative CalDAV source rows from PostgreSQL while keeping Naruon as a +client/control plane, not a calendar host. + +## Implementation + +- Add `calendar_writeback_sources` with two-word snake-case columns and opaque + `source_uid` values for browser-visible source ids. +- Scope registry lookup by signed `AuthContext`: members see only their owner + scope, tenant admins can target same-organization rows, and system admins can + target any row. +- Return intent metadata only: protocol, provider, ETag/If-Match requirement, + provenance, and audit event. No provider write is executed in this slice. +- Keep Strix on direct OpenAI Platform credentials only; no GitHub Models path is + part of this work. + +## Verification + +```bash +python3 -m pytest backend/tests/test_calendar_api.py backend/tests/test_bootstrap_db.py -q +python3 -m pytest backend/tests/test_calendar_api.py -m postgres -q +``` + +Screenshots stay covered by the existing Calendar writeback E2E because the +browser contract and signed `Authorization: Bearer` path are unchanged.