From 97f8cd755b7aa6b3043fd7a4a7990c9500c68525 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 29 May 2026 20:07:04 +0900 Subject: [PATCH] feat(data): show source linked repository assets --- AGENTS.md | 5 ++ README.md | 9 +- backend/api/data.py | 85 ++++++++++++++++++ backend/tests/test_data_api.py | 78 ++++++++++++++++- ...26-05-29-data-repository-assets-surface.md | 69 +++++++++++++++ frontend/src/app/data/page.test.tsx | 34 ++++++++ frontend/src/components/DataLayout.tsx | 86 ++++++++++++++++++- frontend/tests/e2e/dashboard-branding.spec.ts | 5 ++ frontend/tests/e2e/helpers.ts | 36 +++++++- 9 files changed, 395 insertions(+), 12 deletions(-) create mode 100644 docs/plans/2026-05-29-data-repository-assets-surface.md diff --git a/AGENTS.md b/AGENTS.md index 61037987c..7efa84a79 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -144,6 +144,11 @@ or a narrower signed API. Do not reintroduce static model-score fixtures, fake workflow logs, or provider names that are not derived from prompt, provider, or audit data. +- Data document repository assets must be backed by signed + `/api/data/quality-surface` evidence from scoped email and attachment rows. + Do not reintroduce static file lists, sequential attachment/email ids, raw + message ids, raw thread ids, message bodies, provider URLs, usernames, + credentials, or claims that Naruon itself stores customer file capacity. - Icon-only workspace controls must carry localized `aria-label` text matching the visible app language; do not rely on the SVG icon alone for Calendar, Tasks, drawer, modal, or toolbar actions. diff --git a/README.md b/README.md index e51526fa0..642b288cc 100644 --- a/README.md +++ b/README.md @@ -41,10 +41,11 @@ mail/calendar/file systems. RBAC/ABAC policy engine, and returns no sequential account ids, raw credentials, legacy unscoped audit rows, or fake security posture claims. - Data quality is source-backed through signed `/api/data/quality-surface`. - The endpoint summarizes scoped repositories, ingestion inventory, embedding - coverage, quality checks, and connector evidence from existing rows, returns - `provider_write_executed=false`, and does not expose provider credentials, - raw usernames, server URLs, or sequential ids. + The endpoint summarizes scoped repositories, recent email-attachment file + assets, ingestion inventory, embedding coverage, quality checks, and connector + evidence from existing rows, returns `provider_write_executed=false`, and does + not expose provider credentials, raw usernames, server URLs, message bodies, + raw message/thread ids, or sequential ids. - Projects are source-backed through signed `/api/webdav/folders` and `/api/tasks`. The workspace derives project boundaries from customer-owned WebDAV folders, task progress from opaque public ticket ids, and labels diff --git a/backend/api/data.py b/backend/api/data.py index 48e247c64..7cdce5276 100644 --- a/backend/api/data.py +++ b/backend/api/data.py @@ -1,6 +1,7 @@ from __future__ import annotations from datetime import datetime, timezone +import hashlib from typing import Literal from fastapi import APIRouter, Depends @@ -30,6 +31,7 @@ "no_source", ] QualityStatus = Literal["pass", "needs_attention", "pending"] +RepositoryAssetState = Literal["ready", "needs_attention"] RepositoryType = Literal[ "webdav_account", "project_folder", @@ -48,6 +50,20 @@ class DataRepositorySummary(BaseModel): provider_write_executed: bool +class DataRepositoryAsset(BaseModel): + asset_key: str + asset_type: Literal["email_attachment"] + display_name: str + source_label: str + state_code: RepositoryAssetState + detail_text: str + content_chars: int + captured_at: str + evidence_source: str + thread_key: str + provider_write_executed: bool + + class DataPipelineStage(BaseModel): stage_key: str display_name: str @@ -95,6 +111,7 @@ class DataQualitySurfaceResponse(BaseModel): audit_event: Literal["data.quality_surface.viewed"] provider_write_executed: bool repositories: list[DataRepositorySummary] + repository_assets: list[DataRepositoryAsset] pipeline_stages: list[DataPipelineStage] embedding_collections: list[DataEmbeddingCollection] quality_checks: list[DataQualityCheck] @@ -107,6 +124,32 @@ def _datetime_to_utc_iso(value: datetime) -> str: return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") +def _safe_display_text(value: str | None, fallback: str) -> str: + cleaned = (value or fallback).replace("<", "").replace(">", "").strip() + return " ".join(cleaned.split())[:120] or fallback + + +def _opaque_asset_key(email: Email, attachment: Attachment) -> str: + digest = hashlib.sha256( + "|".join( + [ + email.user_id, + email.organization_id, + email.message_id, + attachment.filename, + ] + ).encode("utf-8") + ).hexdigest() + return f"asset_{digest[:24]}" + + +def _opaque_thread_key(email: Email) -> str: + if not email.thread_id: + return "thread_missing" + digest = hashlib.sha256(email.thread_id.encode("utf-8")).hexdigest() + return f"thread_{digest[:16]}" + + def _can_read_org_scope(auth_context: AuthContext) -> bool: return is_admin_role(auth_context.role) and auth_context.organization_id is not None @@ -249,6 +292,39 @@ def _repository_summaries( return repositories +def _repository_assets(rows) -> list[DataRepositoryAsset]: + assets: list[DataRepositoryAsset] = [] + for attachment, email in rows: + content_chars = len((attachment.content or "").strip()) + has_thread = bool((email.thread_id or "").strip()) + state_code: RepositoryAssetState = ( + "ready" if content_chars > 0 and has_thread else "needs_attention" + ) + detail_parts: list[str] = [] + if content_chars <= 0: + detail_parts.append("content extraction pending") + if not has_thread: + detail_parts.append("canonical thread pending") + if not detail_parts: + detail_parts.append("content and thread evidence ready") + assets.append( + DataRepositoryAsset( + asset_key=_opaque_asset_key(email, attachment), + asset_type="email_attachment", + display_name=_safe_display_text(attachment.filename, "email attachment"), + source_label=_safe_display_text(email.subject, "untitled email"), + state_code=state_code, + detail_text=", ".join(detail_parts), + content_chars=content_chars, + captured_at=_datetime_to_utc_iso(email.date), + evidence_source="attachments.content, emails.thread_id", + thread_key=_opaque_thread_key(email), + provider_write_executed=False, + ) + ) + return assets + + def _pipeline_stages( *, source_count: int, @@ -506,6 +582,14 @@ async def get_data_quality_surface( connector_events: list[ConnectorSignalEvent] = [] if connector_statement is not None: connector_events = await _scoped_rows(db, connector_statement) + attachment_asset_result = await db.execute( + select(Attachment, Email) + .join(Email) + .where(*email_scope) + .order_by(Email.date.desc(), Attachment.filename.asc()) + .limit(8) + ) + attachment_asset_rows = list(attachment_asset_result.all()) source_count = len(webdav_accounts) + len(project_folders) embedded_total = embedded_email_count + embedded_attachment_count @@ -521,6 +605,7 @@ async def get_data_quality_surface( email_count, attachment_count, ), + repository_assets=_repository_assets(attachment_asset_rows), pipeline_stages=_pipeline_stages( source_count=source_count, email_count=email_count, diff --git a/backend/tests/test_data_api.py b/backend/tests/test_data_api.py index 4472f3f21..c75bcdd68 100644 --- a/backend/tests/test_data_api.py +++ b/backend/tests/test_data_api.py @@ -17,7 +17,14 @@ from api.auth import get_auth_context, get_current_user from core.config import settings -from db.models import Base, ConnectorSignalEvent, ProjectFolder, WebdavAccount +from db.models import ( + Attachment, + Base, + ConnectorSignalEvent, + Email, + ProjectFolder, + WebdavAccount, +) from db.session import get_db from main import app @@ -104,6 +111,30 @@ def _webdav_account(source_uid: str) -> WebdavAccount: ) +def _email( + message_id: str, + *, + thread_id: str | None, + subject: str = "Data source package", +) -> Email: + return Email( + user_id="owner", + organization_id="org-acme", + message_id=message_id, + thread_id=thread_id, + fingerprint=f"sha256:{message_id}", + sender="partner@example.com", + recipients="owner@example.com", + subject=subject, + date=_now(), + body="source email body", + ) + + +def _attachment(filename: str, content: str) -> Attachment: + return Attachment(filename=filename, content=content) + + def _project_folder(folder_uid: str) -> ProjectFolder: return ProjectFolder( folder_uid=folder_uid, @@ -129,6 +160,12 @@ def _connector_event(event_uid: str) -> ConnectorSignalEvent: @pytest.fixture def mock_db(): + ready_email = _email("", thread_id="thread-ready") + pending_email = _email( + "", + thread_id=None, + subject="", + ) return MockAsyncSession( [ [_webdav_account("webdav_src_primary")], @@ -141,6 +178,10 @@ def mock_db(): 3, 1, [_connector_event("connector_evt_data_quality")], + [ + (_attachment("roadmap.pdf", "extracted attachment text"), ready_email), + (_attachment("quarterly.md", ""), pending_email), + ], ] ) @@ -204,6 +245,23 @@ def test_data_quality_surface_returns_source_backed_counts_without_secrets(mock_ assert quality_by_key["dedupe_fingerprint"]["issue_count"] == 2 assert quality_by_key["attachment_content"]["issue_count"] == 1 assert data["connector_events"][0]["event_uid"] == "connector_evt_data_quality" + assert data["repository_assets"][0] == { + "asset_key": data["repository_assets"][0]["asset_key"], + "asset_type": "email_attachment", + "display_name": "roadmap.pdf", + "source_label": "Data source package", + "state_code": "ready", + "detail_text": "content and thread evidence ready", + "content_chars": 25, + "captured_at": "2026-05-28T05:45:00Z", + "evidence_source": "attachments.content, emails.thread_id", + "thread_key": data["repository_assets"][0]["thread_key"], + "provider_write_executed": False, + } + assert data["repository_assets"][0]["asset_key"].startswith("asset_") + assert data["repository_assets"][0]["thread_key"].startswith("thread_") + assert data["repository_assets"][1]["state_code"] == "needs_attention" + assert data["repository_assets"][1]["source_label"] == "scriptQuarterly source pack/script" serialized = response.text for forbidden in ( @@ -216,6 +274,8 @@ def test_data_quality_surface_returns_source_backed_counts_without_secrets(mock_ "https://files.acme.example", "webdav_path", "/Projects/Naruon_Roadmap_2026", + "", + "thread-ready", ): assert forbidden not in serialized @@ -269,6 +329,10 @@ def test_member_data_quality_queries_are_owner_scoped(mock_db): @pytest.mark.asyncio @pytest.mark.postgres async def test_data_quality_surface_real_postgres_smoke_uses_signed_scope(): + database_url = getattr(settings, "DATABASE_URL", None) + if not database_url: + pytest.skip("PostgreSQL smoke path unavailable: DATABASE_URL is not set") + user_id = f"data_smoke_user_{uuid.uuid4().hex[:12]}" organization_id = f"data_smoke_org_{uuid.uuid4().hex[:12]}" workspace_id = f"workspace_{organization_id}" @@ -279,7 +343,7 @@ async def test_data_quality_surface_real_postgres_smoke_uses_signed_scope(): folder_uid = f"webdav_folder_data_{uuid.uuid4().hex[:18]}" event_uid = f"connector_evt_data_{uuid.uuid4().hex[:18]}" other_workspace_event_uid = f"connector_evt_other_{uuid.uuid4().hex[:18]}" - engine = create_async_engine(settings.DATABASE_URL, echo=False) + engine = create_async_engine(database_url, echo=False) try: async with engine.begin() as conn: await conn.execute(text("SELECT 1")) @@ -557,6 +621,16 @@ async def override_real_db(): assert quality_by_key["dedupe_fingerprint"]["issue_count"] == 1 assert quality_by_key["attachment_content"]["issue_count"] == 1 assert event_uid in {event["event_uid"] for event in data["connector_events"]} + asset_names = {asset["display_name"] for asset in data["repository_assets"]} + assert {"ready.txt", "blank.txt"} <= asset_names + assert "rival.txt" not in response.text + assets_by_name = { + asset["display_name"]: asset for asset in data["repository_assets"] + } + assert assets_by_name["ready.txt"]["state_code"] == "ready" + assert assets_by_name["blank.txt"]["state_code"] == "needs_attention" + assert assets_by_name["ready.txt"]["asset_key"].startswith("asset_") + assert assets_by_name["ready.txt"]["thread_key"].startswith("thread_") assert other_workspace_event_uid not in response.text assert "account_id" not in response.text assert "encrypted-data-secret" not in response.text diff --git a/docs/plans/2026-05-29-data-repository-assets-surface.md b/docs/plans/2026-05-29-data-repository-assets-surface.md new file mode 100644 index 000000000..3c3f4454c --- /dev/null +++ b/docs/plans/2026-05-29-data-repository-assets-surface.md @@ -0,0 +1,69 @@ +# Data Repository Assets Surface + +## Verified Gap + +- `frontend/branding/naruon-ux-mockup-7.png` shows Data as a file-oriented + workspace: document store, file state, metadata, ingestion, embedding, and + quality panels in one operational surface. +- `docs/plans/2026-05-28-data-quality-surface.md` made repository counts, + ingestion, embeddings, and quality source-backed, but the document repository + tab still lacked source-linked file assets derived from actual mail and + attachment rows. +- The next thin slice should not add Naruon-owned file storage. It should expose + read-only evidence from customer-owned email attachments and keep WebDAV + writeback as intent metadata until connector execution can enforce + capability, consent, and ETag/If-Match. + +## External Best-Practice Inputs Checked + +- OpenLineage documents jobs, datasets, and runs as consistently identified + lineage events. This slice keeps stable opaque asset keys and evidence-source + labels so future ingestion jobs can attach lineage without exposing database + primary keys. Source: https://openlineage.io/docs/ +- Great Expectations treats quality as explicit checks with outcomes. This slice + keeps file asset state tied to concrete checks: extracted attachment content + and canonical thread evidence. Source: + https://docs.greatexpectations.io/ +- WebDAV lost-update protection relies on ETag/If-Match semantics. Data remains + read-only here and continues routing provider writes through existing WebDAV + intent endpoints with If-Match evidence. Source: + https://www.rfc-editor.org/rfc/rfc4918 + +## Implemented Slice + +- Extend signed `GET /api/data/quality-surface` with `repository_assets`. +- Derive each asset from existing `attachments` joined to scoped `emails`. +- Return only browser-safe evidence: + - opaque `asset_key`; + - sanitized attachment filename and source subject; + - opaque thread key or `thread_missing`; + - extracted content character count; + - captured timestamp from the source email; + - state based on attachment content and canonical thread evidence; + - `provider_write_executed=false`. +- Do not expose attachment ids, email ids, raw message ids, raw thread ids, + message body, provider URLs, usernames, credentials, or WebDAV storage claims. +- Render the Data document repository tab with the recent file/attachment asset + list before WebDAV writeback intent controls. + +## Verification Plan + +- Backend mocked tests cover signed-session response shape, opaque identifiers, + safe display text, and secret/private identifier omission. +- Backend PostgreSQL smoke seeds scoped and rival email/attachment rows and + proves only signed-scope assets return. +- Frontend unit tests prove the Data page renders repository assets from + `/api/data/quality-surface` and continues using bearer-session headers without + public identity headers. +- Browser E2E covers desktop repository asset screenshots plus existing + pipeline, embedding, quality, mobile scroll, and hamburger checks. + +## Follow-Up Roadmap + +- Add durable ingestion run and dataset lineage tables only after connector jobs + emit source-backed run events. New DB names must remain two-word + `snake_case`. +- Add WebDAV provider write execution only after connector execution can enforce + source capability, consent, credential reference, remote href, and If-Match. +- Add a file metadata side panel only after source rows carry provider-safe MIME, + size, and classification evidence. diff --git a/frontend/src/app/data/page.test.tsx b/frontend/src/app/data/page.test.tsx index 9977f7bea..8e34263d1 100644 --- a/frontend/src/app/data/page.test.tsx +++ b/frontend/src/app/data/page.test.tsx @@ -57,6 +57,34 @@ const dataQualitySurface = { provider_write_executed: false, }, ], + repository_assets: [ + { + asset_key: "asset_repository_ready", + asset_type: "email_attachment", + display_name: "roadmap.pdf", + source_label: "Q2 roadmap source email", + state_code: "ready", + detail_text: "content and thread evidence ready", + content_chars: 4096, + captured_at: "2026-05-28T05:45:00Z", + evidence_source: "attachments.content, emails.thread_id", + thread_key: "thread_repository_ready", + provider_write_executed: false, + }, + { + asset_key: "asset_repository_pending", + asset_type: "email_attachment", + display_name: "blank-notes.md", + source_label: "Forwarded duplicate source email", + state_code: "needs_attention", + detail_text: "content extraction pending, canonical thread pending", + content_chars: 0, + captured_at: "2026-05-28T05:43:00Z", + evidence_source: "attachments.content, emails.thread_id", + thread_key: "thread_missing", + provider_write_executed: false, + }, + ], pipeline_stages: [ { stage_key: "source_registry", @@ -267,6 +295,11 @@ describe("DataPage", () => { expect(container.textContent).toContain("메일/첨부 저장소"); expect(container.textContent).toContain("data.quality_surface.viewed"); expect(container.textContent).toContain("connector_evt_data_quality"); + expect(container.textContent).toContain("최근 파일/첨부 자산"); + expect(container.textContent).toContain("roadmap.pdf"); + expect(container.textContent).toContain("asset_repository_ready"); + expect(container.textContent).toContain("thread_repository_ready"); + expect(container.textContent).toContain("blank-notes.md"); expect(container.textContent).toContain("WebDAV writeback intent 승인"); expect(container.textContent).toContain("etag=etag-webdav-primary"); expect(container.textContent).toContain("webdav_folder_roadmap"); @@ -310,6 +343,7 @@ describe("DataPage", () => { } expect(container.textContent).not.toContain("28,401"); expect(container.textContent).not.toContain("23건"); + expect(container.textContent).not.toContain(""); }); it("renders API-backed pipeline embedding and quality tabs", async () => { diff --git a/frontend/src/components/DataLayout.tsx b/frontend/src/components/DataLayout.tsx index c41c1b45b..7ec6e6a82 100644 --- a/frontend/src/components/DataLayout.tsx +++ b/frontend/src/components/DataLayout.tsx @@ -1,7 +1,7 @@ "use client"; import { useCallback, useState, useEffect } from 'react'; -import { Database, HardDrive, RefreshCw, FolderOpen, CheckCircle2, Server } from 'lucide-react'; +import { Database, FileText, HardDrive, RefreshCw, FolderOpen, CheckCircle2, Server } from 'lucide-react'; import { apiClient } from '@/lib/api-client'; type WebdavWritebackIntentResponse = { @@ -40,6 +40,7 @@ type DataSurfaceStatus = 'loading' | 'ready' | 'error'; type SurfaceStatusCode = 'ready' | 'running' | 'needs_attention' | 'pending' | 'no_source'; type QualityStatusCode = 'pass' | 'needs_attention' | 'pending'; +type RepositoryAssetState = 'ready' | 'needs_attention'; type DataQualitySurfaceResponse = { workspace_id: string; @@ -55,6 +56,19 @@ type DataQualitySurfaceResponse = { evidence_source: string; provider_write_executed: boolean; }>; + repository_assets: Array<{ + asset_key: string; + asset_type: 'email_attachment'; + display_name: string; + source_label: string; + state_code: RepositoryAssetState; + detail_text: string; + content_chars: number; + captured_at: string; + evidence_source: string; + thread_key: string; + provider_write_executed: boolean; + }>; pipeline_stages: Array<{ stage_key: string; display_name: string; @@ -290,6 +304,7 @@ export function DataLayout() { const attachmentRepository = repositories.find((repository) => repository.repository_type === 'attachment_repository'); const embeddingStage = dataQualitySurface?.pipeline_stages.find((stage) => stage.stage_key === 'embedding_inventory'); const connectorEvents = dataQualitySurface?.connector_events ?? []; + const repositoryAssets = dataQualitySurface?.repository_assets ?? []; return (
@@ -388,7 +403,74 @@ export function DataLayout() { provider_write_executed={String(dataQualitySurface?.provider_write_executed ?? false)}
- + + +
+
+
+

최근 파일/첨부 자산

+

메일 첨부에서 파생된 문서 자산을 원본 메일/thread 근거와 함께 추적합니다.

+
+ + {formatCount(repositoryAssets.length)} assets + +
+
+ {dataSurfaceStatus === 'loading' && ( +

문서 자산 근거를 확인하는 중입니다.

+ )} + {dataSurfaceStatus === 'error' && ( +

문서 자산 근거를 불러오지 못했습니다.

+ )} + {dataSurfaceStatus === 'ready' && repositoryAssets.length === 0 && ( +

이 워크스페이스에 source-linked 첨부 자산이 아직 없습니다.

+ )} + {repositoryAssets.map((asset) => ( +
+
+
+
+ +
+

{asset.display_name}

+

{asset.source_label}

+
+
+
+ + {asset.state_code === 'ready' ? '정상' : '점검 필요'} + +
+
+
+
ASSET_KEY
+
{asset.asset_key}
+
+
+
THREAD
+
{asset.thread_key}
+
+
+
CONTENT
+
{formatCount(asset.content_chars)} chars
+
+
+
CAPTURED
+
{asset.captured_at}
+
+
+
PROVIDER_WRITE
+
{String(asset.provider_write_executed)}
+
+
+
EVIDENCE
+
{asset.evidence_source} · {asset.detail_text}
+
+
+
+ ))} +
+
diff --git a/frontend/tests/e2e/dashboard-branding.spec.ts b/frontend/tests/e2e/dashboard-branding.spec.ts index ac9ac8dcc..ab4c3cdb6 100644 --- a/frontend/tests/e2e/dashboard-branding.spec.ts +++ b/frontend/tests/e2e/dashboard-branding.spec.ts @@ -489,6 +489,11 @@ test('renders Data quality surface across viewports with signed API headers', as await expect(page.getByRole('heading', { name: '데이터와 파일' })).toBeVisible(); await expect(page.getByText('data.quality_surface.viewed')).toBeVisible(); + await expect(page.getByText('최근 파일/첨부 자산')).toBeVisible(); + await expect(page.getByText('roadmap.pdf')).toBeVisible(); + await expect(page.getByText('asset_repository_ready')).toBeVisible(); + await expect(page.getByText('thread_repository_ready')).toBeVisible(); + await page.screenshot({ path: testInfo.outputPath('data-quality-desktop-repository-assets.png'), fullPage: false }); await expect(page.getByText('connector_evt_data_quality')).toBeVisible(); await page.getByRole('button', { name: '수집 파이프라인' }).click(); await expect(page.getByText('4 emails and 3 attachments')).toBeVisible(); diff --git a/frontend/tests/e2e/helpers.ts b/frontend/tests/e2e/helpers.ts index 8a81296bb..eb6524747 100644 --- a/frontend/tests/e2e/helpers.ts +++ b/frontend/tests/e2e/helpers.ts @@ -469,7 +469,7 @@ const dataQualitySurface = { organization_id: 'org-acme', audit_event: 'data.quality_surface.viewed', provider_write_executed: false, - repositories: [ + repositories: [ { source_id: 'email_repository', repository_type: 'email_repository', @@ -505,9 +505,37 @@ const dataQualitySurface = { writeback_enabled: null, evidence_source: 'project_folders', provider_write_executed: false, - }, - ], - pipeline_stages: [ + }, + ], + repository_assets: [ + { + asset_key: 'asset_repository_ready', + asset_type: 'email_attachment', + display_name: 'roadmap.pdf', + source_label: 'Q2 roadmap source email', + state_code: 'ready', + detail_text: 'content and thread evidence ready', + content_chars: 4096, + captured_at: '2026-05-28T05:45:00Z', + evidence_source: 'attachments.content, emails.thread_id', + thread_key: 'thread_repository_ready', + provider_write_executed: false, + }, + { + asset_key: 'asset_repository_pending', + asset_type: 'email_attachment', + display_name: 'blank-notes.md', + source_label: 'Forwarded duplicate source email', + state_code: 'needs_attention', + detail_text: 'content extraction pending, canonical thread pending', + content_chars: 0, + captured_at: '2026-05-28T05:43:00Z', + evidence_source: 'attachments.content, emails.thread_id', + thread_key: 'thread_missing', + provider_write_executed: false, + }, + ], + pipeline_stages: [ { stage_key: 'source_registry', display_name: 'Source registry',