diff --git a/docs/middleware-agent-handoff.md b/docs/middleware-agent-handoff.md index 7011022e..225ab6a7 100644 --- a/docs/middleware-agent-handoff.md +++ b/docs/middleware-agent-handoff.md @@ -752,7 +752,10 @@ Neue REST-Oberfläche `src/routes/datasets.ts`, gemountet unter - `POST /api/v1/datasets` — multipart CSV-Upload (`multer`, ein File pro Request, `MAX_UPLOAD_BYTES` = 25 MB). -- `GET /api/v1/datasets` — Liste der eigenen Datasets. +- `GET /api/v1/datasets` — paginierte Liste der eigenen Datasets + (`limit`/`offset` via zod, 400 bei ungültiger Query; Response + `{ items, totalMatched }` — `totalMatched` fehlt nur, wenn die + Graph-Implementierung das optionale `countDatasets` noch nicht kennt). - `GET /api/v1/datasets/:id` — Schema + Metadaten eines Datasets. - `GET /api/v1/datasets/:id/rows` — paginierte Roh-Zeilen. - `DELETE /api/v1/datasets/:id` — Dataset löschen. @@ -1483,9 +1486,14 @@ migrations/0029_datasets.sql`); pro Dataset genau EIN `Dataset`-Graph-Node (`PluginEntity`, `system='dataset'`) für Recall/Zitation. - **Interface:** `KnowledgeGraph.{ingestDataset,listDatasets,getDataset, - queryDatasetRows,deleteDataset}` (`plugin-api/src/knowledgeGraph.ts`), - implementiert in `@omadia/knowledge-graph-neon` (echtes SQL) UND - `@omadia/knowledge-graph-inmemory` (volle Parität, kein Stub). + queryDatasetRows,deleteDataset}` plus das **optionale** `countDatasets` + (`plugin-api/src/knowledgeGraph.ts`; `listDatasets` nimmt seit #532 auch + `offset` — additiv, plugin-api 1.7.0), implementiert in + `@omadia/knowledge-graph-neon` (echtes SQL) UND + `@omadia/knowledge-graph-inmemory` (volle Parität, kein Stub). Die + extras-Wrapper (captureFiltering/inconsistencyTriggering/mergeTriggering) + reichen `countDatasets` nur durch, wenn der innere Graph es kann — + keine fabrizierten Totals. - **Import:** `POST /api/v1/datasets` (multipart CSV, `src/routes/ datasets.ts`) sowie automatisch bei CSV-Chat-Attachments (`attachmentExtract.ts`'s `isCsvAttachment` branch in `orchestrator.ts`'s @@ -2029,17 +2037,38 @@ gekettet, weil `requires` beim Boot enforced wird): docs-RFC (diese PR) omadia-ui-Orchestrator-Consumer. Details + per-PR-Doc-Pflichten in §15 des RFC. -### Phase 14 — Admin-UI für Dataset-Upload/Schema/Delete (#430 Follow-up) +### Phase 14 — Admin-UI für Dataset-Upload/Schema/Delete (#430 Follow-up) — **erledigt (#532)** -Der #430-Scope (CSV-Import + `query_dataset`-Tool, siehe §3 und §7) deckt -absichtlich **keine** Admin-UI ab — Upload/Schema-Browse/Delete bleibt +Der #430-Scope (CSV-Import + `query_dataset`-Tool, siehe §3 und §7) deckte +absichtlich **keine** Admin-UI ab — Upload/Schema-Browse/Delete blieb API-only (`POST/GET/DELETE /api/v1/datasets*`, siehe §3). #430's eigene -Triage-Acceptance-Criteria verlangen aber genau diese UI; der Branch -schließt das Issue deshalb NICHT, sondern "addresses" es — ein -Folge-Issue für die Admin-UI-Seite (`web-ui/app/admin/datasets/` o.ä., -Upload-Dropzone + Schema-Tabelle + Zeilen-Preview + Delete-Bestätigung, -Pattern analog zur bestehenden Package-Upload-Seite) ist offen zu -erfassen. +Triage-Acceptance-Criteria verlangen aber genau diese UI; das +Folge-Issue #532 hat sie nachgezogen. + +Geliefert, überwiegend `web-ui`-seitig; an der REST-Surface aus §3 gab es +EINE additive Änderung: `GET /api/v1/datasets` paginiert jetzt +(`limit`/`offset`, Response `{ items, totalMatched }`, getragen von +`listDatasets.offset` + optionalem `countDatasets` im plugin-api-Interface, +Minor-Bump auf 1.7.0) — ohne sie waren Datasets jenseits des 50er-Caps im +Admin-UI unsichtbar UND unlöschbar: + +- `web-ui/app/admin/datasets/page.tsx` — Upload (Datei + optionaler Name), + Liste, aufklappbares Detail mit Schema-Tabelle und Zeilen-Vorschau, + Delete mit Bestätigung. +- Der Client (`web-ui/app/_lib/api.ts`) spiegelt `DatasetSummary` / + `DatasetColumnSchema` / den unaggregierten Zweig von + `DatasetQueryResult`, weil `web-ui` nicht gegen den + middleware-Workspace baut. +- Die Zeilen-Vorschau paginiert **server-seitig** über `limit`/`offset` + (25 pro Seite, Server clamped auf [1, 200]). Ein Datensatz fasst bis zu + `MAX_DATASET_ROWS` (50 000) Zeilen — genau der Fall, gegen den + `queryDatasetRows` existiert. +- Nach einem Import werden `privacyScan.scannedCells` / + `maskedCells` und eine etwaige Zell-Truncation angezeigt: der Scan läuft + auf diesem Pfad genauso wie beim Chat-Attachment-Auto-Ingest, und das + soll sichtbar sein statt geglaubt werden zu müssen. +- ACL unverändert owner-only: die Seite zeigt ausschließlich Datensätze + des eingeloggten Kontos, nicht die der Instanz. --- diff --git a/middleware/package-lock.json b/middleware/package-lock.json index fb6432a1..3a2a494b 100644 --- a/middleware/package-lock.json +++ b/middleware/package-lock.json @@ -9525,7 +9525,7 @@ }, "packages/plugin-api": { "name": "@omadia/plugin-api", - "version": "1.6.0", + "version": "1.7.0", "license": "MIT", "engines": { "node": ">=20" diff --git a/middleware/packages/harness-knowledge-graph-inmemory/src/inMemoryKnowledgeGraph.ts b/middleware/packages/harness-knowledge-graph-inmemory/src/inMemoryKnowledgeGraph.ts index 2d241fdd..0dea13b8 100644 --- a/middleware/packages/harness-knowledge-graph-inmemory/src/inMemoryKnowledgeGraph.ts +++ b/middleware/packages/harness-knowledge-graph-inmemory/src/inMemoryKnowledgeGraph.ts @@ -2997,15 +2997,23 @@ export class InMemoryKnowledgeGraph implements KnowledgeGraph { async listDatasets(opts: { ownerOmadiaUserId: string; limit?: number; + offset?: number; }): Promise { const limit = Math.max(1, Math.min(opts.limit ?? 50, 200)); + const offset = Math.max(0, opts.offset ?? 0); return [...this.datasets.values()] .filter((d) => d.ownerOmadiaUserId === opts.ownerOmadiaUserId) .sort((a, b) => b.createdAt.localeCompare(a.createdAt)) - .slice(0, limit) + .slice(offset, offset + limit) .map((d) => this.datasetToSummary(d)); } + async countDatasets(opts: { ownerOmadiaUserId: string }): Promise { + return [...this.datasets.values()].filter( + (d) => d.ownerOmadiaUserId === opts.ownerOmadiaUserId, + ).length; + } + async getDataset( datasetId: string, viewerOmadiaUserId: string, diff --git a/middleware/packages/harness-knowledge-graph-neon/src/neonKnowledgeGraph.ts b/middleware/packages/harness-knowledge-graph-neon/src/neonKnowledgeGraph.ts index 02301bc2..58f7122f 100644 --- a/middleware/packages/harness-knowledge-graph-neon/src/neonKnowledgeGraph.ts +++ b/middleware/packages/harness-knowledge-graph-neon/src/neonKnowledgeGraph.ts @@ -703,19 +703,31 @@ export class NeonKnowledgeGraph implements KnowledgeGraph { async listDatasets(opts: { ownerOmadiaUserId: string; limit?: number; + offset?: number; }): Promise { const limit = Math.max(1, Math.min(opts.limit ?? 50, 200)); + const offset = Math.max(0, opts.offset ?? 0); const result = await this.pool.query( `SELECT id, name, source_file_name, owner_omadia_user_id, row_count, columns, created_at FROM datasets WHERE tenant_id = $1 AND owner_omadia_user_id = $2 ORDER BY created_at DESC - LIMIT $3`, - [this.tenantId, opts.ownerOmadiaUserId, limit], + LIMIT $3 OFFSET $4`, + [this.tenantId, opts.ownerOmadiaUserId, limit, offset], ); return result.rows.map((r) => this.datasetRowToSummary(r)); } + async countDatasets(opts: { ownerOmadiaUserId: string }): Promise { + const result = await this.pool.query<{ count: string }>( + `SELECT COUNT(*) AS count + FROM datasets + WHERE tenant_id = $1 AND owner_omadia_user_id = $2`, + [this.tenantId, opts.ownerOmadiaUserId], + ); + return Number(result.rows[0]?.count ?? 0); + } + async getDataset( datasetId: string, viewerOmadiaUserId: string, diff --git a/middleware/packages/harness-orchestrator-extras/src/captureFilteringKnowledgeGraph.ts b/middleware/packages/harness-orchestrator-extras/src/captureFilteringKnowledgeGraph.ts index 2fa2bee2..4424d2b9 100644 --- a/middleware/packages/harness-orchestrator-extras/src/captureFilteringKnowledgeGraph.ts +++ b/middleware/packages/harness-orchestrator-extras/src/captureFilteringKnowledgeGraph.ts @@ -111,10 +111,20 @@ export class CaptureFilteringKnowledgeGraph implements KnowledgeGraph { private readonly filter: CaptureFilter; private readonly log: (msg: string) => void; + /** + * Optional on the interface (plugin-api back-compat). Mirrors the inner + * graph's support instead of fabricating a capped count: the route treats + * absence as "backend cannot count" and the UI then warns instead of + * rendering a confidently wrong total. + */ + readonly countDatasets?: (opts: { ownerOmadiaUserId: string }) => Promise; + constructor(opts: CaptureFilteringKnowledgeGraphOptions) { this.inner = opts.inner; this.filter = opts.filter; this.log = opts.log ?? ((msg): void => console.error(msg)); + const innerCount = opts.inner.countDatasets?.bind(opts.inner); + if (innerCount) this.countDatasets = innerCount; } async ingestTurn(turn: TurnIngest): Promise { @@ -616,6 +626,7 @@ export class CaptureFilteringKnowledgeGraph implements KnowledgeGraph { listDatasets(opts: { ownerOmadiaUserId: string; limit?: number; + offset?: number; }): Promise { return this.inner.listDatasets(opts); } diff --git a/middleware/packages/harness-orchestrator-extras/src/inconsistencyTriggeringKnowledgeGraph.ts b/middleware/packages/harness-orchestrator-extras/src/inconsistencyTriggeringKnowledgeGraph.ts index 8e406840..97dd7cf5 100644 --- a/middleware/packages/harness-orchestrator-extras/src/inconsistencyTriggeringKnowledgeGraph.ts +++ b/middleware/packages/harness-orchestrator-extras/src/inconsistencyTriggeringKnowledgeGraph.ts @@ -95,10 +95,20 @@ export class InconsistencyTriggeringKnowledgeGraph implements KnowledgeGraph { private readonly detector: InconsistencyDetectorService; private readonly log: (msg: string) => void; + /** + * Optional on the interface (plugin-api back-compat). Mirrors the inner + * graph's support instead of fabricating a capped count: the route treats + * absence as "backend cannot count" and the UI then warns instead of + * rendering a confidently wrong total. + */ + readonly countDatasets?: (opts: { ownerOmadiaUserId: string }) => Promise; + constructor(opts: InconsistencyTriggeringKnowledgeGraphOptions) { this.inner = opts.inner; this.detector = opts.detector; this.log = opts.log ?? ((msg: string): void => { console.error(msg); }); + const innerCount = opts.inner.countDatasets?.bind(opts.inner); + if (innerCount) this.countDatasets = innerCount; } private fire(mkId: string): void { @@ -542,6 +552,7 @@ export class InconsistencyTriggeringKnowledgeGraph implements KnowledgeGraph { listDatasets(opts: { ownerOmadiaUserId: string; limit?: number; + offset?: number; }): Promise { return this.inner.listDatasets(opts); } diff --git a/middleware/packages/harness-orchestrator-extras/src/mergeTriggeringKnowledgeGraph.ts b/middleware/packages/harness-orchestrator-extras/src/mergeTriggeringKnowledgeGraph.ts index 57218523..5a6a6c6d 100644 --- a/middleware/packages/harness-orchestrator-extras/src/mergeTriggeringKnowledgeGraph.ts +++ b/middleware/packages/harness-orchestrator-extras/src/mergeTriggeringKnowledgeGraph.ts @@ -100,10 +100,20 @@ export class MergeTriggeringKnowledgeGraph implements KnowledgeGraph { private readonly detector: MergeCandidateDetectorService; private readonly log: (msg: string) => void; + /** + * Optional on the interface (plugin-api back-compat). Mirrors the inner + * graph's support instead of fabricating a capped count: the route treats + * absence as "backend cannot count" and the UI then warns instead of + * rendering a confidently wrong total. + */ + readonly countDatasets?: (opts: { ownerOmadiaUserId: string }) => Promise; + constructor(opts: MergeTriggeringKnowledgeGraphOptions) { this.inner = opts.inner; this.detector = opts.detector; this.log = opts.log ?? ((msg: string): void => { console.error(msg); }); + const innerCount = opts.inner.countDatasets?.bind(opts.inner); + if (innerCount) this.countDatasets = innerCount; } private fire(mkId: string): void { @@ -597,6 +607,7 @@ export class MergeTriggeringKnowledgeGraph implements KnowledgeGraph { listDatasets(opts: { ownerOmadiaUserId: string; limit?: number; + offset?: number; }): Promise { return this.inner.listDatasets(opts); } diff --git a/middleware/packages/plugin-api/api-snapshot/plugin-api.d.ts.snap b/middleware/packages/plugin-api/api-snapshot/plugin-api.d.ts.snap index bc3696d3..4c0482ec 100644 --- a/middleware/packages/plugin-api/api-snapshot/plugin-api.d.ts.snap +++ b/middleware/packages/plugin-api/api-snapshot/plugin-api.d.ts.snap @@ -531,7 +531,11 @@ ingestDataset(input: DatasetIngest): Promise; listDatasets(opts: { ownerOmadiaUserId: string; limit?: number; +offset?: number; }): Promise; +countDatasets?(opts: { +ownerOmadiaUserId: string; +}): Promise; getDataset(datasetId: string, viewerOmadiaUserId: string): Promise; queryDatasetRows(datasetId: string, viewerOmadiaUserId: string, opts?: DatasetQueryOptions): Promise; deleteDataset(datasetId: string, actor: AclMutationOptions): Promise; diff --git a/middleware/packages/plugin-api/src/knowledgeGraph.ts b/middleware/packages/plugin-api/src/knowledgeGraph.ts index 7e8db060..77e5bff5 100644 --- a/middleware/packages/plugin-api/src/knowledgeGraph.ts +++ b/middleware/packages/plugin-api/src/knowledgeGraph.ts @@ -711,11 +711,25 @@ export interface KnowledgeGraph { * KnowledgeGraph boundary, never inside it. */ ingestDataset(input: DatasetIngest): Promise; - /** #430 — list datasets owned by the caller, most-recent first. */ + /** + * #430 — list datasets owned by the caller, most-recent first. + * `limit` is clamped by implementations (default 50, max 200); `offset` + * skips that many newest datasets so the admin UI can page past the cap + * (#532 review: without it, datasets beyond the cap were invisible AND + * undeletable). + */ listDatasets(opts: { ownerOmadiaUserId: string; limit?: number; + offset?: number; }): Promise; + /** + * #532 — total number of datasets owned by the caller, so list surfaces + * can render real pagination ("showing N of M") instead of a silently + * truncated page. Optional for plugin-api back-compat: implementations + * that predate it keep working, and callers fall back to the page length. + */ + countDatasets?(opts: { ownerOmadiaUserId: string }): Promise; /** * #430 — read one dataset's metadata + inferred schema. Null when * missing or the viewer doesn't own it (ACL mirrors `/api/v1/memory`: diff --git a/middleware/src/routes/datasets.ts b/middleware/src/routes/datasets.ts index 1786d909..4c62a3a7 100644 --- a/middleware/src/routes/datasets.ts +++ b/middleware/src/routes/datasets.ts @@ -29,6 +29,15 @@ const RowsQuerySchema = z.object({ offset: z.coerce.number().int().min(0).optional(), }); +// Same page shape as the rows endpoint, but its own schema on purpose: a +// future change to the rows bounds must not silently change list validation +// (#532 review must-fix 1: without limit/offset the route served only the +// newest 50 datasets). +const ListQuerySchema = z.object({ + limit: z.coerce.number().int().min(1).max(200).optional(), + offset: z.coerce.number().int().min(0).optional(), +}); + function requireSessionUserId(req: Request, res: Response): string | null { const id = req.session?.omadia_user_id; if (!id) { @@ -130,13 +139,29 @@ export function createDatasetsRouter(deps: { graph: KnowledgeGraph }): Router { }, ); - // ── GET / — list current user's datasets ──────────────────────────────── + // ── GET / — list current user's datasets (paginated) ──────────────────── router.get('/', async (req: Request, res: Response) => { const sessionUserId = requireSessionUserId(req, res); if (!sessionUserId) return; + const parsed = ListQuerySchema.safeParse(req.query); + if (!parsed.success) { + res.status(400).json({ code: 'dataset.invalid_query', issues: parsed.error.issues }); + return; + } try { - const items = await deps.graph.listDatasets({ ownerOmadiaUserId: sessionUserId }); - res.json({ items }); + const [items, totalMatched] = await Promise.all([ + deps.graph.listDatasets({ + ownerOmadiaUserId: sessionUserId, + ...(parsed.data.limit !== undefined ? { limit: parsed.data.limit } : {}), + ...(parsed.data.offset !== undefined ? { offset: parsed.data.offset } : {}), + }), + deps.graph.countDatasets + ? deps.graph.countDatasets({ ownerOmadiaUserId: sessionUserId }) + : Promise.resolve(undefined), + ]); + // `totalMatched` mirrors GET /:id/rows; absent only when the graph + // implementation predates the optional `countDatasets`. + res.json({ items, ...(totalMatched !== undefined ? { totalMatched } : {}) }); } catch (err) { const { status, code, message } = mapErrorToHttp(err); res.status(status).json({ code, message }); diff --git a/middleware/test/datasetsRoute.test.ts b/middleware/test/datasetsRoute.test.ts index d1b95a9c..7321b5b7 100644 --- a/middleware/test/datasetsRoute.test.ts +++ b/middleware/test/datasetsRoute.test.ts @@ -137,6 +137,45 @@ describe('POST /api/v1/datasets', () => { await throwing.close(); }); + it('paginates the list: limit/offset pass through and totalMatched is returned (#532)', async () => { + const graph = new InMemoryKnowledgeGraph(); + const paged = await makeHarness('user-1', graph); + for (let i = 0; i < 5; i++) { + await graph.ingestDataset({ + name: `ds-${String(i)}`, + sourceFileName: `ds-${String(i)}.csv`, + ownerOmadiaUserId: 'user-1', + columns: [{ name: 'a', type: 'string' }], + rows: [{ a: 'x' }], + }); + } + + const page = (await (await fetch(`${paged.baseUrl}?limit=2`)).json()) as { + items: unknown[]; + totalMatched: number; + }; + assert.equal(page.items.length, 2); + assert.equal(page.totalMatched, 5); + + const lastPage = (await (await fetch(`${paged.baseUrl}?limit=2&offset=4`)).json()) as { + items: unknown[]; + totalMatched: number; + }; + assert.equal(lastPage.items.length, 1); + assert.equal(lastPage.totalMatched, 5); + + await paged.close(); + }); + + it('400s on an invalid list query instead of silently ignoring it', async () => { + for (const qs of ['?limit=0', '?limit=201', '?offset=-1', '?limit=abc']) { + const res = await fetch(`${h.baseUrl}${qs}`); + assert.equal(res.status, 400, qs); + const body = (await res.json()) as { code: string }; + assert.equal(body.code, 'dataset.invalid_query', qs); + } + }); + it('scopes datasets per owner — a different session cannot see or delete them', async () => { const form = new FormData(); form.append('file', new Blob([CSV], { type: 'text/csv' }), 'people.csv'); diff --git a/middleware/test/inMemoryKnowledgeGraph.test.ts b/middleware/test/inMemoryKnowledgeGraph.test.ts index 0f0cd2ca..3ed3df51 100644 --- a/middleware/test/inMemoryKnowledgeGraph.test.ts +++ b/middleware/test/inMemoryKnowledgeGraph.test.ts @@ -172,6 +172,41 @@ describe('InMemoryKnowledgeGraph — datasets (#430)', () => { assert.equal(fetched?.rowCount, 2); }); + it('pages listDatasets with offset and counts all owned datasets (#532)', async () => { + const g = new InMemoryKnowledgeGraph(); + for (let i = 0; i < 5; i++) { + await g.ingestDataset({ + ownerOmadiaUserId: 'user-1', + name: `ds-${String(i)}`, + sourceFileName: `ds-${String(i)}.csv`, + columns: [{ name: 'v', type: 'number' }], + rows: [{ v: i }], + }); + } + await g.ingestDataset({ + ownerOmadiaUserId: 'user-2', + name: 'other', + sourceFileName: 'other.csv', + columns: [{ name: 'v', type: 'number' }], + rows: [{ v: 0 }], + }); + + const firstPage = await g.listDatasets({ ownerOmadiaUserId: 'user-1', limit: 2 }); + assert.equal(firstPage.length, 2); + const lastPage = await g.listDatasets({ ownerOmadiaUserId: 'user-1', limit: 2, offset: 4 }); + assert.equal(lastPage.length, 1); + // Pages are disjoint slices of the same ordering. + const all = await g.listDatasets({ ownerOmadiaUserId: 'user-1' }); + assert.deepEqual( + [...firstPage, ...(await g.listDatasets({ ownerOmadiaUserId: 'user-1', limit: 2, offset: 2 })), ...lastPage].map((d) => d.id), + all.map((d) => d.id), + ); + + assert.equal(await g.countDatasets({ ownerOmadiaUserId: 'user-1' }), 5); + assert.equal(await g.countDatasets({ ownerOmadiaUserId: 'user-2' }), 1); + assert.equal(await g.countDatasets({ ownerOmadiaUserId: 'nobody' }), 0); + }); + it('hides a dataset from a non-owner (getDataset/listDatasets/queryDatasetRows all return null/empty)', async () => { const g = new InMemoryKnowledgeGraph(); const result = await g.ingestDataset({ diff --git a/web-ui/app/_lib/api.ts b/web-ui/app/_lib/api.ts index 8aa68299..55e85f8d 100644 --- a/web-ui/app/_lib/api.ts +++ b/web-ui/app/_lib/api.ts @@ -3523,6 +3523,156 @@ export async function reclusterTopics(opts: { return postJson('/v1/admin/topics/recluster', opts); } +// ----------------------------------------------------------------------------- +// Structured datasets (#430 / #532). Backed by /api/v1/datasets, gated by +// `requireAuth` plus a per-route session-user ACL — the same shape as +// /api/v1/memory above, so a dataset is only ever visible to its importer. +// +// The types mirror `DatasetSummary` / `DatasetColumnSchema` / `DatasetQueryResult` +// in `@omadia/plugin-api` rather than importing them: web-ui does not depend on +// the middleware workspace, so every backend shape it consumes is re-declared +// here (same convention as `MemorableKnowledgeNode` above). +// ----------------------------------------------------------------------------- + +/** Inferred per-column type from the CSV import. No `unknown` catch-all — a + * column with no confidently-typed values falls back to `'string'`. */ +export type DatasetColumnType = 'string' | 'number' | 'boolean' | 'date'; + +export interface DatasetColumnSchema { + name: string; + type: DatasetColumnType; + /** First non-empty value seen for this column — schema preview only. */ + sample?: string; +} + +export interface DatasetSummary { + id: string; + name: string; + sourceFileName: string; + ownerOmadiaUserId: string; + rowCount: number; + columns: DatasetColumnSchema[]; + createdAt: string; +} + +/** + * `GET /:id/rows` — the unaggregated branch of `queryDatasetRows`, so `rows` + * is always populated and `groups` / `aggregateValue` never are. `totalMatched` + * is the count BEFORE limit/offset, which is what lets the preview say + * "showing 25 of 4,213" instead of silently truncating. + */ +export interface DatasetRowsPage { + rows?: Array>; + totalMatched: number; +} + +/** + * `POST /` — the import receipt. `privacyScan` is the count of cells the + * privacy pipeline looked at and masked; surfacing it is the point, since the + * REST upload runs the SAME scan as the chat-attachment auto-ingest path. + * `truncation` reports cells cut at the 4 000-char per-cell cap. + */ +export interface DatasetUploadResponse { + dataset: { datasetId: string; rowCount: number; graphNodeId: string }; + privacyScan: { scannedCells: number; maskedCells: number }; + truncation: { truncatedCellCount: number; truncatedColumns: string[] }; +} + +/** + * `GET /` — one page of the caller's datasets. `totalMatched` mirrors the + * rows endpoint (count before limit/offset); it is absent only when the + * middleware's graph implementation predates the optional `countDatasets` + * (#532 pagination follow-up to #430). + */ +export interface DatasetListPage { + items: DatasetSummary[]; + totalMatched?: number; +} + +export async function listDatasets( + opts: { limit?: number; offset?: number } = {}, +): Promise { + const qs = new URLSearchParams(); + if (opts.limit !== undefined) qs.set('limit', String(opts.limit)); + if (opts.offset !== undefined) qs.set('offset', String(opts.offset)); + const suffix = qs.toString() ? `?${qs.toString()}` : ''; + const res = await getJson<{ items: DatasetSummary[]; totalMatched?: number }>( + `/v1/datasets${suffix}`, + ); + return { + items: expectArray(res.items, '/v1/datasets', 'items'), + ...(typeof res.totalMatched === 'number' + ? { totalMatched: res.totalMatched } + : {}), + }; +} + +export async function getDataset(id: string): Promise { + return getJson(`/v1/datasets/${encodeURIComponent(id)}`); +} + +export async function getDatasetRows( + id: string, + opts: { limit?: number; offset?: number } = {}, +): Promise { + const qs = new URLSearchParams(); + if (opts.limit !== undefined) qs.set('limit', String(opts.limit)); + if (opts.offset !== undefined) qs.set('offset', String(opts.offset)); + const suffix = qs.toString() ? `?${qs.toString()}` : ''; + return getJson( + `/v1/datasets/${encodeURIComponent(id)}/rows${suffix}`, + ); +} + +export async function deleteDataset(id: string): Promise { + const forwarded = await forwardCookieHeader(); + const res = await fetch(botApi(`/v1/datasets/${encodeURIComponent(id)}`), { + method: 'DELETE', + headers: { accept: 'application/json', ...forwarded }, + credentials: 'include', + cache: 'no-store', + }); + if (res.status === 204) return; + const text = await res.text().catch(() => ''); + maybeNavigateToLogin(res.status); + throw new ApiError( + res.status, + `DELETE datasets/${id} failed: ${res.status}`, + text, + ); +} + +/** + * Multipart CSV upload. Same contract as `uploadPackage` above — exactly one + * `file` field and NO manual content-type, so the browser writes the boundary. + * `name` is optional; the route falls back to the file name when it is absent + * or blank. + */ +export async function uploadDataset( + file: File, + name?: string, +): Promise { + const forwarded = await forwardCookieHeader(); + const form = new FormData(); + form.append('file', file, file.name); + if (name !== undefined && name.trim().length > 0) { + form.append('name', name.trim()); + } + const res = await fetch(botApi('/v1/datasets'), { + method: 'POST', + body: form, + headers: { accept: 'application/json', ...forwarded }, + credentials: 'include', + cache: 'no-store', + }); + const text = await res.text(); + if (!res.ok) { + maybeNavigateToLogin(res.status); + throw new ApiError(res.status, `POST /v1/datasets failed: ${res.status}`, text); + } + return JSON.parse(text) as DatasetUploadResponse; +} + // ----------------------------------------------------------------------------- // Chat session reset (2026-05-26). // ----------------------------------------------------------------------------- diff --git a/web-ui/app/admin/datasets/__tests__/page.test.tsx b/web-ui/app/admin/datasets/__tests__/page.test.tsx new file mode 100644 index 00000000..4465dd7f --- /dev/null +++ b/web-ui/app/admin/datasets/__tests__/page.test.tsx @@ -0,0 +1,483 @@ +import { screen, waitFor } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { renderWithIntl } from '../../../_lib/test-utils'; +import AdminDatasetsPage from '../page'; + +/** + * Coverage for /admin/datasets (#532): + * - lists datasets with their row/column counts, + * - pages the row preview SERVER-side (limit/offset), never client-side — + * a dataset holds up to 50 000 rows, so a "fetch all, slice locally" + * regression is the failure this page exists to avoid, + * - surfaces the privacy-scan counts on a successful import, + * - refuses to delete without a confirmation, + * - maps the route's `dataset.*` error codes to localized copy rather than + * rendering the middleware's English `message` as the headline. + */ + +// Hoisted together with the mock factory: `vi.mock` is lifted above the +// imports, so the stand-in ApiError has to exist by then too — the page's +// `err instanceof ApiError` branch is exactly what these tests exercise. +const { + MockApiError, + mockListDatasets, + mockGetDataset, + mockGetDatasetRows, + mockDeleteDataset, + mockUploadDataset, +} = vi.hoisted(() => ({ + MockApiError: class MockApiError extends Error { + public readonly code: string | null; + constructor( + public status: number, + message: string, + public body = '', + ) { + super(message); + try { + const parsed = JSON.parse(body) as { code?: unknown }; + this.code = typeof parsed.code === 'string' ? parsed.code : null; + } catch { + this.code = null; + } + } + }, + mockListDatasets: vi.fn(), + mockGetDataset: vi.fn(), + mockGetDatasetRows: vi.fn(), + mockDeleteDataset: vi.fn(), + mockUploadDataset: vi.fn(), +})); + +vi.mock('../../../_lib/api', () => ({ + ApiError: MockApiError, + listDatasets: mockListDatasets, + getDataset: mockGetDataset, + getDatasetRows: mockGetDatasetRows, + deleteDataset: mockDeleteDataset, + uploadDataset: mockUploadDataset, +})); + +const DATASET = { + id: 'ds-1', + name: 'Q3 orders', + sourceFileName: 'orders-q3.csv', + ownerOmadiaUserId: 'user-1', + rowCount: 4213, + columns: [ + { name: 'order_ref', type: 'string' as const, sample: 'SO-1001' }, + { name: 'amount', type: 'number' as const, sample: '19.99' }, + ], + createdAt: '2026-08-01T09:30:00.000Z', +}; + +/** Row refs start at SO-2000 so they never collide with the schema's own + * `sample` (SO-1001) — the two tables must be told apart in assertions. */ +function rowPage(offset: number) { + return { + rows: Array.from({ length: 25 }, (_, i) => ({ + order_ref: `SO-${String(2000 + offset + i)}`, + amount: 19.99, + })), + totalMatched: 4213, + }; +} + +beforeEach(() => { + mockListDatasets.mockResolvedValue({ items: [DATASET], totalMatched: 1 }); + mockGetDataset.mockResolvedValue(DATASET); + mockGetDatasetRows.mockImplementation((_id: string, opts: { offset?: number }) => + Promise.resolve(rowPage(opts.offset ?? 0)), + ); + mockDeleteDataset.mockResolvedValue(undefined); +}); + +afterEach(() => { + vi.clearAllMocks(); + vi.unstubAllGlobals(); +}); + +describe('', () => { + it('lists datasets with row and column counts', async () => { + renderWithIntl(); + + expect(await screen.findByText('Q3 orders')).toBeTruthy(); + expect(screen.getByText('orders-q3.csv')).toBeTruthy(); + expect(screen.getByText(/4,213 rows · 2 columns/)).toBeTruthy(); + }); + + it('renders the empty state when the account has imported nothing', async () => { + mockListDatasets.mockResolvedValue({ items: [], totalMatched: 0 }); + renderWithIntl(); + + expect(await screen.findByText('No datasets imported yet.')).toBeTruthy(); + }); + + it('loads the schema and the first row page only when details are opened', async () => { + const user = userEvent.setup(); + renderWithIntl(); + + await screen.findByText('Q3 orders'); + // Nothing is fetched for a collapsed row — the list alone must not cost + // one detail + one rows request per dataset. + expect(mockGetDatasetRows).not.toHaveBeenCalled(); + + await user.click(screen.getByRole('button', { name: 'Schema & rows' })); + + await waitFor(() => + expect(mockGetDatasetRows).toHaveBeenCalledWith('ds-1', { + limit: 25, + offset: 0, + }), + ); + // Twice on purpose: once as a schema row, once as a preview-table header. + expect(await screen.findAllByText('order_ref')).toHaveLength(2); + // The type the importer derived, not the raw CSV string it came from. + expect(screen.getByText('number')).toBeTruthy(); + // Schema sample value… + expect(screen.getByText('SO-1001')).toBeTruthy(); + // …and a real row from the preview page. + expect(screen.getByText('SO-2000')).toBeTruthy(); + expect(screen.getByText('Showing 1–25 of 4,213 rows')).toBeTruthy(); + }); + + it('pages the preview server-side with a new offset', async () => { + const user = userEvent.setup(); + renderWithIntl(); + + await screen.findByText('Q3 orders'); + await user.click(screen.getByRole('button', { name: 'Schema & rows' })); + await screen.findByText('Showing 1–25 of 4,213 rows'); + + // "Back" is unreachable on the first page. + expect( + (screen.getByRole('button', { name: 'Back' }) as HTMLButtonElement).disabled, + ).toBe(true); + + await user.click(screen.getByRole('button', { name: 'Next' })); + + await waitFor(() => + expect(mockGetDatasetRows).toHaveBeenLastCalledWith('ds-1', { + limit: 25, + offset: 25, + }), + ); + expect(await screen.findByText('Showing 26–50 of 4,213 rows')).toBeTruthy(); + // The schema is read once, on open — turning a page costs one request, + // not two. + expect(mockGetDataset).toHaveBeenCalledTimes(1); + }); + + it('pages the dataset LIST server-side and shows the real total (#532 must-fix 1)', async () => { + // 60 datasets — more than one page. The pre-pagination page served the + // newest 50 with no hint that 10 more existed and no way to delete them. + const user = userEvent.setup(); + const all = Array.from({ length: 60 }, (_, i) => ({ + ...DATASET, + id: `ds-${String(i)}`, + name: `Dataset ${String(i)}`, + })); + mockListDatasets.mockImplementation( + (opts: { limit?: number; offset?: number } = {}) => { + const offset = opts.offset ?? 0; + return Promise.resolve({ + items: all.slice(offset, offset + (opts.limit ?? 50)), + totalMatched: 60, + }); + }, + ); + renderWithIntl(); + + expect(await screen.findByText('Showing 1–50 of 60 datasets')).toBeTruthy(); + expect(mockListDatasets).toHaveBeenCalledWith({ limit: 50, offset: 0 }); + expect( + (screen.getByRole('button', { name: 'Back' }) as HTMLButtonElement).disabled, + ).toBe(true); + + await user.click(screen.getByRole('button', { name: 'Next' })); + + expect(await screen.findByText('Showing 51–60 of 60 datasets')).toBeTruthy(); + expect(mockListDatasets).toHaveBeenLastCalledWith({ limit: 50, offset: 50 }); + // The formerly unreachable 51st dataset is now listed — and deletable. + expect(screen.getByText('Dataset 50')).toBeTruthy(); + expect( + (screen.getByRole('button', { name: 'Next' }) as HTMLButtonElement).disabled, + ).toBe(true); + }); + + it('hides the pager when one page holds everything', async () => { + renderWithIntl(); + + await screen.findByText('Q3 orders'); + expect(screen.queryByText(/of 1 datasets/)).toBeNull(); + expect(screen.queryByRole('button', { name: 'Next' })).toBeNull(); + }); + + it('steps back a page when the last dataset of the last page is deleted', async () => { + const user = userEvent.setup(); + vi.stubGlobal('confirm', vi.fn().mockReturnValue(true)); + let deleted = false; + const all = Array.from({ length: 51 }, (_, i) => ({ + ...DATASET, + id: `ds-${String(i)}`, + name: `Dataset ${String(i)}`, + })); + mockListDatasets.mockImplementation( + (opts: { limit?: number; offset?: number } = {}) => { + const remaining = deleted ? all.slice(0, 50) : all; + const offset = opts.offset ?? 0; + return Promise.resolve({ + items: remaining.slice(offset, offset + (opts.limit ?? 50)), + totalMatched: remaining.length, + }); + }, + ); + mockDeleteDataset.mockImplementation(() => { + deleted = true; + return Promise.resolve(undefined); + }); + renderWithIntl(); + + await screen.findByText('Showing 1–50 of 51 datasets'); + await user.click(screen.getByRole('button', { name: 'Next' })); + await screen.findByText('Showing 51–51 of 51 datasets'); + + // Page 2 holds exactly one dataset — delete it. + await user.click(screen.getByRole('button', { name: 'Delete' })); + + // The now-empty page 2 is not rendered as "no datasets"; the page steps + // back and shows the (single, pager-less) remaining page. + expect(await screen.findByText('Dataset 0')).toBeTruthy(); + expect(mockListDatasets).toHaveBeenLastCalledWith({ limit: 50, offset: 0 }); + expect(screen.queryByText('No datasets imported yet.')).toBeNull(); + }); + + it('falls back to an explicit cap warning when the backend cannot count', async () => { + // A graph implementation predating `countDatasets` sends no totalMatched. + mockListDatasets.mockResolvedValue({ + items: Array.from({ length: 50 }, (_, i) => ({ + ...DATASET, + id: `ds-${String(i)}`, + name: `Dataset ${String(i)}`, + })), + }); + renderWithIntl(); + + expect(await screen.findByText(/Showing the 50 most recent datasets/)).toBeTruthy(); + }); + + it('reports the privacy-scan counts after an import', async () => { + const user = userEvent.setup(); + mockUploadDataset.mockResolvedValue({ + dataset: { datasetId: 'ds-2', rowCount: 120, graphNodeId: 'node-2' }, + privacyScan: { scannedCells: 480, maskedCells: 12 }, + truncation: { truncatedCellCount: 0, truncatedColumns: [] }, + }); + renderWithIntl(); + await screen.findByText('Q3 orders'); + + const file = new File(['a,b\n1,2\n'], 'sales.csv', { type: 'text/csv' }); + await user.upload(screen.getByLabelText('CSV file'), file); + await user.click(screen.getByRole('button', { name: 'Import' })); + + expect( + await screen.findByText('Privacy scan: 480 cells checked, 12 masked.'), + ).toBeTruthy(); + expect(screen.getByText('Import complete — 120 rows stored.')).toBeTruthy(); + // The list is re-read so the new dataset appears without a manual refresh. + expect(mockListDatasets).toHaveBeenCalledTimes(2); + }); + + it('clears the file input after an import so the SAME file can be re-imported', async () => { + // Found by clicking the real page: a file input is uncontrolled, so + // resetting `file` state leaves the element holding the old filename. + // Re-picking that same file fires no `change` event, and the Import + // button stays disabled with the element still showing a file — dead UI + // with no explanation. The element itself must be reset. + const user = userEvent.setup(); + mockUploadDataset.mockResolvedValue({ + dataset: { datasetId: 'ds-4', rowCount: 2, graphNodeId: 'node-4' }, + privacyScan: { scannedCells: 4, maskedCells: 0 }, + truncation: { truncatedCellCount: 0, truncatedColumns: [] }, + }); + renderWithIntl(); + await screen.findByText('Q3 orders'); + + const input = screen.getByLabelText('CSV file') as HTMLInputElement; + const file = new File(['a,b\n1,2\n'], 'again.csv', { type: 'text/csv' }); + + await user.upload(input, file); + await user.click(screen.getByRole('button', { name: 'Import' })); + await screen.findByText('Import complete — 2 rows stored.'); + + // The element is empty, so selecting the same file again is a real change. + expect(input.value).toBe(''); + expect(input.files?.length ?? 0).toBe(0); + + await user.upload(input, file); + expect( + (screen.getByRole('button', { name: 'Import' }) as HTMLButtonElement).disabled, + ).toBe(false); + }); + + it('flags truncated cells instead of importing them silently', async () => { + const user = userEvent.setup(); + mockUploadDataset.mockResolvedValue({ + dataset: { datasetId: 'ds-3', rowCount: 4, graphNodeId: 'node-3' }, + privacyScan: { scannedCells: 16, maskedCells: 0 }, + truncation: { truncatedCellCount: 3, truncatedColumns: ['notes'] }, + }); + renderWithIntl(); + await screen.findByText('Q3 orders'); + + await user.upload( + screen.getByLabelText('CSV file'), + new File(['x'], 'big.csv', { type: 'text/csv' }), + ); + await user.click(screen.getByRole('button', { name: 'Import' })); + + expect(await screen.findByText(/3 cells exceeded/)).toBeTruthy(); + expect(screen.getByText(/Affected columns: notes/)).toBeTruthy(); + }); + + it('never deletes without a confirmation', async () => { + const user = userEvent.setup(); + vi.stubGlobal('confirm', vi.fn().mockReturnValue(false)); + renderWithIntl(); + + await screen.findByText('Q3 orders'); + await user.click(screen.getByRole('button', { name: 'Delete' })); + + expect(mockDeleteDataset).not.toHaveBeenCalled(); + }); + + it('deletes and reloads once confirmed', async () => { + const user = userEvent.setup(); + vi.stubGlobal('confirm', vi.fn().mockReturnValue(true)); + renderWithIntl(); + + await screen.findByText('Q3 orders'); + await user.click(screen.getByRole('button', { name: 'Delete' })); + + await waitFor(() => expect(mockDeleteDataset).toHaveBeenCalledWith('ds-1')); + expect(mockListDatasets).toHaveBeenCalledTimes(2); + }); + + it('reports a failed delete in its own error slot and resyncs the list', async () => { + const user = userEvent.setup(); + vi.stubGlobal('confirm', vi.fn().mockReturnValue(true)); + mockDeleteDataset.mockRejectedValue( + new MockApiError(500, 'boom', JSON.stringify({ code: 'dataset.internal_error' })), + ); + renderWithIntl(); + + await screen.findByText('Q3 orders'); + await user.click(screen.getByRole('button', { name: 'Delete' })); + + // The delete failure is NOT dressed up as a list-load failure (#598 + // review must-fix 3) — the list stays rendered next to the error. + expect(await screen.findByText('The request failed (HTTP 500).')).toBeTruthy(); + expect(screen.getByText('Q3 orders')).toBeTruthy(); + expect(screen.queryByText(/Loading failed/)).toBeNull(); + // And the list is re-read: the server may have deleted the row despite + // the error, and a stale row invites a second delete. + expect(mockListDatasets).toHaveBeenCalledTimes(2); + }); + + it('never renders one dataset’s rows under another’s schema (#598 must-fix 2)', async () => { + // Open A (slow rows fetch), then switch to B (fast). A's response lands + // LAST — without the request guard it would overwrite B's panel. + const user = userEvent.setup(); + const datasetB = { + ...DATASET, + id: 'ds-2', + name: 'Suppliers', + sourceFileName: 'suppliers.csv', + columns: [{ name: 'supplier', type: 'string' as const, sample: 'ACME' }], + }; + mockListDatasets.mockResolvedValue({ + items: [DATASET, datasetB], + totalMatched: 2, + }); + mockGetDataset.mockImplementation((id: string) => + Promise.resolve(id === 'ds-1' ? DATASET : datasetB), + ); + let resolveA!: (page: ReturnType) => void; + mockGetDatasetRows.mockImplementation((id: string) => { + if (id === 'ds-1') { + return new Promise((resolve) => { + resolveA = resolve; + }); + } + return Promise.resolve({ + rows: [{ supplier: 'ACME Corp' }], + totalMatched: 1, + }); + }); + renderWithIntl(); + await screen.findByText('Suppliers'); + + const toggles = screen.getAllByRole('button', { name: 'Schema & rows' }); + await user.click(toggles[0]!); // A — its rows promise stays pending + await user.click(toggles[1]!); // B — resolves immediately + await screen.findByText('ACME Corp'); + + resolveA(rowPage(0)); // A's stale response lands after B is open + // Let A's promise chain fully flush before asserting it changed nothing. + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(screen.queryByText('SO-2000')).toBeNull(); + expect(screen.getByText('ACME Corp')).toBeTruthy(); + }); + + it('translates a dataset.* error code rather than showing the raw message', async () => { + const user = userEvent.setup(); + mockUploadDataset.mockRejectedValue( + new MockApiError( + 422, + 'POST /v1/datasets failed: 422', + JSON.stringify({ + code: 'dataset.unsupported_type', + message: 'Nur CSV-Dateien werden aktuell unterstützt (v1 scope, siehe #430).', + }), + ), + ); + renderWithIntl(); + await screen.findByText('Q3 orders'); + + // Named .csv so the input's own `accept` filter lets it through — the + // rejection under test is the SERVER's (`isCsvAttachment` also inspects + // the mimetype), which is the only one that can be trusted anyway. + await user.upload( + screen.getByLabelText('CSV file'), + new File(['x'], 'notes.csv', { type: 'text/plain' }), + ); + await user.click(screen.getByRole('button', { name: 'Import' })); + + expect( + await screen.findByText('Only CSV files are supported so far.'), + ).toBeTruthy(); + // The server's own (English-only, untranslated) message is not the headline. + expect(screen.queryByText(/v1 scope, siehe #430/)).toBeNull(); + }); + + it('maps a 413 to the upload-limit copy', async () => { + const user = userEvent.setup(); + mockUploadDataset.mockRejectedValue( + new MockApiError(413, 'too large', JSON.stringify({ code: 'dataset.limit_file_size' })), + ); + renderWithIntl(); + await screen.findByText('Q3 orders'); + + await user.upload( + screen.getByLabelText('CSV file'), + new File(['x'], 'huge.csv', { type: 'text/csv' }), + ); + await user.click(screen.getByRole('button', { name: 'Import' })); + + expect( + await screen.findByText('The file exceeds the 25 MB upload limit.'), + ).toBeTruthy(); + }); +}); diff --git a/web-ui/app/admin/datasets/_components/DatasetDetail.tsx b/web-ui/app/admin/datasets/_components/DatasetDetail.tsx new file mode 100644 index 00000000..29fd81d8 --- /dev/null +++ b/web-ui/app/admin/datasets/_components/DatasetDetail.tsx @@ -0,0 +1,133 @@ +'use client'; + +import { useFormatter, useTranslations } from 'next-intl'; + +import { Button } from '@/app/_components/ui/Button'; +import { renderCell, ROWS_PER_PAGE, tdCls, thCls, type DetailState } from './shared'; + +/** Schema table + paginated row preview for the one expanded dataset. */ +export function DatasetDetail({ + detail, + onPage, +}: { + detail: DetailState; + onPage: (offset: number) => void; +}): React.ReactElement { + const t = useTranslations('adminDatasets'); + const format = useFormatter(); + + if (detail.error !== null) { + return ( +

+ {detail.error} +

+ ); + } + if (detail.dataset === null) { + return ( +

+ {t('loading')} +

+ ); + } + + const columns = detail.dataset.columns; + const from = detail.totalMatched === 0 ? 0 : detail.offset + 1; + const to = Math.min(detail.offset + detail.rows.length, detail.totalMatched); + + return ( +
+

+ {t('schemaHeading')} +

+
+ + + + + + + + + + {columns.map((c) => ( + + + + + + ))} + +
{t('schema.column')}{t('schema.type')}{t('schema.sample')}
{c.name}{c.type} + {c.sample ?? '—'} +
+
+ +

+ {t('rowsHeading')} +

+ {detail.loading ? ( +

{t('loading')}

+ ) : detail.rows.length === 0 ? ( +

{t('noRows')}

+ ) : ( + <> +
+ + + + {columns.map((c) => ( + + ))} + + + + {detail.rows.map((row, i) => ( + + {columns.map((c) => ( + + ))} + + ))} + +
+ {c.name} +
+ {renderCell(row[c.name])} +
+
+
+ + {t('rowsRange', { + from: format.number(from), + to: format.number(to), + total: format.number(detail.totalMatched), + })} + +
+ + +
+
+ + )} +
+ ); +} diff --git a/web-ui/app/admin/datasets/_components/UploadSection.tsx b/web-ui/app/admin/datasets/_components/UploadSection.tsx new file mode 100644 index 00000000..473a44b0 --- /dev/null +++ b/web-ui/app/admin/datasets/_components/UploadSection.tsx @@ -0,0 +1,140 @@ +'use client'; + +import { useCallback, useRef, useState } from 'react'; + +import { useFormatter, useTranslations } from 'next-intl'; + +import { Button } from '@/app/_components/ui/Button'; +import { uploadDataset, type DatasetUploadResponse } from '../../../_lib/api'; +import { + Field, + inputCls, + MAX_DATASET_ROWS, + MAX_UPLOAD_MB, + toFriendlyError, +} from './shared'; + +/** + * CSV import form + the post-import receipt (privacy-scan and truncation + * counts). Owns its own error slot so an upload failure can never be + * mistaken for a list or delete failure (#532 review must-fix 3). + */ +export function UploadSection({ + onUploaded, +}: { + /** Called after a successful import so the owner can refresh the list. */ + onUploaded: () => Promise; +}): React.ReactElement { + const t = useTranslations('adminDatasets'); + const format = useFormatter(); + + const [file, setFile] = useState(null); + const [name, setName] = useState(''); + const [uploading, setUploading] = useState(false); + const [receipt, setReceipt] = useState(null); + const [error, setError] = useState(null); + /** + * A file input is uncontrolled: clearing `file` state does NOT clear the + * element, and re-picking the SAME file then fires no `change` event + * (the element's value is unchanged), so state would stay `null` while the + * element still shows a filename — the Import button dead with no + * explanation. Reset the element itself after a successful import. + */ + const fileInputRef = useRef(null); + + const onUpload = useCallback(async (): Promise => { + if (!file) return; + setError(null); + setReceipt(null); + setUploading(true); + try { + const result = await uploadDataset(file, name); + setReceipt(result); + setFile(null); + setName(''); + if (fileInputRef.current) fileInputRef.current.value = ''; + await onUploaded(); + } catch (err) { + setError(toFriendlyError(err, t)); + } finally { + setUploading(false); + } + }, [file, name, onUploaded, t]); + + return ( + <> +
+

+ {t('uploadHeading')} +

+
+ + { + setFile(e.target.files?.[0] ?? null); + setReceipt(null); + setError(null); + }} + className={inputCls} + /> + + + setName(e.target.value)} + placeholder={file?.name ?? t('placeholders.name')} + className={inputCls} + /> + + +
+

+ {t('uploadLimits', { + maxRows: format.number(MAX_DATASET_ROWS), + maxMb: format.number(MAX_UPLOAD_MB), + })} +

+
+ + {receipt !== null && ( +
+

+ {t('receipt.imported', { + rows: format.number(receipt.dataset.rowCount), + })} +

+

+ {t('receipt.privacyScan', { + scanned: format.number(receipt.privacyScan.scannedCells), + masked: format.number(receipt.privacyScan.maskedCells), + })} +

+ {receipt.truncation.truncatedCellCount > 0 && ( +

+ {t('receipt.truncated', { + cells: format.number(receipt.truncation.truncatedCellCount), + columns: receipt.truncation.truncatedColumns.join(', '), + })} +

+ )} +
+ )} + + {error !== null && ( +

{error}

+ )} + + ); +} diff --git a/web-ui/app/admin/datasets/_components/shared.tsx b/web-ui/app/admin/datasets/_components/shared.tsx new file mode 100644 index 00000000..791edde4 --- /dev/null +++ b/web-ui/app/admin/datasets/_components/shared.tsx @@ -0,0 +1,110 @@ +'use client'; + +import { ApiError, type DatasetSummary } from '../../../_lib/api'; + +/** + * Shared constants, types, and helpers for Admin → Knowledge · Datasets + * (#532, the admin half of #430). Split out of `page.tsx` to keep every file + * under the repo's 500-line rule. + */ + +/** Row-preview page size. Server clamps `limit` to [1, 200]. */ +export const ROWS_PER_PAGE = 25; +/** + * Dataset-list page size (#532 review must-fix 1: the endpoint used to serve + * only its default 50 with no way to page past them). Matches the server's + * old default so one page looks identical to the pre-pagination list. + */ +export const LIST_PER_PAGE = 50; +/** Mirrors `MAX_DATASET_ROWS` in `@omadia/orchestrator`'s `datasetImport.ts` — + * stated up front so an operator learns the cap before a 50 001-row CSV is + * rejected, not after. Display only; the server owns enforcement. */ +export const MAX_DATASET_ROWS = 50_000; +/** Mirrors `MAX_UPLOAD_BYTES` in the datasets route (25 MiB), same reason. */ +export const MAX_UPLOAD_MB = 25; + +/** The expanded dataset's schema + current row page. */ +export type DetailState = { + id: string; + dataset: DatasetSummary | null; + rows: Array>; + totalMatched: number; + offset: number; + loading: boolean; + error: string | null; +}; + +export type TFn = (key: string, values?: Record) => string; + +export const inputCls = + 'w-full rounded-md border border-[color:var(--border)] bg-transparent px-3 py-2 text-sm outline-none focus:border-[color:var(--accent)]'; +export const thCls = + 'px-2 py-1 text-[11px] font-semibold uppercase tracking-[0.12em]'; +export const tdCls = 'px-2 py-1 align-top'; + +export function Field({ + label, + children, +}: { + label: string; + children: React.ReactNode; +}): React.ReactElement { + return ( + + ); +} + +/** + * Cells arrive as `unknown` — the row store keeps the CSV's own values, which + * are strings today but typed loosely enough that a future non-string backend + * value would otherwise render as `[object Object]` or crash React. + */ +export function renderCell(value: unknown): string { + if (value === null || value === undefined) return '—'; + if (typeof value === 'object') return JSON.stringify(value); + return String(value); +} + +/** + * The `dataset.*` codes the route emits, mapped to localized copy. Kept local + * rather than added to `errorHelp.ts`: that catalogue's coverage test scopes it + * to five specific route files and fails on codes from anywhere else as + * orphans. Same local-mapping shape as `/admin/registries`. + */ +export function toFriendlyError(err: unknown, t: TFn): string { + if (err instanceof ApiError) { + if (err.status === 413 || err.code === 'dataset.limit_file_size') { + return t('errors.tooLarge', { maxMb: MAX_UPLOAD_MB }); + } + if (err.code === 'dataset.unsupported_type') return t('errors.notCsv'); + if (err.code === 'dataset.no_file') return t('errors.noFile'); + if (err.code === 'dataset.import_failed') { + // The route's `reason` is the parser's own diagnostic (ragged rows, row + // cap, empty file) and has no stable code behind it — surface it as + // detail under a localized headline rather than as the headline. + return t('errors.importFailed', { detail: apiMessage(err) }); + } + if (err.code === 'dataset.not_found') return t('errors.notFound'); + return t('errors.generic', { status: err.status }); + } + // Non-ApiError (network failure, programming error): localized headline, + // raw diagnostic only as secondary detail — never as the primary copy. + return t('errors.unexpected', { + detail: err instanceof Error ? err.message : String(err), + }); +} + +/** The server's untranslated `message`, for use as secondary detail only. */ +function apiMessage(err: ApiError): string { + try { + const parsed = JSON.parse(err.body) as { message?: unknown }; + return typeof parsed.message === 'string' ? parsed.message : ''; + } catch { + return ''; + } +} diff --git a/web-ui/app/admin/datasets/page.tsx b/web-ui/app/admin/datasets/page.tsx new file mode 100644 index 00000000..3032f139 --- /dev/null +++ b/web-ui/app/admin/datasets/page.tsx @@ -0,0 +1,352 @@ +'use client'; + +import { useCallback, useEffect, useRef, useState } from 'react'; + +import Link from 'next/link'; +import { useFormatter, useTranslations } from 'next-intl'; + +import { Button } from '@/app/_components/ui/Button'; +import { + deleteDataset, + getDataset, + getDatasetRows, + listDatasets, + type DatasetSummary, +} from '../../_lib/api'; +import { DatasetDetail } from './_components/DatasetDetail'; +import { UploadSection } from './_components/UploadSection'; +import { + LIST_PER_PAGE, + ROWS_PER_PAGE, + toFriendlyError, + type DetailState, +} from './_components/shared'; + +/** + * Admin → Knowledge · Datasets (#532, the admin half of #430). + * + * Upload a CSV, read back its inferred schema and a row preview, delete it + * again. Backed by `/bot-api/v1/datasets` — `requireAuth` plus a per-route + * session-user ACL, so this page only ever shows datasets the logged-in + * operator imported themselves. There is no team or public visibility tier + * yet; "no datasets" here does not mean the instance has none. + * + * Both the dataset list and the row preview are server-paginated + * (`limit`/`offset`, clamped to 200 server-side). For rows that guards + * against 50 000-row datasets; for the list it guards against the review + * finding on #598: datasets past the server's page cap used to be invisible + * AND undeletable from this page. + * + * Upload receipt: every imported row runs through the SAME privacy scan as the + * chat-attachment auto-ingest path, so the scanned/masked cell counts are + * shown on success — an operator uploading customer data should be able to see + * that the masking actually ran, not take it on trust. + */ + +type ListState = + | { kind: 'loading' } + | { + kind: 'ready'; + datasets: DatasetSummary[]; + /** Count before limit/offset; `null` when the backend can't count. */ + totalMatched: number | null; + } + | { kind: 'error'; message: string }; + +export default function AdminDatasetsPage(): React.ReactElement { + const t = useTranslations('adminDatasets'); + const format = useFormatter(); + + const [state, setState] = useState({ kind: 'loading' }); + const [listOffset, setListOffset] = useState(0); + const [deleteError, setDeleteError] = useState(null); + + // per-row pending delete + the single expanded detail panel + const [pending, setPending] = useState(null); + const [detail, setDetail] = useState(null); + + /** + * Monotonic request ids. An awaited response only writes state when it is + * still the LATEST request of its kind — otherwise a slow page 1 arriving + * after a fast page 2 (or dataset A's rows arriving after B was opened) + * would overwrite the newer view with stale data (#598 review must-fix 2). + */ + const listReq = useRef(0); + const detailReq = useRef(0); + /** Which dataset the latest detail request targets — so deleting dataset A + * invalidates A's in-flight load without stranding a concurrent load of B. */ + const detailTarget = useRef(null); + + const reload = useCallback( + async function reloadPage(offset: number): Promise { + const req = ++listReq.current; + try { + const page = await listDatasets({ limit: LIST_PER_PAGE, offset }); + if (listReq.current !== req) return; + if (page.items.length === 0 && offset > 0) { + // Deleting the only dataset of the last page leaves an empty page — + // step back instead of showing "no datasets" while some exist. + await reloadPage(Math.max(0, offset - LIST_PER_PAGE)); + return; + } + setListOffset(offset); + setState({ + kind: 'ready', + datasets: page.items, + totalMatched: page.totalMatched ?? null, + }); + } catch (err) { + if (listReq.current !== req) return; + setState({ + kind: 'error', + message: err instanceof Error ? err.message : String(err), + }); + } + }, + [], + ); + + useEffect(() => { + // Fetch-on-mount: `state` already starts at `loading`, so the awaited + // fetch is the only thing that moves it. + // eslint-disable-next-line react-hooks/set-state-in-effect + void reload(0); + }, [reload]); + + /** + * Open (or re-page) the detail panel. `known` is the schema already held for + * this dataset, so turning a page costs ONE request — the schema is fetched + * on the first open and never again while the panel stays open. + */ + const loadDetail = useCallback( + async ( + id: string, + offset: number, + known: DatasetSummary | null, + ): Promise => { + const req = ++detailReq.current; + detailTarget.current = id; + setDetail((prev) => + prev?.id === id + ? { ...prev, loading: true, error: null } + : { + id, + dataset: known, + rows: [], + totalMatched: 0, + offset, + loading: true, + error: null, + }, + ); + try { + const [dataset, page] = await Promise.all([ + known ?? getDataset(id), + getDatasetRows(id, { limit: ROWS_PER_PAGE, offset }), + ]); + if (detailReq.current !== req) return; + setDetail({ + id, + dataset, + rows: page.rows ?? [], + totalMatched: page.totalMatched, + offset, + loading: false, + error: null, + }); + } catch (err) { + if (detailReq.current !== req) return; + setDetail({ + id, + dataset: null, + rows: [], + totalMatched: 0, + offset, + loading: false, + error: toFriendlyError(err, t), + }); + } + }, + [t], + ); + + const onToggleDetail = useCallback( + (id: string): void => { + if (detail?.id === id) { + // Invalidate any in-flight load so it cannot re-open the panel. + detailReq.current++; + setDetail(null); + return; + } + // `null`, not the summary already in the list: on open the schema is + // re-read alongside the rows, so the preview table's headers and its + // rows are guaranteed to describe the same moment. The listing can be + // minutes old (another tab, another import). + void loadDetail(id, 0, null); + }, + [detail, loadDetail], + ); + + const onDelete = useCallback( + async (d: DatasetSummary): Promise => { + if (!confirm(t('confirmDelete', { name: d.name }))) return; + setDeleteError(null); + setPending(d.id); + try { + await deleteDataset(d.id); + if (detailTarget.current === d.id) detailReq.current++; + setDetail((prev) => (prev?.id === d.id ? null : prev)); + await reload(listOffset); + } catch (err) { + setDeleteError(toFriendlyError(err, t)); + // Resync anyway: the server may have deleted the row despite the + // error surfacing here, and a stale row invites a second delete. + await reload(listOffset); + } finally { + setPending(null); + } + }, + [listOffset, reload, t], + ); + + return ( +
+
+ + ← /admin + +

+ {t('title')} +

+

+ {t.rich('intro', { + code: (chunks) => ( + + {chunks} + + ), + })} +

+
+ + reload(0)} /> + + {deleteError !== null && ( +

{deleteError}

+ )} + + {/* Listing */} + {state.kind === 'loading' ? ( +

{t('loading')}

+ ) : state.kind === 'error' ? ( +

+ {t('loadError', { message: state.message })} +

+ ) : state.datasets.length === 0 ? ( +

{t('empty')}

+ ) : ( + <> + {state.totalMatched === null && state.datasets.length >= LIST_PER_PAGE && ( + // Backend without `countDatasets`: we cannot page reliably, so at + // least say the list may be cut instead of ending it silently. +

+ {t('listCapped', { cap: format.number(LIST_PER_PAGE) })} +

+ )} +
    + {state.datasets.map((d) => ( +
  • +
    +
    + + {d.name} + + + {t('meta', { + rows: format.number(d.rowCount), + columns: format.number(d.columns.length), + created: format.dateTime(new Date(d.createdAt), { + dateStyle: 'medium', + timeStyle: 'short', + }), + })} + + + {d.sourceFileName} + +
    +
    + + +
    +
    + + {detail?.id === d.id && ( + + void loadDetail(d.id, offset, detail.dataset) + } + /> + )} +
    • + ))} +
+ + {state.totalMatched !== null && state.totalMatched > LIST_PER_PAGE && ( +
+ + {t('listRange', { + from: format.number(listOffset + 1), + to: format.number(listOffset + state.datasets.length), + total: format.number(state.totalMatched), + })} + +
+ + +
+
+ )} + + )} +
+ ); +} diff --git a/web-ui/app/admin/page.tsx b/web-ui/app/admin/page.tsx index 176e0367..6bbe4cad 100644 --- a/web-ui/app/admin/page.tsx +++ b/web-ui/app/admin/page.tsx @@ -54,6 +54,8 @@ const GROUPS: readonly GroupDef[] = [ cards: [ { href: '/admin/kg-lifecycle', key: 'kgLifecycle' }, { href: '/admin/kg-priorities', key: 'kgPriorities' }, + // #532 — the admin half of #430's CSV dataset pipeline. + { href: '/admin/datasets', key: 'datasets' }, { href: '/admin/bulk-promote', key: 'bulkPromote' }, { href: '/admin/inconsistencies', key: 'inconsistencies' }, { href: '/admin/memory-backend', key: 'memoryBackend' }, diff --git a/web-ui/i18n/request.ts b/web-ui/i18n/request.ts index 49c3ac17..297847c2 100644 --- a/web-ui/i18n/request.ts +++ b/web-ui/i18n/request.ts @@ -99,5 +99,12 @@ export default getRequestConfig(async () => { async function loadConfig(locale: Locale) { const messages = (await import(`../messages/${locale}.json`)).default; - return { locale, messages }; + // Explicit timeZone: without it next-intl logs an ENVIRONMENT_FALLBACK + // IntlError for every `format.dateTime` call (one per rendered table row on + // /admin/datasets). The host's zone is what the fallback resolved to anyway. + return { + locale, + messages, + timeZone: Intl.DateTimeFormat().resolvedOptions().timeZone, + }; } diff --git a/web-ui/messages/de.json b/web-ui/messages/de.json index eb4d00df..33ec606f 100644 --- a/web-ui/messages/de.json +++ b/web-ui/messages/de.json @@ -195,6 +195,10 @@ "title": "Knowledge-Graph Prioritäten", "description": "Pro Agent Block- und Boost-Listen setzen, die Recall-Treffer im Token-Budget-Assembler überschreiben." }, + "datasets": { + "title": "Datensätze", + "description": "CSVs als strukturierte Datensätze importieren, die Agenten abfragen können, Schema und Zeilen prüfen und sie wieder löschen." + }, "bulkPromote": { "title": "Bulk-Promotion", "description": "Historische Turns nachträglich scoren und hoch bewertete als dauerhaftes Wissen promoten (idempotent)." @@ -4142,6 +4146,56 @@ "colReason": "Grund", "colUpdated": "Aktualisiert" }, + "adminDatasets": { + "title": "Wissen · Datensätze", + "intro": "Eine CSV als strukturierten Datensatz importieren, das von omadia abgeleitete Schema nachlesen und ihn wieder löschen. Agenten greifen über das Tool query_dataset darauf zu — serverseitig gefiltert und aggregiert, nie als ganze Tabelle in einem Turn. Ein Datensatz ist nur für das Konto sichtbar, das ihn importiert hat: Diese Liste ist Ihre, nicht die der Instanz.", + "uploadHeading": "CSV importieren", + "fields": { + "file": "CSV-Datei", + "nameOptional": "Name (optional)" + }, + "placeholders": { + "name": "Standard ist der Dateiname" + }, + "upload": "Importieren", + "uploading": "Importiert", + "uploadLimits": "Höchstens {maxRows} Zeilen und {maxMb} MB pro Datei. Jede Zelle durchläuft vor dem Speichern den Privacy-Scan.", + "receipt": { + "imported": "Import abgeschlossen — {rows} Zeilen gespeichert.", + "privacyScan": "Privacy-Scan: {scanned} Zellen geprüft, {masked} maskiert.", + "truncated": "{cells} Zellen überschritten das Limit von 4.000 Zeichen pro Zelle und wurden gekürzt. Betroffene Spalten: {columns}." + }, + "loading": "lädt…", + "loadError": "Laden fehlgeschlagen: {message}", + "empty": "Noch keine Datensätze importiert.", + "listCapped": "Es werden die {cap} neuesten Datensätze angezeigt — mehr liefert der Endpunkt nicht zurück, ältere können also existieren, ohne hier aufzutauchen.", + "listRange": "Zeige {from}–{to} von {total} Datensätzen", + "meta": "{rows} Zeilen · {columns} Spalten · importiert {created}", + "showDetails": "Schema & Zeilen", + "hideDetails": "Schließen", + "remove": "Löschen", + "confirmDelete": "Den Datensatz „{name}“ mit allen Zeilen und seinem Graph-Knoten löschen? Das lässt sich nicht rückgängig machen.", + "schemaHeading": "Abgeleitetes Schema", + "schema": { + "column": "Spalte", + "type": "Typ", + "sample": "Beispielwert" + }, + "rowsHeading": "Zeilenvorschau", + "noRows": "Dieser Datensatz enthält keine Zeilen.", + "rowsRange": "{from}–{to} von {total} Zeilen", + "previousPage": "Zurück", + "nextPage": "Weiter", + "errors": { + "tooLarge": "Die Datei überschreitet das Upload-Limit von {maxMb} MB.", + "notCsv": "Bisher werden ausschließlich CSV-Dateien unterstützt.", + "noFile": "Es kam keine Datei an — bitte zuerst eine CSV auswählen.", + "importFailed": "Die CSV konnte nicht importiert werden. {detail}", + "notFound": "Diesen Datensatz gibt es nicht mehr — womöglich wurde er in einem anderen Tab gelöscht.", + "generic": "Die Anfrage ist fehlgeschlagen (HTTP {status}).", + "unexpected": "Die Anfrage ist fehlgeschlagen. {detail}" + } + }, "adminMemoryBackend": { "title": "Memory · Speicher-Backend", "intro": "Schaltet das Memory-Storage zwischen Postgres und In-Memory (flüchtig, ohne Datenbank) um. Postgres benötigt DATABASE_URL (Neon-KG/graphPool). Die Auswahl wird gespeichert, der Wechsel greift erst nach einem Neustart.", diff --git a/web-ui/messages/en.json b/web-ui/messages/en.json index a140301f..a999c42d 100644 --- a/web-ui/messages/en.json +++ b/web-ui/messages/en.json @@ -195,6 +195,10 @@ "title": "Knowledge-Graph Priorities", "description": "Set per-agent block and boost lists that override recall hits in the token-budget assembler." }, + "datasets": { + "title": "Datasets", + "description": "Import CSVs as structured datasets agents can query, inspect the inferred schema and rows, and delete them again." + }, "bulkPromote": { "title": "Bulk Promotion", "description": "Re-score historical turns and promote high-significance ones to durable knowledge (idempotent)." @@ -4142,6 +4146,56 @@ "colReason": "Reason", "colUpdated": "Updated" }, + "adminDatasets": { + "title": "Knowledge · Datasets", + "intro": "Import a CSV as a structured dataset, read back the schema omadia inferred from it, and delete it again. Agents reach a dataset through the query_dataset tool — filtered and aggregated server-side, never as a whole table pulled into a turn. A dataset is visible only to the account that imported it, so this list is yours, not the instance's.", + "uploadHeading": "Import CSV", + "fields": { + "file": "CSV file", + "nameOptional": "Name (optional)" + }, + "placeholders": { + "name": "Defaults to the file name" + }, + "upload": "Import", + "uploading": "Importing", + "uploadLimits": "Up to {maxRows} rows and {maxMb} MB per file. Every cell passes the privacy scan before it is stored.", + "receipt": { + "imported": "Import complete — {rows} rows stored.", + "privacyScan": "Privacy scan: {scanned} cells checked, {masked} masked.", + "truncated": "{cells} cells exceeded the 4,000-character per-cell cap and were cut. Affected columns: {columns}." + }, + "loading": "loading…", + "loadError": "Loading failed: {message}", + "empty": "No datasets imported yet.", + "listCapped": "Showing the {cap} most recent datasets — the endpoint returns no more than that, so older ones may exist without appearing here.", + "listRange": "Showing {from}–{to} of {total} datasets", + "meta": "{rows} rows · {columns} columns · imported {created}", + "showDetails": "Schema & rows", + "hideDetails": "Close", + "remove": "Delete", + "confirmDelete": "Delete the dataset “{name}” with all its rows and its graph node? This cannot be undone.", + "schemaHeading": "Inferred schema", + "schema": { + "column": "Column", + "type": "Type", + "sample": "Sample value" + }, + "rowsHeading": "Row preview", + "noRows": "This dataset holds no rows.", + "rowsRange": "Showing {from}–{to} of {total} rows", + "previousPage": "Back", + "nextPage": "Next", + "errors": { + "tooLarge": "The file exceeds the {maxMb} MB upload limit.", + "notCsv": "Only CSV files are supported so far.", + "noFile": "No file arrived — pick a CSV first.", + "importFailed": "The CSV could not be imported. {detail}", + "notFound": "This dataset no longer exists — it may have been deleted in another tab.", + "generic": "The request failed (HTTP {status}).", + "unexpected": "The request failed. {detail}" + } + }, "adminMemoryBackend": { "title": "Memory · Storage Backend", "intro": "Switches the memory storage between Postgres and In-Memory (volatile, no database). Postgres requires DATABASE_URL (Neon KG/graphPool). The choice is saved; the switch only takes effect after a restart.", diff --git a/web-ui/scripts/i18n-identical-allowlist.json b/web-ui/scripts/i18n-identical-allowlist.json index 4db8b86f..0941338c 100644 --- a/web-ui/scripts/i18n-identical-allowlist.json +++ b/web-ui/scripts/i18n-identical-allowlist.json @@ -23,6 +23,7 @@ " it stops being useful for diagnosis." ], "keys": { + "adminDatasets.fields.nameOptional": "loanword", "dashboard.onboarding.cases.hr.name": "loanword", "conductor.graphLabel": "loanword", "conductor.guardLabel": "glossary",