feat: knowledge-graph datasets admin UI with paginated list (#532)

docs/middleware-agent-handoff.md

@@ -752,7 +752,10 @@ Neue REST-Oberfläche `src/routes/datasets.ts`, gemountet unter
 
 - `POST /api/v1/datasets` — multipart CSV-Upload (`multer`, ein File pro
   Request, `MAX_UPLOAD_BYTES` = 25 MB).
-- `GET /api/v1/datasets` — Liste der eigenen Datasets.
+- `GET /api/v1/datasets` — paginierte Liste der eigenen Datasets
+  (`limit`/`offset` via zod, 400 bei ungültiger Query; Response
+  `{ items, totalMatched }` — `totalMatched` fehlt nur, wenn die
+  Graph-Implementierung das optionale `countDatasets` noch nicht kennt).
 - `GET /api/v1/datasets/:id` — Schema + Metadaten eines Datasets.
 - `GET /api/v1/datasets/:id/rows` — paginierte Roh-Zeilen.
 - `DELETE /api/v1/datasets/:id` — Dataset löschen.
@@ -1483,9 +1486,14 @@ migrations/0029_datasets.sql`); pro Dataset genau EIN `Dataset`-Graph-Node
 (`PluginEntity`, `system='dataset'`) für Recall/Zitation.
 
 - **Interface:** `KnowledgeGraph.{ingestDataset,listDatasets,getDataset,
-  queryDatasetRows,deleteDataset}` (`plugin-api/src/knowledgeGraph.ts`),
-  implementiert in `@omadia/knowledge-graph-neon` (echtes SQL) UND
-  `@omadia/knowledge-graph-inmemory` (volle Parität, kein Stub).
+  queryDatasetRows,deleteDataset}` plus das **optionale** `countDatasets`
+  (`plugin-api/src/knowledgeGraph.ts`; `listDatasets` nimmt seit #532 auch
+  `offset` — additiv, plugin-api 1.7.0), implementiert in
+  `@omadia/knowledge-graph-neon` (echtes SQL) UND
+  `@omadia/knowledge-graph-inmemory` (volle Parität, kein Stub). Die
+  extras-Wrapper (captureFiltering/inconsistencyTriggering/mergeTriggering)
+  reichen `countDatasets` nur durch, wenn der innere Graph es kann —
+  keine fabrizierten Totals.
 - **Import:** `POST /api/v1/datasets` (multipart CSV, `src/routes/
   datasets.ts`) sowie automatisch bei CSV-Chat-Attachments
   (`attachmentExtract.ts`'s `isCsvAttachment` branch in `orchestrator.ts`'s
@@ -2029,17 +2037,38 @@ gekettet, weil `requires` beim Boot enforced wird): docs-RFC (diese PR)
 omadia-ui-Orchestrator-Consumer. Details + per-PR-Doc-Pflichten in §15
 des RFC.
 
-### Phase 14 — Admin-UI für Dataset-Upload/Schema/Delete (#430 Follow-up)
+### Phase 14 — Admin-UI für Dataset-Upload/Schema/Delete (#430 Follow-up) — **erledigt (#532)**
 
-Der #430-Scope (CSV-Import + `query_dataset`-Tool, siehe §3 und §7) deckt
-absichtlich **keine** Admin-UI ab — Upload/Schema-Browse/Delete bleibt
+Der #430-Scope (CSV-Import + `query_dataset`-Tool, siehe §3 und §7) deckte
+absichtlich **keine** Admin-UI ab — Upload/Schema-Browse/Delete blieb
 API-only (`POST/GET/DELETE /api/v1/datasets*`, siehe §3). #430's eigene
-Triage-Acceptance-Criteria verlangen aber genau diese UI; der Branch
-schließt das Issue deshalb NICHT, sondern "addresses" es — ein
-Folge-Issue für die Admin-UI-Seite (`web-ui/app/admin/datasets/` o.ä.,
-Upload-Dropzone + Schema-Tabelle + Zeilen-Preview + Delete-Bestätigung,
-Pattern analog zur bestehenden Package-Upload-Seite) ist offen zu
-erfassen.
+Triage-Acceptance-Criteria verlangen aber genau diese UI; das
+Folge-Issue #532 hat sie nachgezogen.
+
+Geliefert, überwiegend `web-ui`-seitig; an der REST-Surface aus §3 gab es
+EINE additive Änderung: `GET /api/v1/datasets` paginiert jetzt
+(`limit`/`offset`, Response `{ items, totalMatched }`, getragen von
+`listDatasets.offset` + optionalem `countDatasets` im plugin-api-Interface,
+Minor-Bump auf 1.7.0) — ohne sie waren Datasets jenseits des 50er-Caps im
+Admin-UI unsichtbar UND unlöschbar:
+
+- `web-ui/app/admin/datasets/page.tsx` — Upload (Datei + optionaler Name),
+  Liste, aufklappbares Detail mit Schema-Tabelle und Zeilen-Vorschau,
+  Delete mit Bestätigung.
+- Der Client (`web-ui/app/_lib/api.ts`) spiegelt `DatasetSummary` /
+  `DatasetColumnSchema` / den unaggregierten Zweig von
+  `DatasetQueryResult`, weil `web-ui` nicht gegen den
+  middleware-Workspace baut.
+- Die Zeilen-Vorschau paginiert **server-seitig** über `limit`/`offset`
+  (25 pro Seite, Server clamped auf [1, 200]). Ein Datensatz fasst bis zu
+  `MAX_DATASET_ROWS` (50 000) Zeilen — genau der Fall, gegen den
+  `queryDatasetRows` existiert.
+- Nach einem Import werden `privacyScan.scannedCells` /
+  `maskedCells` und eine etwaige Zell-Truncation angezeigt: der Scan läuft
+  auf diesem Pfad genauso wie beim Chat-Attachment-Auto-Ingest, und das
+  soll sichtbar sein statt geglaubt werden zu müssen.
+- ACL unverändert owner-only: die Seite zeigt ausschließlich Datensätze
+  des eingeloggten Kontos, nicht die der Instanz.
 
 ---
 

middleware/packages/harness-knowledge-graph-inmemory/src/inMemoryKnowledgeGraph.ts

@@ -2997,15 +2997,23 @@ export class InMemoryKnowledgeGraph implements KnowledgeGraph {
   async listDatasets(opts: {
     ownerOmadiaUserId: string;
     limit?: number;
+    offset?: number;
   }): Promise<DatasetSummary[]> {
     const limit = Math.max(1, Math.min(opts.limit ?? 50, 200));
+    const offset = Math.max(0, opts.offset ?? 0);
     return [...this.datasets.values()]
       .filter((d) => d.ownerOmadiaUserId === opts.ownerOmadiaUserId)
       .sort((a, b) => b.createdAt.localeCompare(a.createdAt))
-      .slice(0, limit)
+      .slice(offset, offset + limit)
       .map((d) => this.datasetToSummary(d));
   }
 
+  async countDatasets(opts: { ownerOmadiaUserId: string }): Promise<number> {
+    return [...this.datasets.values()].filter(
+      (d) => d.ownerOmadiaUserId === opts.ownerOmadiaUserId,
+    ).length;
+  }
+
   async getDataset(
     datasetId: string,
     viewerOmadiaUserId: string,

middleware/packages/harness-knowledge-graph-neon/src/neonKnowledgeGraph.ts

@@ -703,19 +703,31 @@ export class NeonKnowledgeGraph implements KnowledgeGraph {
   async listDatasets(opts: {
     ownerOmadiaUserId: string;
     limit?: number;
+    offset?: number;
   }): Promise<DatasetSummary[]> {
     const limit = Math.max(1, Math.min(opts.limit ?? 50, 200));
+    const offset = Math.max(0, opts.offset ?? 0);
     const result = await this.pool.query<DatasetRow>(
       `SELECT id, name, source_file_name, owner_omadia_user_id, row_count, columns, created_at
        FROM datasets
        WHERE tenant_id = $1 AND owner_omadia_user_id = $2
        ORDER BY created_at DESC
-       LIMIT $3`,
-      [this.tenantId, opts.ownerOmadiaUserId, limit],
+       LIMIT $3 OFFSET $4`,
+      [this.tenantId, opts.ownerOmadiaUserId, limit, offset],
     );
     return result.rows.map((r) => this.datasetRowToSummary(r));
   }
 
+  async countDatasets(opts: { ownerOmadiaUserId: string }): Promise<number> {
+    const result = await this.pool.query<{ count: string }>(
+      `SELECT COUNT(*) AS count
+       FROM datasets
+       WHERE tenant_id = $1 AND owner_omadia_user_id = $2`,
+      [this.tenantId, opts.ownerOmadiaUserId],
+    );
+    return Number(result.rows[0]?.count ?? 0);
+  }
+
   async getDataset(
     datasetId: string,
     viewerOmadiaUserId: string,

middleware/packages/harness-orchestrator-extras/src/captureFilteringKnowledgeGraph.ts

@@ -111,10 +111,20 @@ export class CaptureFilteringKnowledgeGraph implements KnowledgeGraph {
   private readonly filter: CaptureFilter;
   private readonly log: (msg: string) => void;
 
+  /**
+   * Optional on the interface (plugin-api back-compat). Mirrors the inner
+   * graph's support instead of fabricating a capped count: the route treats
+   * absence as "backend cannot count" and the UI then warns instead of
+   * rendering a confidently wrong total.
+   */
+  readonly countDatasets?: (opts: { ownerOmadiaUserId: string }) => Promise<number>;
+
   constructor(opts: CaptureFilteringKnowledgeGraphOptions) {
     this.inner = opts.inner;
     this.filter = opts.filter;
     this.log = opts.log ?? ((msg): void => console.error(msg));
+    const innerCount = opts.inner.countDatasets?.bind(opts.inner);
+    if (innerCount) this.countDatasets = innerCount;
   }
 
   async ingestTurn(turn: TurnIngest): Promise<TurnIngestResult> {
@@ -616,6 +626,7 @@ export class CaptureFilteringKnowledgeGraph implements KnowledgeGraph {
   listDatasets(opts: {
     ownerOmadiaUserId: string;
     limit?: number;
+    offset?: number;
   }): Promise<DatasetSummary[]> {
     return this.inner.listDatasets(opts);
   }

middleware/packages/harness-orchestrator-extras/src/inconsistencyTriggeringKnowledgeGraph.ts

@@ -95,10 +95,20 @@ export class InconsistencyTriggeringKnowledgeGraph implements KnowledgeGraph {
   private readonly detector: InconsistencyDetectorService;
   private readonly log: (msg: string) => void;
 
+  /**
+   * Optional on the interface (plugin-api back-compat). Mirrors the inner
+   * graph's support instead of fabricating a capped count: the route treats
+   * absence as "backend cannot count" and the UI then warns instead of
+   * rendering a confidently wrong total.
+   */
+  readonly countDatasets?: (opts: { ownerOmadiaUserId: string }) => Promise<number>;
+
   constructor(opts: InconsistencyTriggeringKnowledgeGraphOptions) {
     this.inner = opts.inner;
     this.detector = opts.detector;
     this.log = opts.log ?? ((msg: string): void => { console.error(msg); });
+    const innerCount = opts.inner.countDatasets?.bind(opts.inner);
+    if (innerCount) this.countDatasets = innerCount;
   }
 
   private fire(mkId: string): void {
@@ -542,6 +552,7 @@ export class InconsistencyTriggeringKnowledgeGraph implements KnowledgeGraph {
   listDatasets(opts: {
     ownerOmadiaUserId: string;
     limit?: number;
+    offset?: number;
   }): Promise<DatasetSummary[]> {
     return this.inner.listDatasets(opts);
   }

middleware/packages/harness-orchestrator-extras/src/mergeTriggeringKnowledgeGraph.ts

@@ -100,10 +100,20 @@ export class MergeTriggeringKnowledgeGraph implements KnowledgeGraph {
   private readonly detector: MergeCandidateDetectorService;
   private readonly log: (msg: string) => void;
 
+  /**
+   * Optional on the interface (plugin-api back-compat). Mirrors the inner
+   * graph's support instead of fabricating a capped count: the route treats
+   * absence as "backend cannot count" and the UI then warns instead of
+   * rendering a confidently wrong total.
+   */
+  readonly countDatasets?: (opts: { ownerOmadiaUserId: string }) => Promise<number>;
+
   constructor(opts: MergeTriggeringKnowledgeGraphOptions) {
     this.inner = opts.inner;
     this.detector = opts.detector;
     this.log = opts.log ?? ((msg: string): void => { console.error(msg); });
+    const innerCount = opts.inner.countDatasets?.bind(opts.inner);
+    if (innerCount) this.countDatasets = innerCount;
   }
 
   private fire(mkId: string): void {
@@ -597,6 +607,7 @@ export class MergeTriggeringKnowledgeGraph implements KnowledgeGraph {
   listDatasets(opts: {
     ownerOmadiaUserId: string;
     limit?: number;
+    offset?: number;
   }): Promise<DatasetSummary[]> {
     return this.inner.listDatasets(opts);
   }

middleware/packages/plugin-api/api-snapshot/plugin-api.d.ts.snap

@@ -531,7 +531,11 @@ ingestDataset(input: DatasetIngest): Promise<DatasetIngestResult>;
 listDatasets(opts: {
 ownerOmadiaUserId: string;
 limit?: number;
+offset?: number;
 }): Promise<DatasetSummary[]>;
+countDatasets?(opts: {
+ownerOmadiaUserId: string;
+}): Promise<number>;
 getDataset(datasetId: string, viewerOmadiaUserId: string): Promise<DatasetSummary | null>;
 queryDatasetRows(datasetId: string, viewerOmadiaUserId: string, opts?: DatasetQueryOptions): Promise<DatasetQueryResult | null>;
 deleteDataset(datasetId: string, actor: AclMutationOptions): Promise<boolean>;

middleware/packages/plugin-api/src/knowledgeGraph.ts

@@ -711,11 +711,25 @@ export interface KnowledgeGraph {
    * KnowledgeGraph boundary, never inside it.
    */
   ingestDataset(input: DatasetIngest): Promise<DatasetIngestResult>;
-  /** #430 — list datasets owned by the caller, most-recent first. */
+  /**
+   * #430 — list datasets owned by the caller, most-recent first.
+   * `limit` is clamped by implementations (default 50, max 200); `offset`
+   * skips that many newest datasets so the admin UI can page past the cap
+   * (#532 review: without it, datasets beyond the cap were invisible AND
+   * undeletable).
+   */
   listDatasets(opts: {
     ownerOmadiaUserId: string;
     limit?: number;
+    offset?: number;
   }): Promise<DatasetSummary[]>;
+  /**
+   * #532 — total number of datasets owned by the caller, so list surfaces
+   * can render real pagination ("showing N of M") instead of a silently
+   * truncated page. Optional for plugin-api back-compat: implementations
+   * that predate it keep working, and callers fall back to the page length.
+   */
+  countDatasets?(opts: { ownerOmadiaUserId: string }): Promise<number>;
   /**
    * #430 — read one dataset's metadata + inferred schema. Null when
    * missing or the viewer doesn't own it (ACL mirrors `/api/v1/memory`:

middleware/src/routes/datasets.ts

@@ -29,6 +29,15 @@ const RowsQuerySchema = z.object({
   offset: z.coerce.number().int().min(0).optional(),
 });
 
+// Same page shape as the rows endpoint, but its own schema on purpose: a
+// future change to the rows bounds must not silently change list validation
+// (#532 review must-fix 1: without limit/offset the route served only the
+// newest 50 datasets).
+const ListQuerySchema = z.object({
+  limit: z.coerce.number().int().min(1).max(200).optional(),
+  offset: z.coerce.number().int().min(0).optional(),
+});
+
 function requireSessionUserId(req: Request, res: Response): string | null {
   const id = req.session?.omadia_user_id;
   if (!id) {
@@ -130,13 +139,29 @@ export function createDatasetsRouter(deps: { graph: KnowledgeGraph }): Router {
     },
   );
 
-  // ── GET / — list current user's datasets ────────────────────────────────
+  // ── GET / — list current user's datasets (paginated) ────────────────────
   router.get('/', async (req: Request, res: Response) => {
     const sessionUserId = requireSessionUserId(req, res);
     if (!sessionUserId) return;
+    const parsed = ListQuerySchema.safeParse(req.query);
+    if (!parsed.success) {
+      res.status(400).json({ code: 'dataset.invalid_query', issues: parsed.error.issues });
+      return;
+    }
     try {
-      const items = await deps.graph.listDatasets({ ownerOmadiaUserId: sessionUserId });
-      res.json({ items });
+      const [items, totalMatched] = await Promise.all([
+        deps.graph.listDatasets({
+          ownerOmadiaUserId: sessionUserId,
+          ...(parsed.data.limit !== undefined ? { limit: parsed.data.limit } : {}),
+          ...(parsed.data.offset !== undefined ? { offset: parsed.data.offset } : {}),
+        }),
+        deps.graph.countDatasets
+          ? deps.graph.countDatasets({ ownerOmadiaUserId: sessionUserId })
+          : Promise.resolve(undefined),
+      ]);
+      // `totalMatched` mirrors GET /:id/rows; absent only when the graph
+      // implementation predates the optional `countDatasets`.
+      res.json({ items, ...(totalMatched !== undefined ? { totalMatched } : {}) });
     } catch (err) {
       const { status, code, message } = mapErrorToHttp(err);
       res.status(status).json({ code, message });

middleware/test/datasetsRoute.test.ts

@@ -137,6 +137,45 @@ describe('POST /api/v1/datasets', () => {
     await throwing.close();
   });
 
+  it('paginates the list: limit/offset pass through and totalMatched is returned (#532)', async () => {
+    const graph = new InMemoryKnowledgeGraph();
+    const paged = await makeHarness('user-1', graph);
+    for (let i = 0; i < 5; i++) {
+      await graph.ingestDataset({
+        name: `ds-${String(i)}`,
+        sourceFileName: `ds-${String(i)}.csv`,
+        ownerOmadiaUserId: 'user-1',
+        columns: [{ name: 'a', type: 'string' }],
+        rows: [{ a: 'x' }],
+      });
+    }
+
+    const page = (await (await fetch(`${paged.baseUrl}?limit=2`)).json()) as {
+      items: unknown[];
+      totalMatched: number;
+    };
+    assert.equal(page.items.length, 2);
+    assert.equal(page.totalMatched, 5);
+
+    const lastPage = (await (await fetch(`${paged.baseUrl}?limit=2&offset=4`)).json()) as {
+      items: unknown[];
+      totalMatched: number;
+    };
+    assert.equal(lastPage.items.length, 1);
+    assert.equal(lastPage.totalMatched, 5);
+
+    await paged.close();
+  });
+
+  it('400s on an invalid list query instead of silently ignoring it', async () => {
+    for (const qs of ['?limit=0', '?limit=201', '?offset=-1', '?limit=abc']) {
+      const res = await fetch(`${h.baseUrl}${qs}`);
+      assert.equal(res.status, 400, qs);
+      const body = (await res.json()) as { code: string };
+      assert.equal(body.code, 'dataset.invalid_query', qs);
+    }
+  });
+
   it('scopes datasets per owner — a different session cannot see or delete them', async () => {
     const form = new FormData();
     form.append('file', new Blob([CSV], { type: 'text/csv' }), 'people.csv');