[codex] SQLite node store read scaling and UI materialized views

apps/electron/src/data-process/data-service.ts

@@ -118,6 +118,7 @@ interface SerializedNodeState {
 interface ListNodesOptions {
   schemaId?: string
   includeDeleted?: boolean
+  orderBy?: Partial<Record<'createdAt' | 'updatedAt', 'asc' | 'desc'>>
   limit?: number
   offset?: number
 }
@@ -127,6 +128,10 @@ interface CountNodesOptions {
   includeDeleted?: boolean
 }
 
+interface SetNodeOptions {
+  indexProperties?: boolean
+}
+
 export interface DataService {
   initialize(): Promise<void>
   shutdown(): Promise<void>
@@ -160,7 +165,7 @@ export interface DataService {
   getChangeByHash(hash: string): Promise<SerializedNodeChange | null>
   getLastChange(nodeId: string): Promise<SerializedNodeChange | null>
   getNode(id: string): Promise<SerializedNodeState | null>
-  setNode(node: SerializedNodeState): Promise<void>
+  setNode(node: SerializedNodeState, options?: SetNodeOptions): Promise<void>
   deleteNode(id: string): Promise<void>
   listNodes(options?: ListNodesOptions): Promise<SerializedNodeState[]>
   countNodes(options?: CountNodesOptions): Promise<number>
@@ -264,6 +269,134 @@ const PEER_TIMEOUT_MS = 4444
 const PEER_CHECK_INTERVAL_MS = 1111
 const HEARTBEAT_INTERVAL_MS = 1111
 
+type ScalarIndexValue = {
+  valueType: 'text' | 'number' | 'boolean' | 'null'
+  valueText: string | null
+  valueNumber: number | null
+  valueBoolean: number | null
+  valueHash: string
+}
+
+function hashScalarValue(value: string): string {
+  let hash = 2166136261
+  for (let i = 0; i < value.length; i++) {
+    hash ^= value.charCodeAt(i)
+    hash = Math.imul(hash, 16777619)
+  }
+  return (hash >>> 0).toString(16).padStart(8, '0')
+}
+
+function toScalarIndexValue(value: unknown): ScalarIndexValue | null {
+  if (value === null) {
+    return {
+      valueType: 'null',
+      valueText: null,
+      valueNumber: null,
+      valueBoolean: null,
+      valueHash: 'null'
+    }
+  }
+
+  if (typeof value === 'string') {
+    return {
+      valueType: 'text',
+      valueText: value,
+      valueNumber: null,
+      valueBoolean: null,
+      valueHash: hashScalarValue(value)
+    }
+  }
+
+  if (typeof value === 'number' && Number.isFinite(value)) {
+    return {
+      valueType: 'number',
+      valueText: null,
+      valueNumber: value,
+      valueBoolean: null,
+      valueHash: hashScalarValue(String(value))
+    }
+  }
+
+  if (typeof value === 'boolean') {
+    return {
+      valueType: 'boolean',
+      valueText: null,
+      valueNumber: null,
+      valueBoolean: value ? 1 : 0,
+      valueHash: value ? 'true' : 'false'
+    }
+  }
+
+  return null
+}
+
+function buildNodeListOrderBy(
+  orderBy?: Partial<Record<'createdAt' | 'updatedAt', 'asc' | 'desc'>>
+): string {
+  const entries = Object.entries(orderBy ?? {}).filter(
+    (entry): entry is ['createdAt' | 'updatedAt', 'asc' | 'desc'] =>
+      entry[1] === 'asc' || entry[1] === 'desc'
+  )
+
+  if (entries.length === 0) {
+    return 'updated_at DESC'
+  }
+
+  return entries
+    .map(([field, direction]) => {
+      const column = field === 'createdAt' ? 'created_at' : 'updated_at'
+      return `${column} ${direction.toUpperCase()}`
+    })
+    .join(', ')
+}
+
+async function syncScalarRowsForNode(
+  adapter: ElectronSQLiteAdapter,
+  node: SerializedNodeState,
+  indexProperties: boolean
+): Promise<void> {
+  await adapter.run('DELETE FROM node_property_scalars WHERE node_id = ?', [node.id])
+
+  if (!indexProperties) {
+    return
+  }
+
+  for (const [key, value] of Object.entries(node.properties)) {
+    const timestamp = node.timestamps[key]
+    const scalar = toScalarIndexValue(value)
+    if (!timestamp || !scalar) continue
+
+    await adapter.run(
+      `INSERT INTO node_property_scalars
+        (
+          node_id,
+          schema_id,
+          property_key,
+          value_type,
+          value_text,
+          value_number,
+          value_boolean,
+          value_hash,
+          updated_at,
+          lamport_time
+        )
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+      [
+        node.id,
+        node.schemaId,
+        key,
+        scalar.valueType,
+        scalar.valueText,
+        scalar.valueNumber,
+        scalar.valueBoolean,
+        scalar.valueHash,
+        timestamp.wallTime,
+        timestamp.lamport.time
+      ]
+    )
+  }
+}
+
 // ─── Data Service Factory ───────────────────────────────────────────────────
 
 export function createDataService(config: DataServiceConfig): DataService {
@@ -1406,7 +1539,7 @@ export function createDataService(config: DataServiceConfig): DataService {
       }
     },
 
-    async setNode(node: SerializedNodeState): Promise<void> {
+    async setNode(node: SerializedNodeState, options?: SetNodeOptions): Promise<void> {
       if (!adapter) throw new Error('Database not initialized')
 
       // Upsert node
@@ -1423,6 +1556,17 @@ export function createDataService(config: DataServiceConfig): DataService {
         ]
       )
 
+      const propertyKeys = Object.keys(node.properties)
+      if (propertyKeys.length === 0) {
+        await adapter.run('DELETE FROM node_properties WHERE node_id = ?', [node.id])
+      } else {
+        const placeholders = propertyKeys.map(() => '?').join(', ')
+        await adapter.run(
+          `DELETE FROM node_properties WHERE node_id = ? AND property_key NOT IN (${placeholders})`,
+          [node.id, ...propertyKeys]
+        )
+      }
+
       // Upsert properties
       for (const [key, value] of Object.entries(node.properties)) {
         const ts = node.timestamps[key]
@@ -1440,6 +1584,8 @@ export function createDataService(config: DataServiceConfig): DataService {
         )
       }
 
+      await syncScalarRowsForNode(adapter, node, options?.indexProperties ?? true)
+
       // Store document content if present
       if (node.documentContent) {
         await adapter.run(
@@ -1456,6 +1602,7 @@ export function createDataService(config: DataServiceConfig): DataService {
       if (!adapter) throw new Error('Database not initialized')
 
       // Delete in order to respect foreign keys
+      await adapter.run('DELETE FROM node_property_scalars WHERE node_id = ?', [id])
       await adapter.run('DELETE FROM node_properties WHERE node_id = ?', [id])
       await adapter.run('DELETE FROM yjs_state WHERE node_id = ?', [id])
       await adapter.run('DELETE FROM changes WHERE node_id = ?', [id])
@@ -1479,14 +1626,17 @@ export function createDataService(config: DataServiceConfig): DataService {
         sql += ' AND deleted_at IS NULL'
       }
 
-      sql += ' ORDER BY updated_at DESC'
+      sql += ` ORDER BY ${buildNodeListOrderBy(options?.orderBy)}`
 
-      if (options?.limit) {
+      if (options?.limit !== undefined) {
         sql += ' LIMIT ?'
         params.push(options.limit)
       }
 
-      if (options?.offset) {
+      if (options?.offset !== undefined) {
+        if (options.limit === undefined) {
+          sql += ' LIMIT -1'
+        }
         sql += ' OFFSET ?'
         params.push(options.offset)
       }

apps/electron/src/data-process/index.ts

@@ -375,9 +375,12 @@ process.parentPort?.on('message', async (event) => {
       }
 
       case 'nodes:setNode': {
-        const { node } = payload as { node: unknown }
+        const { node, options } = payload as { node: unknown; options?: unknown }
         if (dataService) {
-          await dataService.setNode(node as Parameters<typeof dataService.setNode>[0])
+          await dataService.setNode(
+            node as Parameters<typeof dataService.setNode>[0],
+            options as Parameters<typeof dataService.setNode>[1]
+          )
         }
         sendResponse(requestId, { success: true })
         break

apps/electron/src/main/data-process-manager.ts

@@ -510,9 +510,12 @@ export function setupDataProcessIPC(getMainWindow: () => BrowserWindow | null):
     return result.node
   })
 
-  ipcMain.handle('xnet:nodes:setNode', async (_event, opts: { node: unknown }) => {
-    await sendRequest('nodes:setNode', opts)
-  })
+  ipcMain.handle(
+    'xnet:nodes:setNode',
+    async (_event, opts: { node: unknown; options?: unknown }) => {
+      await sendRequest('nodes:setNode', opts)
+    }
+  )
 
   ipcMain.handle('xnet:nodes:deleteNode', async (_event, opts: { id: string }) => {
     await sendRequest('nodes:deleteNode', opts)

apps/electron/src/preload/index.ts

@@ -354,7 +354,8 @@ contextBridge.exposeInMainWorld('xnetNodes', {
 
   // Materialized state operations
   getNode: (id: string) => ipcRenderer.invoke('xnet:nodes:getNode', { id }),
-  setNode: (node: unknown) => ipcRenderer.invoke('xnet:nodes:setNode', { node }),
+  setNode: (node: unknown, options?: unknown) =>
+    ipcRenderer.invoke('xnet:nodes:setNode', { node, options }),
   deleteNode: (id: string) => ipcRenderer.invoke('xnet:nodes:deleteNode', { id }),
   listNodes: (options?: unknown) => ipcRenderer.invoke('xnet:nodes:listNodes', options ?? {}),
   countNodes: (options?: unknown) => ipcRenderer.invoke('xnet:nodes:countNodes', options ?? {}),
@@ -525,7 +526,7 @@ export interface XNetNodesAPI {
 
   // Materialized state operations
   getNode(id: string): Promise<unknown | null>
-  setNode(node: unknown): Promise<void>
+  setNode(node: unknown, options?: unknown): Promise<void>
   deleteNode(id: string): Promise<void>
   listNodes(options?: unknown): Promise<unknown[]>
   countNodes(options?: unknown): Promise<number>

apps/electron/src/renderer/components/DatabaseView.test.tsx

@@ -328,4 +328,21 @@ describe('DatabaseView minimal chrome', () => {
     expect(screen.getByTestId('share-button')).toBeTruthy()
     expect(screen.queryByRole('button', { name: /open database actions/i })).toBeNull()
   })
+
+  it('keys row queries to the visible materialized database view', async () => {
+    render(<DatabaseView docId="db-1" />)
+
+    expect(await screen.findByDisplayValue('Focus DB')).toBeTruthy()
+    expect(mockUseDatabase).toHaveBeenLastCalledWith('db-1', {
+      view: 'table-view',
+      materializedView: { viewId: 'database:db-1:view:table-view' }
+    })
+
+    fireEvent.click(screen.getByRole('button', { name: /board/i }))
+
+    expect(mockUseDatabase).toHaveBeenLastCalledWith('db-1', {
+      view: 'board-view',
+      materializedView: { viewId: 'database:db-1:view:board-view' }
+    })
+  })
 })

apps/electron/src/renderer/components/DatabaseView.tsx

@@ -433,14 +433,29 @@ export function DatabaseView({ docId, minimalChrome = false }: DatabaseViewProps
     createView,
     updateView
   } = useDatabaseDoc(docId)
+  const [viewMode, setViewMode] = useState<ViewMode>('table')
+  const activeDatabaseViewId = useMemo(
+    () => views.find((view) => view.type === viewMode)?.id,
+    [views, viewMode]
+  )
+  const rowQueryOptions = useMemo(
+    () =>
+      activeDatabaseViewId
+        ? {
+            view: activeDatabaseViewId,
+            materializedView: { viewId: `database:${docId}:view:${activeDatabaseViewId}` }
+          }
+        : { materializedView: false as const },
+    [activeDatabaseViewId, docId]
+  )
   const {
     rows,
     loading: rowsLoading,
     createRow,
     updateRow,
     deleteRow,
     reorderRow
-  } = useDatabase(docId)
+  } = useDatabase(docId, rowQueryOptions)
   const undoScope = useMemo(() => [docId, ...rows.map((row) => row.id)], [docId, rows])
   const {
     undo: undoStructured,
@@ -451,7 +466,6 @@ export function DatabaseView({ docId, minimalChrome = false }: DatabaseViewProps
     localDID: did ?? null
   })
 
-  const [viewMode, setViewMode] = useState<ViewMode>('table')
   const [cellPresences, setCellPresences] = useState<CellPresence[]>([])
   const [selectedCardId, setSelectedCardId] = useState<string | null>(null)
   const [sidebarOpen, setSidebarOpen] = useState(false)