Skip to content

Commit 5831d5d

Browse files
KiteniteKitenite
authored
feat: improve index caching (#2827)
1 parent 28e46d8 commit 5831d5d

File tree

13 files changed

+71
-73
lines changed

13 files changed

+71
-73
lines changed

‎apps/web/client/src/app/project/[id]/_components/canvas/frame/top-bar/page-selector.tsx‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ import { Separator } from '@onlook/ui/separator';
1212
import { cn } from '@onlook/ui/utils';
1313
import { inferPageFromUrl } from '@onlook/utility';
1414
import { observer } from 'mobx-react-lite';
15-
import React, { useEffect, useMemo, useState } from 'react';
15+
import React, { useMemo, useState } from 'react';
1616
import { HoverOnlyTooltip } from '../../../editor-bar/hover-tooltip';
1717
import { PageModal } from '../../../left-panel/page-tab/page-modal';
1818

@@ -97,12 +97,6 @@ export const PageSelector = observer(({ frame, className, tooltipSide = "top", s
9797
return items;
9898
};
9999

100-
useEffect(() => {
101-
if (editorEngine.activeSandbox.routerConfig) {
102-
editorEngine.pages.scanPages();
103-
}
104-
}, [editorEngine.activeSandbox.routerConfig]);
105-
106100
const displayPages = useMemo(() => {
107101
if (allPages.length > 0) {
108102
return allPages;

‎apps/web/client/src/app/project/[id]/_components/left-panel/page-tab/index.tsx‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import { Icons } from '@onlook/ui/icons';
55
import { Input } from '@onlook/ui/input';
66
import { Tooltip, TooltipContent, TooltipPortal, TooltipTrigger } from '@onlook/ui/tooltip';
77
import { observer } from 'mobx-react-lite';
8-
import { useEffect, useMemo, useRef, useState } from 'react';
8+
import { useMemo, useRef, useState } from 'react';
99
import { type NodeApi, Tree, type TreeApi } from 'react-arborist';
1010
import useResizeObserver from 'use-resize-observer';
1111
import { PageTreeNode } from '../layers-tab/tree/page-tree-node';
@@ -21,10 +21,6 @@ export const PagesTab = observer(() => {
2121
const treeRef = useRef<TreeApi<PageNode>>(null);
2222
const inputRef = useRef<HTMLInputElement>(null);
2323

24-
useEffect(() => {
25-
editorEngine.pages.scanPages();
26-
}, []);
27-
2824
const filteredPages = useMemo(() => {
2925
if (!searchQuery.trim()) {
3026
return editorEngine.pages.tree;

‎apps/web/client/src/components/store/editor/cache/file-cache.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,21 +7,21 @@ export class FileCacheManager {
77
private fileCache: UnifiedCacheManager<SandboxFile>;
88
private directoryCache: UnifiedCacheManager<SandboxDirectory>;
99

10-
constructor() {
10+
constructor(projectId: string, branchId: string) {
1111
this.fileCache = new UnifiedCacheManager({
12-
name: 'sandbox-files',
12+
name: `${projectId}-${branchId}-sandbox-files`,
1313
maxItems: 500,
1414
maxSizeBytes: 50 * 1024 * 1024, // 50MB
1515
ttlMs: 1000 * 60 * 30, // 30 minutes
1616
persistent: true,
1717
});
1818

1919
this.directoryCache = new UnifiedCacheManager({
20-
name: 'sandbox-directories',
20+
name: `${projectId}-${branchId}-sandbox-directories`,
2121
maxItems: 1000,
2222
maxSizeBytes: 5 * 1024 * 1024, // 5MB
2323
ttlMs: 1000 * 60 * 60, // 1 hour
24-
persistent: false, // Directories are lightweight, no need to persist
24+
persistent: true,
2525
});
2626
}
2727

‎apps/web/client/src/components/store/editor/cache/unified-cache.ts‎

Lines changed: 10 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
'use client';
22

3-
import { LRUCache } from 'lru-cache';
4-
import localforage from 'localforage';
53
import { jsonClone } from '@onlook/utility';
4+
import localforage from 'localforage';
5+
import { LRUCache } from 'lru-cache';
66
import type { CacheConfig, CachedItem, PersistentCacheData, Serializable } from './types';
77

88
export class UnifiedCacheManager<T extends Serializable = Serializable> {
@@ -13,7 +13,7 @@ export class UnifiedCacheManager<T extends Serializable = Serializable> {
1313

1414
constructor(config: CacheConfig) {
1515
this.config = config;
16-
16+
1717
this.memoryCache = new LRUCache({
1818
max: config.maxItems,
1919
maxSize: config.maxSizeBytes,
@@ -36,7 +36,7 @@ export class UnifiedCacheManager<T extends Serializable = Serializable> {
3636
if (this.persistentStore && this.memoryCache.size === 0) {
3737
await this.loadFromPersistent();
3838
}
39-
39+
4040
this.initialized = true;
4141
}
4242

@@ -53,9 +53,9 @@ export class UnifiedCacheManager<T extends Serializable = Serializable> {
5353
contentHash,
5454
size,
5555
};
56-
56+
5757
this.memoryCache.set(key, item);
58-
58+
5959
// Trigger periodic persistence
6060
if (this.shouldPersist()) {
6161
this.saveToPersistent().catch(console.warn);
@@ -94,12 +94,12 @@ export class UnifiedCacheManager<T extends Serializable = Serializable> {
9494
getCached(key: string, currentContentHash?: string): T | undefined {
9595
const cached = this.memoryCache.get(key);
9696
if (!cached) return undefined;
97-
97+
9898
if (currentContentHash && cached.contentHash !== currentContentHash) {
9999
this.delete(key);
100100
return undefined;
101101
}
102-
102+
103103
return cached.data;
104104
}
105105

@@ -126,7 +126,6 @@ export class UnifiedCacheManager<T extends Serializable = Serializable> {
126126
}
127127
}
128128

129-
130129
async saveToPersistent(): Promise<void> {
131130
if (!this.persistentStore || this.memoryCache.size === 0) return;
132131

@@ -144,7 +143,7 @@ export class UnifiedCacheManager<T extends Serializable = Serializable> {
144143
}
145144

146145
if (entries.length === 0) {
147-
return;
146+
return;
148147
}
149148

150149
const data: PersistentCacheData = {
@@ -197,4 +196,4 @@ export class UnifiedCacheManager<T extends Serializable = Serializable> {
197196
return 1000; // Fallback size
198197
}
199198
}
200-
}
199+
}

‎apps/web/client/src/components/store/editor/engine.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import type { Branch } from '@onlook/models';
12
import { makeAutoObservable } from 'mobx';
23
import type { PostHog } from 'posthog-js';
34
import { ActionManager } from './action';
@@ -28,7 +29,6 @@ import { TemplateNodeManager } from './template-nodes';
2829
import { TextEditingManager } from './text';
2930
import { ThemeManager } from './theme';
3031
import { VersionsManager } from './version';
31-
import type { Branch } from '@onlook/models';
3232

3333
export class EditorEngine {
3434
readonly projectId: string;
@@ -68,11 +68,12 @@ export class EditorEngine {
6868
readonly preloadScript: PreloadScriptManager = new PreloadScriptManager(this);
6969
readonly screenshot: ScreenshotManager = new ScreenshotManager(this);
7070
readonly snap: SnapManager = new SnapManager(this);
71-
readonly templateNodes: TemplateNodeManager = new TemplateNodeManager(this);
71+
readonly templateNodes: TemplateNodeManager;
7272

7373
constructor(projectId: string, posthog: PostHog) {
7474
this.projectId = projectId;
7575
this.posthog = posthog;
76+
this.templateNodes = new TemplateNodeManager(this, projectId);
7677
makeAutoObservable(this);
7778
}
7879

‎apps/web/client/src/components/store/editor/font/index.ts‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
'use client';
22

3-
import { type CodeDiff, type FontUploadFile } from '@onlook/models';
3+
import { LeftPanelTabValue, type CodeDiff, type FontUploadFile } from '@onlook/models';
44
import type { Font } from '@onlook/models/assets';
55
import { generate } from '@onlook/parser';
66
import { makeAutoObservable, reaction } from 'mobx';
@@ -55,6 +55,9 @@ export class FontManager {
5555
};
5656
},
5757
(sandboxStatus) => {
58+
if (this.editorEngine.state.leftPanelTab !== LeftPanelTabValue.BRAND) {
59+
return;
60+
}
5861
if (sandboxStatus.isIndexed && !sandboxStatus.isIndexing) {
5962
this.loadInitialFonts();
6063
this.getCurrentDefaultFont();

‎apps/web/client/src/components/store/editor/pages/index.ts‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
1+
import { LeftPanelTabValue } from '@onlook/models';
12
import type { PageMetadata, PageNode } from '@onlook/models/pages';
2-
import { makeAutoObservable } from 'mobx';
3+
import { makeAutoObservable, reaction } from 'mobx';
34
import type { EditorEngine } from '../engine';
45
import type { FrameData } from '../frames';
56
import {
@@ -25,6 +26,18 @@ export class PagesManager {
2526
makeAutoObservable(this);
2627
}
2728

29+
init() {
30+
reaction(
31+
() => this.editorEngine.activeSandbox.routerConfig,
32+
() => {
33+
if (this.editorEngine.state.leftPanelTab !== LeftPanelTabValue.PAGES) {
34+
return;
35+
}
36+
this.scanPages();
37+
},
38+
);
39+
}
40+
2841
get tree() {
2942
return this.pages;
3043
}

‎apps/web/client/src/components/store/editor/sandbox/file-sync.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,8 @@ import { normalizePath } from './helpers';
66
export class FileSyncManager {
77
private cacheManager: FileCacheManager;
88

9-
constructor() {
10-
this.cacheManager = new FileCacheManager();
9+
constructor(projectId: string, branchId: string) {
10+
this.cacheManager = new FileCacheManager(projectId, branchId);
1111
makeAutoObservable(this);
1212
}
1313

‎apps/web/client/src/components/store/editor/sandbox/index.ts‎

Lines changed: 20 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,7 @@ import {
1515
getDirName,
1616
isImageFile,
1717
isRootLayoutFile,
18-
isSubdirectory,
19-
LogTimer,
18+
isSubdirectory
2019
} from '@onlook/utility';
2120
import { makeAutoObservable, reaction } from 'mobx';
2221
import path from 'path';
@@ -54,7 +53,7 @@ export class SandboxManager {
5453
this.branch,
5554
this.errorManager
5655
);
57-
this.fileSync = new FileSyncManager();
56+
this.fileSync = new FileSyncManager(this.branch.projectId, this.branch.id);
5857
makeAutoObservable(this);
5958
}
6059

@@ -88,7 +87,7 @@ export class SandboxManager {
8887
}
8988

9089
async index(force = false) {
91-
console.log('[SandboxManager] Starting indexing, force:', force);
90+
console.log(`[SandboxManager] Starting indexing for ${this.branch.projectId}/${this.branch.id}, force: ${force}`);
9291

9392
if (this._isIndexing || (this._isIndexed && !force)) {
9493
return;
@@ -100,30 +99,22 @@ export class SandboxManager {
10099
}
101100

102101
this._isIndexing = true;
103-
const timer = new LogTimer('Sandbox Indexing');
104102

105103
try {
106104
// Detect router configuration first
107105
if (!this._routerConfig) {
108106
this._routerConfig = await detectRouterTypeInSandbox(this);
109-
if (this._routerConfig) {
110-
timer.log(
111-
`Router detected: ${this._routerConfig.type} at ${this._routerConfig.basePath}`,
112-
);
113-
}
114107
}
115108

116109
// Get all file paths
117110
const allFilePaths = await this.getAllFilePathsFlat('./', EXCLUDED_SYNC_DIRECTORIES);
118111
this._discoveredFiles = allFilePaths;
119-
timer.log(`File discovery completed - ${allFilePaths.length} files found`);
120112

121113
// Process files in non-blocking batches
122114
await this.processFilesInBatches(allFilePaths);
123115

124116
await this.watchFiles();
125117
this._isIndexed = true;
126-
timer.log('Indexing completed successfully');
127118
} catch (error) {
128119
console.error('Error during indexing:', error);
129120
throw error;
@@ -135,39 +126,39 @@ export class SandboxManager {
135126
/**
136127
* Process files in non-blocking batches to avoid blocking the UI thread
137128
*/
138-
private async processFilesInBatches(allFilePaths: string[], batchSize: number = 20): Promise<void> {
139-
let processed = 0;
140-
129+
private async processFilesInBatches(allFilePaths: string[], batchSize: number = 10): Promise<void> {
141130
for (let i = 0; i < allFilePaths.length; i += batchSize) {
142131
const batch = allFilePaths.slice(i, i + batchSize);
143132

144-
// Process current batch
145-
for (const filePath of batch) {
133+
// Process batch in parallel for better performance
134+
const batchPromises = batch.map(async (filePath) => {
146135
// Track image files first
147136
if (isImageFile(filePath)) {
148137
this.fileSync.writeEmptyFile(filePath, 'binary');
149-
processed++;
150-
continue;
138+
return;
151139
}
152140

153-
const remoteFile = await this.readRemoteFile(filePath);
154-
if (remoteFile) {
155-
this.fileSync.updateCache(remoteFile);
141+
// Check cache first
142+
const cachedFile = this.fileSync.readCache(filePath);
143+
if (cachedFile && cachedFile.content !== null) {
156144
if (this.isJsxFile(filePath)) {
157-
await this.processFileForMapping(remoteFile);
145+
await this.processFileForMapping(cachedFile);
146+
}
147+
} else {
148+
const file = await this.fileSync.readOrFetch(filePath, this.readRemoteFile.bind(this));
149+
if (file && this.isJsxFile(filePath)) {
150+
await this.processFileForMapping(file);
158151
}
159152
}
160-
processed++;
161-
}
153+
});
154+
155+
await Promise.all(batchPromises);
162156

163157
// Yield control to the event loop after each batch
164158
if (i + batchSize < allFilePaths.length) {
165-
console.log(`[SandboxManager] Processed ${processed}/${allFilePaths.length} files...`);
166-
await new Promise(resolve => setTimeout(resolve, 0));
159+
await new Promise(resolve => setTimeout(resolve, 1));
167160
}
168161
}
169-
170-
console.log(`[SandboxManager] Completed processing ${processed} files`);
171162
}
172163

173164
/**

‎apps/web/client/src/components/store/editor/template-nodes/index.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,10 +27,10 @@ export class TemplateNodeManager {
2727
private templateNodes = new Map<string, TemplateNode>();
2828
private processCache: UnifiedCacheManager<TemplateNodeCacheData>;
2929

30-
constructor(editorEngine: EditorEngine) {
30+
constructor(editorEngine: EditorEngine, projectId: string) {
3131
this.editorEngine = editorEngine;
3232
this.processCache = new UnifiedCacheManager({
33-
name: 'template-nodes',
33+
name: `template-nodes-${projectId}`,
3434
maxItems: 200,
3535
maxSizeBytes: 25 * 1024 * 1024, // 25MB
3636
ttlMs: 1000 * 60 * 30, // 30 minutes

0 commit comments

Comments
 (0)