Skip to content

Commit 75bc4c4

Browse files
goniszewskigoniszewski
committed
feat(import): add batch processing, transactions, storing images, display errors
Signed-off-by: Robert Goniszewski <[email protected]>
1 parent 1a6abb6 commit 75bc4c4

File tree

3 files changed

+158
-67
lines changed

3 files changed

+158
-67
lines changed

‎src/lib/types/BookmarkImport.type.ts

+14-1
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,26 @@ export type ImportResult = {
1717
tags: ImportedTag[];
1818
};
1919

20+
export type ImportProgress = {
21+
processed: number;
22+
total: number;
23+
successful: number;
24+
failed: number;
25+
};
26+
2027
export type ImportExecutionResult = {
2128
total: number;
2229
successful: number;
2330
failed: number;
2431
results: Array<{
2532
success: boolean;
26-
bookmark: Bookmark;
33+
bookmark: {
34+
id: number;
35+
url: string;
36+
title: string;
37+
category: string;
38+
success: boolean
39+
};
2740
error?: string;
2841
}>;
2942
};

‎src/lib/utils/bookmark-import/execute-import.ts

+138-64
Original file line numberDiff line numberDiff line change
@@ -1,88 +1,162 @@
1-
import type { BookmarkEdit } from '$lib/types/Bookmark.type';
1+
import type { Bookmark, BookmarkEdit } from '$lib/types/Bookmark.type';
22
import { db } from '$lib/database/db';
33
import { getOrCreateCategory } from '$lib/database/repositories/Category.repository';
44
import { getOrCreateTag } from '$lib/database/repositories/Tag.repository';
55
import { bookmarkSchema, bookmarksToTagsSchema } from '$lib/database/schema';
6-
76
import { createSlug } from '../create-slug';
7+
import type { ImportExecutionResult, ImportProgress } from '$lib/types/BookmarkImport.type';
8+
import { Storage } from '$lib/storage/storage';
9+
import { eq } from 'drizzle-orm';
10+
11+
const BATCH_SIZE = 50;
12+
13+
const storage = new Storage();
814

9-
import type { ImportExecutionResult } from '$lib/types/BookmarkImport.type';
1015
export async function executeImport(
1116
bookmarks: BookmarkEdit[],
12-
userId: number
17+
userId: number,
18+
onProgress?: (progress: ImportProgress) => void
1319
): Promise<ImportExecutionResult> {
14-
const results = [];
20+
const results: ImportExecutionResult['results'] = [];
21+
let processedCount = 0;
1522

16-
for (const bookmark of bookmarks) {
17-
try {
18-
const category = await getOrCreateCategory(userId, {
19-
name: bookmark.category.name,
20-
slug: createSlug(bookmark.category.name)
21-
});
23+
const batches = Array.from({ length: Math.ceil(bookmarks.length / BATCH_SIZE) }, (_, i) =>
24+
bookmarks.slice(i * BATCH_SIZE, (i + 1) * BATCH_SIZE)
25+
);
26+
27+
for (const batch of batches) {
28+
await db.transaction(async (tx) => {
29+
for (const bookmark of batch) {
30+
try {
31+
if (!bookmark.url || !bookmark.title || !bookmark.category?.name) {
32+
throw new Error('Missing required fields');
33+
}
34+
35+
const category = await getOrCreateCategory(userId, {
36+
name: bookmark.category.name,
37+
slug: createSlug(bookmark.category.name)
38+
});
39+
40+
const [newBookmark] = await tx
41+
.insert(bookmarkSchema)
42+
.values({
43+
url: bookmark.url.trim(),
44+
domain: bookmark.domain,
45+
title: bookmark.title.trim(),
46+
description: bookmark.description?.trim() || null,
47+
iconUrl: bookmark.iconUrl,
48+
mainImageUrl: bookmark.mainImageUrl,
49+
importance: bookmark.importance,
50+
flagged: bookmark.flagged,
51+
contentHtml: bookmark.contentHtml,
52+
contentText: bookmark.contentText,
53+
contentType: bookmark.contentType,
54+
author: bookmark.author,
55+
contentPublishedDate: bookmark.contentPublishedDate,
56+
slug: createSlug(bookmark.title),
57+
ownerId: userId,
58+
categoryId: category.id,
59+
created: new Date(),
60+
updated: new Date()
61+
} as typeof bookmarkSchema.$inferInsert)
62+
.returning();
63+
64+
let iconId: number | null = null;
65+
let mainImageId: number | null = null;
2266

23-
const [newBookmark] = await db
24-
.insert(bookmarkSchema)
25-
.values({
26-
url: bookmark.url,
27-
domain: bookmark.domain,
28-
title: bookmark.title,
29-
description: bookmark.description || null,
30-
iconUrl: bookmark.iconUrl,
31-
mainImageUrl: bookmark.mainImageUrl,
32-
importance: bookmark.importance,
33-
flagged: bookmark.flagged,
34-
contentHtml: bookmark.contentHtml,
35-
contentText: bookmark.contentText,
36-
contentType: bookmark.contentType,
37-
author: bookmark.author,
38-
contentPublishedDate: bookmark.contentPublishedDate,
39-
slug: createSlug(bookmark.title),
40-
ownerId: userId,
41-
categoryId: category.id,
42-
created: new Date(),
43-
updated: new Date()
44-
} as typeof bookmarkSchema.$inferInsert)
45-
.returning();
46-
47-
if (bookmark.bookmarkTags?.length) {
48-
const tags = await Promise.all(
49-
bookmark.bookmarkTags.map((tag) =>
50-
getOrCreateTag(userId, {
51-
name: tag.label,
52-
slug: createSlug(tag.label),
53-
ownerId: userId
54-
})
55-
)
56-
);
57-
58-
await db.insert(bookmarksToTagsSchema).values(
59-
tags.map((tag) => ({
60-
bookmarkId: newBookmark.id,
61-
tagId: tag.id
62-
}))
63-
);
67+
if (bookmark.mainImageUrl) {
68+
({ id: mainImageId } = await storage.storeImage(
69+
bookmark.mainImageUrl,
70+
bookmark.title,
71+
userId,
72+
newBookmark.id
73+
));
74+
}
75+
76+
if (bookmark.iconUrl) {
77+
({ id: iconId } = await storage.storeImage(
78+
bookmark.iconUrl,
79+
bookmark.title,
80+
userId,
81+
newBookmark.id
82+
));
83+
}
84+
85+
if (iconId || mainImageId) {
86+
await tx
87+
.update(bookmarkSchema)
88+
.set({
89+
iconId,
90+
mainImageId,
91+
updated: new Date()
92+
})
93+
.where(eq(bookmarkSchema.id, newBookmark.id));
94+
}
95+
96+
if (bookmark.bookmarkTags?.length) {
97+
const tags = await Promise.all(
98+
bookmark.bookmarkTags.map((tag) =>
99+
getOrCreateTag(userId, {
100+
name: tag.label.trim(),
101+
slug: createSlug(tag.label),
102+
ownerId: userId
103+
})
104+
)
105+
);
106+
107+
await tx.insert(bookmarksToTagsSchema).values(
108+
tags.map((tag) => ({
109+
bookmarkId: newBookmark.id,
110+
tagId: tag.id
111+
}))
112+
);
113+
}
114+
115+
results.push({
116+
success: true,
117+
bookmark: {
118+
id: bookmark.id,
119+
url: bookmark.url,
120+
title: bookmark.title,
121+
category: bookmark.category.name,
122+
success: true
123+
}
124+
});
125+
} catch (error) {
126+
console.error('Failed to import bookmark:', bookmark.url, error);
127+
results.push({
128+
success: false,
129+
error: error instanceof Error ? error.message : 'Unknown error',
130+
bookmark: {
131+
id: bookmark.id,
132+
url: bookmark.url,
133+
title: bookmark.title,
134+
category: bookmark.category.name,
135+
success: false
136+
}
137+
});
138+
}
64139
}
140+
});
65141

66-
results.push({
67-
success: true,
68-
bookmark: newBookmark.id
69-
});
70-
} catch (error) {
71-
console.error(error);
72-
results.push({
73-
success: false,
74-
error: error instanceof Error ? error.message : 'Unknown error',
75-
bookmark: bookmark.url
142+
processedCount += batch.length;
143+
144+
if (onProgress) {
145+
onProgress({
146+
processed: processedCount,
147+
total: bookmarks.length,
148+
successful: results.filter((r) => r.success).length,
149+
failed: results.filter((r) => !r.success).length
76150
});
77151
}
78152
}
79153

80-
const result = {
154+
const result: ImportExecutionResult = {
81155
total: bookmarks.length,
82156
successful: results.filter((r) => r.success).length,
83157
failed: results.filter((r) => !r.success).length,
84158
results
85-
} as unknown as ImportExecutionResult;
159+
};
86160

87161
return result;
88162
}

‎src/routes/import/html/+page.svelte

+6-2
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ import { IconFileTypeHtml } from '@tabler/icons-svelte';
1717
const defaultCategory = '[No parent]';
1818
1919
const user = $page.data.user;
20-
const step = writable<number>(3);
20+
const step = writable<number>(1);
2121
const isFetchingMetadata = writable<boolean>(true);
2222
const selectedCategory = writable<string>();
2323
const processedItems = writable<number>(0);
@@ -336,10 +336,11 @@ const onSetSelectedCategory = () => {
336336
<th>Title</th>
337337
<th>Category</th>
338338
<th>URL</th>
339+
<th>Error</th>
339340
</tr>
340341
</thead>
341342
<tbody>
342-
{#each $importResult.results.filter((item) => !item.success) as { bookmark }, i (bookmark.id)}
343+
{#each $importResult.results.filter((item) => !item.success) as { bookmark, error }, i (bookmark.id)}
343344
<tr class="bg-base-200">
344345
<th>{i + 1}</th>
345346
<td class="break-all font-bold">{bookmark.title}</td>
@@ -348,6 +349,9 @@ const onSetSelectedCategory = () => {
348349
><a class="link link-primary" href={bookmark.url} target="_blank"
349350
>{bookmark.url.slice(0, 10)}{bookmark.url.length > 10 ? '...' : ''}</a
350351
></td>
352+
<td class="break-all font-bold text-error">
353+
{error}
354+
</td>
351355
</tr>
352356
{/each}
353357
</tbody>

0 commit comments

Comments
 (0)