From 5866433e82cd59b97ba9ffc9e64be46f05ef4adc Mon Sep 17 00:00:00 2001 From: hmjn Date: Mon, 29 Jun 2026 23:25:26 +0900 Subject: [PATCH 01/19] fix(search): improve vector search recovery and AI logs --- .../infrastructure/api/routers/ai-router.ts | 118 ++++++++++++++++-- .../api/routers/media-router.ts | 57 +++++++-- .../src/infrastructure/jobs/ccip-jobs.ts | 13 +- .../src/infrastructure/jobs/job-worker.ts | 33 ++++- .../src/infrastructure/jobs/tagging-jobs.ts | 23 +++- apps/server/src/routes/api/rpc.$.ts | 10 +- packages/ui/src/search-control-panel.tsx | 11 ++ packages/ui/src/stores/search-store.ts | 8 ++ 8 files changed, 253 insertions(+), 20 deletions(-) diff --git a/apps/server/src/infrastructure/api/routers/ai-router.ts b/apps/server/src/infrastructure/api/routers/ai-router.ts index 4116a9207..bda537112 100644 --- a/apps/server/src/infrastructure/api/routers/ai-router.ts +++ b/apps/server/src/infrastructure/api/routers/ai-router.ts @@ -187,10 +187,29 @@ export const aiRouter = { ]), ) .handler(async ({ input }) => { + const startedAt = Date.now(); + const logContext = + "file" in input + ? { + inputType: "file", + fileName: input.file.name, + fileSize: input.file.size, + } + : { + inputType: "media", + mediaSourceId: input.mediaSourceId, + mediaId: input.mediaId, + }; + logger.info(logContext, "AI tagging started"); try { if ("file" in input) { const buffer = await input.file.arrayBuffer(); - return await taggingService.getTags(buffer); + const result = await taggingService.getTags(buffer); + logger.info( + { ...logContext, durationMs: Date.now() - startedAt }, + "AI tagging completed", + ); + return result; } const { mediaSourceId, mediaId } = input; @@ -224,17 +243,45 @@ export const aiRouter = { } const fullPath = path.join(connectionInfo.path, media.filePath); const fileBuffer = await readFileBuffer(fullPath); - return (await callRemoteTagging( + const result = (await callRemoteTagging( remoteUrl, fileBuffer, path.basename(fullPath), config.ai.timeoutMs, )) as import("@solid-imager/core/domain/tagging/schemas").TaggingResponse; + logger.info( + { + ...logContext, + execution: "remote", + durationMs: Date.now() - startedAt, + }, + "AI tagging completed", + ); + return result; } - return await taggingService.getTagsForMedia(mediaSourceId, mediaId); + const result = await taggingService.getTagsForMedia( + mediaSourceId, + mediaId, + ); + logger.info( + { + ...logContext, + execution: "local", + durationMs: Date.now() - startedAt, + }, + "AI tagging completed", + ); + return result; } catch (error) { - logger.error({ err: error, input }, "AI tagging failed"); + logger.error( + { + err: error, + ...logContext, + durationMs: Date.now() - startedAt, + }, + "AI tagging failed", + ); const message = error instanceof Error ? error.message : "Unknown error"; throw new ORPCError("UNPROCESSABLE_CONTENT", { @@ -443,6 +490,15 @@ export const aiRouter = { mediaSourceId: input.mediaSourceId, payload: { mediaId: input.mediaId, force: input.force }, }); + logger.info( + { + jobId: job.id, + mediaSourceId: input.mediaSourceId, + mediaId: input.mediaId, + force: input.force, + }, + "CCIP vector extraction queued", + ); return { success: true, message: "CCIP vector extraction queued", @@ -499,7 +555,9 @@ export const aiRouter = { where: and( inArray(medias.id, input.mediaIds), eq(medias.mediaType, "image"), - input.mediaSourceId ? eq(medias.mediaSourceId, input.mediaSourceId) : undefined, + input.mediaSourceId + ? eq(medias.mediaSourceId, input.mediaSourceId) + : undefined, ), columns: { id: true, mediaSourceId: true }, }); @@ -550,6 +608,21 @@ export const aiRouter = { ]), ) .handler(async ({ input }) => { + const startedAt = Date.now(); + const logContext = + "file" in input + ? { + inputType: "file", + fileName: input.file.name, + fileSize: input.file.size, + transparent: input.transparent, + } + : { + inputType: "media", + mediaId: input.mediaId, + transparent: input.transparent, + }; + logger.info(logContext, "Character detection and cropping started"); try { const transparent = input.transparent ?? false; @@ -570,6 +643,14 @@ export const aiRouter = { ), ); + logger.info( + { + ...logContext, + detectionCount: resultDetections.length, + durationMs: Date.now() - startedAt, + }, + "Character detection and cropping completed", + ); return { detections: resultDetections }; } finally { await Bun.file(tmpPath) @@ -608,7 +689,7 @@ export const aiRouter = { const config = services.getConfigService().getConfig(); if (remoteUrl && !isRemoteServerLocal(remoteUrl)) { const fileBuffer = await readFileBuffer(fullPath); - return (await callRemoteCrop( + const result = (await callRemoteCrop( remoteUrl, fileBuffer, path.basename(fullPath), @@ -624,6 +705,16 @@ export const aiRouter = { height: number; }>; }; + logger.info( + { + ...logContext, + execution: "remote", + detectionCount: result.detections.length, + durationMs: Date.now() - startedAt, + }, + "Character detection and cropping completed", + ); + return result; } const { detectPerson } = await import("dghs-imgutils-rs"); @@ -635,10 +726,23 @@ export const aiRouter = { ), ); + logger.info( + { + ...logContext, + execution: "local", + detectionCount: resultDetections.length, + durationMs: Date.now() - startedAt, + }, + "Character detection and cropping completed", + ); return { detections: resultDetections }; } catch (error) { logger.error( - { err: error, input }, + { + err: error, + ...logContext, + durationMs: Date.now() - startedAt, + }, "Character detection and cropping failed", ); const message = diff --git a/apps/server/src/infrastructure/api/routers/media-router.ts b/apps/server/src/infrastructure/api/routers/media-router.ts index 3f70cdae8..e891830d0 100644 --- a/apps/server/src/infrastructure/api/routers/media-router.ts +++ b/apps/server/src/infrastructure/api/routers/media-router.ts @@ -43,11 +43,45 @@ export const mediaRouter = { .input(similarMediaRequestSchema) .output(similarMediaSearchResponseSchema) .handler(async ({ input }) => { - return await ccipVectorService.searchSimilar( - input.anchorMediaId, - input.topK, - input.mediaSourceId, + const startedAt = Date.now(); + logger.info( + { + anchorMediaId: input.anchorMediaId, + mediaSourceId: input.mediaSourceId, + topK: input.topK, + }, + "Vector similarity search started", ); + try { + const result = await ccipVectorService.searchSimilar( + input.anchorMediaId, + input.topK, + input.mediaSourceId, + ); + logger.info( + { + anchorMediaId: input.anchorMediaId, + mediaSourceId: input.mediaSourceId, + topK: input.topK, + resultCount: result.media.length, + durationMs: Date.now() - startedAt, + }, + "Vector similarity search completed", + ); + return result; + } catch (error) { + logger.error( + { + err: error, + anchorMediaId: input.anchorMediaId, + mediaSourceId: input.mediaSourceId, + topK: input.topK, + durationMs: Date.now() - startedAt, + }, + "Vector similarity search failed", + ); + throw error; + } }), /** @@ -204,7 +238,10 @@ export const mediaRouter = { try { await ccipVectorService.delete(input.mediaId); } catch (err) { - logger.warn({ err, mediaId: input.mediaId }, "[MediaRouter] Vector delete failed after media delete"); + logger.warn( + { err, mediaId: input.mediaId }, + "[MediaRouter] Vector delete failed after media delete", + ); } return { success: true }; }), @@ -242,7 +279,10 @@ export const mediaRouter = { try { await ccipVectorService.delete(input.mediaId); } catch (err) { - logger.warn({ err, mediaId: input.mediaId }, "[MediaRouter] Vector delete failed after media move"); + logger.warn( + { err, mediaId: input.mediaId }, + "[MediaRouter] Vector delete failed after media move", + ); } return result; }), @@ -303,7 +343,10 @@ export const mediaRouter = { try { await ccipVectorService.delete(mediaId); } catch (err) { - logger.warn({ err, mediaId }, "[MediaRouter] Vector delete failed after bulk media delete"); + logger.warn( + { err, mediaId }, + "[MediaRouter] Vector delete failed after bulk media delete", + ); } }), ); diff --git a/apps/server/src/infrastructure/jobs/ccip-jobs.ts b/apps/server/src/infrastructure/jobs/ccip-jobs.ts index 9072cf915..dc10ae472 100644 --- a/apps/server/src/infrastructure/jobs/ccip-jobs.ts +++ b/apps/server/src/infrastructure/jobs/ccip-jobs.ts @@ -47,11 +47,22 @@ export async function processCcipExtractionJob(job: Job): Promise { throw new Error("CCIP extraction job is missing mediaSourceId"); } try { - await ccipVectorService.extract( + const result = await ccipVectorService.extract( job.mediaSourceId, payload.mediaId, payload.force, ); + logger.info( + { + jobId: job.id, + parentId: job.parentId, + mediaSourceId: job.mediaSourceId, + mediaId: payload.mediaId, + force: payload.force, + skipped: result.skipped, + }, + "CCIP vector extraction completed", + ); RealtimeEventBus.publishJob("job-completed", { jobId: job.id, message: "CCIP vector extraction completed", diff --git a/apps/server/src/infrastructure/jobs/job-worker.ts b/apps/server/src/infrastructure/jobs/job-worker.ts index e29843663..46d22922b 100644 --- a/apps/server/src/infrastructure/jobs/job-worker.ts +++ b/apps/server/src/infrastructure/jobs/job-worker.ts @@ -152,17 +152,48 @@ export class JobWorker { private async processJob(job: Job, lanceDbSyncKey?: string) { this.activeJobs++; const isAiJob = this.aiJobTypes.has(job.type); + const startedAt = Date.now(); if (isAiJob) { this.activeAiJobs++; } + logger.info( + { + jobId: job.id, + type: job.type, + mediaSourceId: job.mediaSourceId, + parentId: job.parentId, + isAiJob, + }, + "Job started", + ); try { await this.processor(job); await this.jobRepo.markAsCompleted(job.id, { success: true }); + logger.info( + { + jobId: job.id, + type: job.type, + mediaSourceId: job.mediaSourceId, + parentId: job.parentId, + durationMs: Date.now() - startedAt, + }, + "Job completed", + ); } catch (error) { const errorMessage = error instanceof Error ? error.message : String(error); - logger.error({ err: error, jobId: job.id }, "Job failed"); + logger.error( + { + err: error, + jobId: job.id, + type: job.type, + mediaSourceId: job.mediaSourceId, + parentId: job.parentId, + durationMs: Date.now() - startedAt, + }, + "Job failed", + ); await this.jobRepo.markAsFailed(job.id, errorMessage); } finally { this.activeJobs--; diff --git a/apps/server/src/infrastructure/jobs/tagging-jobs.ts b/apps/server/src/infrastructure/jobs/tagging-jobs.ts index 11460eae8..a44654ae1 100644 --- a/apps/server/src/infrastructure/jobs/tagging-jobs.ts +++ b/apps/server/src/infrastructure/jobs/tagging-jobs.ts @@ -37,9 +37,26 @@ export async function processAutoTaggingJob(job: Job): Promise { } try { - await taggingService.getTagsForMedia(mediaSourceId, mediaId, { - skipCache: force, - }); + const result = await taggingService.getTagsForMedia( + mediaSourceId, + mediaId, + { + skipCache: force, + }, + ); + logger.info( + { + jobId: job.id, + parentId, + mediaSourceId, + mediaId, + force: force ?? false, + tagCount: result ? Object.keys(result.general).length : 0, + characterCount: result ? Object.keys(result.character).length : 0, + ipCount: result?.ips.length ?? 0, + }, + "Auto tagging completed", + ); await services.getJobRepository().createIfUnique({ type: "sync_lancedb_delta", mediaSourceId, diff --git a/apps/server/src/routes/api/rpc.$.ts b/apps/server/src/routes/api/rpc.$.ts index b2cd5024b..a21fda7d8 100644 --- a/apps/server/src/routes/api/rpc.$.ts +++ b/apps/server/src/routes/api/rpc.$.ts @@ -2,6 +2,7 @@ import { RPCHandler } from "@orpc/server/fetch"; import { createFileRoute } from "@tanstack/solid-router"; import { appRouter } from "~/domain/shared/api-contract"; import { bootstrap } from "~/infrastructure/bootstrap"; +import { logger } from "~/infrastructure/logger"; const handler = new RPCHandler(appRouter); @@ -14,7 +15,14 @@ export const Route = createFileRoute("/api/rpc/$")({ prefix: "/api/rpc", context: {}, }); - return response ?? new Response("Not Found", { status: 404 }); + if (response) { + return response; + } + logger.warn( + { method: request.method, url: request.url }, + "Unmatched RPC request", + ); + return new Response("Not Found", { status: 404 }); }, }, }, diff --git a/packages/ui/src/search-control-panel.tsx b/packages/ui/src/search-control-panel.tsx index df0382fa0..44615e4df 100644 --- a/packages/ui/src/search-control-panel.tsx +++ b/packages/ui/src/search-control-panel.tsx @@ -19,6 +19,7 @@ import { } from "./select"; import { SortControls } from "./sort-controls"; import { + clearVectorSearchAnchor, searchState, setSearchMode, setSearchState, @@ -161,6 +162,16 @@ export function SearchControlPanel(props: SearchControlPanelProps) { {searchState.similarityAnchorMediaId ?? "メディア個別画面の「Find Similar」から選択してください。"} + + +
diff --git a/packages/ui/src/stores/search-store.ts b/packages/ui/src/stores/search-store.ts index 5c812ddaa..3f8e351ab 100644 --- a/packages/ui/src/stores/search-store.ts +++ b/packages/ui/src/stores/search-store.ts @@ -61,5 +61,13 @@ export const activateVectorSearch = (mediaId: string) => { } }; +export const clearVectorSearchAnchor = () => { + setSearchState({ + similarityAnchorMediaId: null, + offset: 0, + scrollY: 0, + }); +}; + export const getSearchCondition = () => getSearchConditionFromState(searchState); From 032f3e590b305aac6e85adb2bb1d9527529bcf51 Mon Sep 17 00:00:00 2001 From: hmjn Date: Tue, 30 Jun 2026 00:44:40 +0900 Subject: [PATCH 02/19] fix: correct mock return values in ccip-jobs and tagging-jobs tests --- .../src/tests/unit/infrastructure/jobs/ccip-jobs.test.ts | 5 +++-- .../src/tests/unit/infrastructure/jobs/tagging-jobs.test.ts | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/apps/server/src/tests/unit/infrastructure/jobs/ccip-jobs.test.ts b/apps/server/src/tests/unit/infrastructure/jobs/ccip-jobs.test.ts index d62b4f3c1..c8d4d6049 100644 --- a/apps/server/src/tests/unit/infrastructure/jobs/ccip-jobs.test.ts +++ b/apps/server/src/tests/unit/infrastructure/jobs/ccip-jobs.test.ts @@ -40,6 +40,7 @@ vi.mock("~/infrastructure/events/realtime-event-bus", () => ({ vi.mock("~/infrastructure/logger", () => ({ logger: { + info: vi.fn(), error: (...args: Parameters) => loggerError(...args), }, })); @@ -50,7 +51,7 @@ describe("processCcipExtractionJob", () => { }); it("publishes child completion and completes the parent once", async () => { - extract.mockResolvedValue(undefined); + extract.mockResolvedValue({ record: { mediaId: "00000000-0000-4000-8000-000000000030" }, skipped: false }); vi.mocked(jobRepository.incrementProgress).mockResolvedValue(true); vi.mocked(jobRepository.findById).mockResolvedValue({ id: "00000000-0000-4000-8000-000000000010", @@ -106,7 +107,7 @@ describe("processCcipExtractionJob", () => { }); it("skips parent progress when the child was already counted", async () => { - extract.mockResolvedValue(undefined); + extract.mockResolvedValue({ record: { mediaId: "00000000-0000-4000-8000-000000000031" }, skipped: false }); vi.mocked(jobRepository.incrementProgress).mockResolvedValue(false); await processCcipExtractionJob({ diff --git a/apps/server/src/tests/unit/infrastructure/jobs/tagging-jobs.test.ts b/apps/server/src/tests/unit/infrastructure/jobs/tagging-jobs.test.ts index 9e4889447..42017d384 100644 --- a/apps/server/src/tests/unit/infrastructure/jobs/tagging-jobs.test.ts +++ b/apps/server/src/tests/unit/infrastructure/jobs/tagging-jobs.test.ts @@ -58,7 +58,7 @@ vi.mock("~/infrastructure/logger", () => ({ describe("processAutoTaggingJob", () => { beforeEach(() => { vi.clearAllMocks(); - getTagsForMedia.mockResolvedValue({}); + getTagsForMedia.mockResolvedValue({ general: {}, character: {}, ips: [], ips_mapping: {} }); createIfUnique.mockResolvedValue(null); incrementProgress.mockResolvedValue(false); }); From 38f3159aa96f36df18903bcd08b143d59c26d303 Mon Sep 17 00:00:00 2001 From: hmjn Date: Tue, 30 Jun 2026 01:05:16 +0900 Subject: [PATCH 03/19] perf(backup): use existing LanceDB cache for dump instead of rebuilding from DB createDump lancedb mode now checks for an existing cache directory (.cache/lancedb-cache/source-{id}/). When found, the cache is tarred directly without DB queries or writeToLanceDB rebuild. Falls back to the original rebuild path when cache is unavailable. --- .../application/services/backup-service.ts | 67 ++++++++++++++++++- .../services/lancedb-dump-service.ts | 1 + .../src/ports/lancedb-dump-service.ts | 3 + .../src/services/lancedb-dump-service.ts | 14 ++++ 4 files changed, 82 insertions(+), 3 deletions(-) diff --git a/apps/server/src/application/services/backup-service.ts b/apps/server/src/application/services/backup-service.ts index 02df22314..9f6d26e84 100644 --- a/apps/server/src/application/services/backup-service.ts +++ b/apps/server/src/application/services/backup-service.ts @@ -1524,6 +1524,70 @@ export const BackupService = { } if (targetMode === "lancedb") { + const { services } = await import("~/application/registry"); + const config = services.getConfigService().getConfig(); + const baseCacheDir = config.lancedb?.cacheDir ?? ".cache/lancedb-cache"; + const cacheDir = path.resolve( + process.cwd(), + baseCacheDir, + `source-${mediaSourceId}`, + ); + + const includeImages = options?.includeImages ?? true; + const archiverMod = await importArchiverModule(); + const { PassThrough } = await import("node:stream"); + + let cacheExists = false; + try { + await fs.access(cacheDir); + cacheExists = true; + } catch { + // cache does not exist + } + + if (cacheExists) { + const { readMediaFilePaths } = await import( + "~/application/services/lancedb-dump-service" + ); + const driver = getDriver(mediaSource); + const mediaFilePaths = await readMediaFilePaths(cacheDir); + + const passThrough = new PassThrough(); + const archive = new archiverMod.TarArchive(); + archive.pipe(passThrough); + + (async () => { + try { + archive.directory(cacheDir, false); + + if (includeImages) { + for (const item of mediaFilePaths) { + if (!item.filePath) continue; + try { + const buffer = await driver.get(item.filePath); + archive.append(buffer, { + name: `images/${item.filePath}`, + }); + } catch { + // ignore missing files + } + } + } + + await archive.finalize(); + } catch (err) { + logger.error( + { err }, + "Error generating LanceDB TAR dump from cache", + ); + archive.abort(); + } + })(); + + return nodeStreamToWebReadable(passThrough); + } + + // Fallback: rebuild from DB when cache is unavailable const { writeToLanceDB, cleanupLanceDBDir } = await import( "~/application/services/lancedb-dump-service" ); @@ -1533,7 +1597,6 @@ export const BackupService = { "lancedb-dump", `source-${mediaSourceId}`, ); - const includeImages = options?.includeImages ?? true; const dumpedItems: MediaDumpItem[] = []; const limit = 1000; let lastId: string | null = null; @@ -1574,8 +1637,6 @@ export const BackupService = { tempDir: tempBaseDir, }); const driver = getDriver(mediaSource); - const archiverMod = await importArchiverModule(); - const { PassThrough } = await import("node:stream"); const passThrough = new PassThrough(); const archive = new archiverMod.TarArchive(); diff --git a/apps/server/src/application/services/lancedb-dump-service.ts b/apps/server/src/application/services/lancedb-dump-service.ts index 5cca837e1..67e2d0ab4 100644 --- a/apps/server/src/application/services/lancedb-dump-service.ts +++ b/apps/server/src/application/services/lancedb-dump-service.ts @@ -20,4 +20,5 @@ export const syncLanceDBPages = service.syncLanceDBPages; export const syncLanceDBDelta = service.syncLanceDBDelta; export const readFromLanceDB = service.readFromLanceDB; export const readMediaIds = service.readMediaIds; +export const readMediaFilePaths = service.readMediaFilePaths; export const cleanupLanceDBDir = service.cleanupLanceDBDir; diff --git a/packages/application/src/ports/lancedb-dump-service.ts b/packages/application/src/ports/lancedb-dump-service.ts index 43f1645d3..169675dc9 100644 --- a/packages/application/src/ports/lancedb-dump-service.ts +++ b/packages/application/src/ports/lancedb-dump-service.ts @@ -59,5 +59,8 @@ export interface ILanceDbDumpService { options?: ReadOptions, ): Promise; readMediaIds(lanceDbDir: string): Promise; + readMediaFilePaths( + lanceDbDir: string, + ): Promise<{ id: string; filePath: string | undefined }[]>; cleanupLanceDBDir(dir: string): Promise; } diff --git a/packages/application/src/services/lancedb-dump-service.ts b/packages/application/src/services/lancedb-dump-service.ts index d73110dd6..a982beae2 100644 --- a/packages/application/src/services/lancedb-dump-service.ts +++ b/packages/application/src/services/lancedb-dump-service.ts @@ -856,6 +856,19 @@ export function createLanceDbDumpService(deps?: { }); } + async function readMediaFilePaths( + lanceDbDir: string, + ): Promise<{ id: string; filePath: string | undefined }[]> { + const connect = await getConnect(); + const db = await connect(lanceDbDir); + const mediaTable = await db.openTable("media"); + const rows = await mediaTable.query().select(["id", "filePath"]).toArray(); + return rows.flatMap((row) => { + const id = safeString(row.id); + return id ? [{ id, filePath: safeString(row.filePath) }] : []; + }); + } + async function cleanupLanceDBDir(dir: string): Promise { await fs.rm(dir, { recursive: true, force: true }); } @@ -867,6 +880,7 @@ export function createLanceDbDumpService(deps?: { syncLanceDBDelta, readFromLanceDB, readMediaIds, + readMediaFilePaths, cleanupLanceDBDir, }; } From 21a822a916045108a715ca96564311f20321b621 Mon Sep 17 00:00:00 2001 From: hmjn Date: Tue, 30 Jun 2026 01:10:26 +0900 Subject: [PATCH 04/19] fix(backup): address review feedback for LanceDB cache dump - Check manifest.json instead of just directory existence - Wrap cache path in try-catch for safe fallback to DB rebuild - Parallelize image downloads with Promise.all - Only readMediaFilePaths when includeImages is true --- .../application/services/backup-service.ts | 91 ++++++++++--------- 1 file changed, 50 insertions(+), 41 deletions(-) diff --git a/apps/server/src/application/services/backup-service.ts b/apps/server/src/application/services/backup-service.ts index 9f6d26e84..10c7f2960 100644 --- a/apps/server/src/application/services/backup-service.ts +++ b/apps/server/src/application/services/backup-service.ts @@ -1537,54 +1537,63 @@ export const BackupService = { const archiverMod = await importArchiverModule(); const { PassThrough } = await import("node:stream"); - let cacheExists = false; + let cacheValid = false; try { - await fs.access(cacheDir); - cacheExists = true; + await fs.access(path.join(cacheDir, "manifest.json")); + cacheValid = true; } catch { - // cache does not exist + // cache or manifest does not exist } - if (cacheExists) { - const { readMediaFilePaths } = await import( - "~/application/services/lancedb-dump-service" - ); - const driver = getDriver(mediaSource); - const mediaFilePaths = await readMediaFilePaths(cacheDir); - - const passThrough = new PassThrough(); - const archive = new archiverMod.TarArchive(); - archive.pipe(passThrough); - - (async () => { - try { - archive.directory(cacheDir, false); - - if (includeImages) { - for (const item of mediaFilePaths) { - if (!item.filePath) continue; - try { - const buffer = await driver.get(item.filePath); - archive.append(buffer, { - name: `images/${item.filePath}`, - }); - } catch { - // ignore missing files - } + if (cacheValid) { + try { + const driver = getDriver(mediaSource); + const passThrough = new PassThrough(); + const archive = new archiverMod.TarArchive(); + archive.pipe(passThrough); + + (async () => { + try { + archive.directory(cacheDir, false); + + if (includeImages) { + const { readMediaFilePaths } = await import( + "~/application/services/lancedb-dump-service" + ); + const mediaFilePaths = await readMediaFilePaths(cacheDir); + + await Promise.all( + mediaFilePaths.map(async (item) => { + if (!item.filePath) return; + try { + const buffer = await driver.get(item.filePath); + archive.append(buffer, { + name: `images/${item.filePath}`, + }); + } catch { + // ignore missing files + } + }), + ); } - } - await archive.finalize(); - } catch (err) { - logger.error( - { err }, - "Error generating LanceDB TAR dump from cache", - ); - archive.abort(); - } - })(); + await archive.finalize(); + } catch (err) { + logger.error( + { err }, + "Error generating LanceDB TAR dump from cache", + ); + archive.abort(); + } + })(); - return nodeStreamToWebReadable(passThrough); + return nodeStreamToWebReadable(passThrough); + } catch (err) { + logger.warn( + { err }, + "Failed to read LanceDB cache, falling back to DB rebuild", + ); + } } // Fallback: rebuild from DB when cache is unavailable From 7e096c0674278fb09e3b51345160e7edc2b39c4b Mon Sep 17 00:00:00 2001 From: hmjn Date: Wed, 1 Jul 2026 00:14:06 +0900 Subject: [PATCH 05/19] feat(xtracter): add pixivFANBOX support Add FANBOX image extraction, platform-scoped author accounts, metadata persistence across JSON and LanceDB, and deterministic media filenames. --- apps/server/drizzle/0017_ambitious_gideon.sql | 22 + apps/server/drizzle/meta/0017_snapshot.json | 3009 +++++++++++++++++ apps/server/drizzle/meta/_journal.json | 7 + .../application/services/backup-service.ts | 55 +- .../src/infrastructure/jobs/download-jobs.ts | 2 + .../integration/backup/backup-service.test.ts | 25 +- .../repository/author-dedupe.test.ts | 45 +- .../services/lancedb-dump-service.test.ts | 65 +- .../services/media-processing-service.test.ts | 2 +- .../tests/unit/domain/media/schemas.test.ts | 19 +- .../infrastructure/jobs/download-jobs.test.ts | 4 +- apps/xtracter/README.md | 4 +- apps/xtracter/public/manifest.json | 3 + apps/xtracter/src/content/danbooru.ts | 12 +- apps/xtracter/src/content/fanbox.ts | 111 + apps/xtracter/src/content/index.ts | 3 + apps/xtracter/src/content/twitter.ts | 1 + apps/xtracter/src/schema.ts | 1 + .../src/services/lancedb-dump-service.ts | 23 +- .../src/services/media-processing-service.ts | 3 +- packages/core/src/domain/authors/schemas.ts | 13 + packages/core/src/domain/media/schemas.ts | 10 + .../src/domain/media/utils/filename-utils.ts | 101 +- .../domain/repositories/author-repository.ts | 4 +- packages/core/tests/filename-utils.test.ts | 173 +- .../db/src/repositories/author-repository.ts | 99 +- packages/db/src/schema.ts | 40 + 27 files changed, 3688 insertions(+), 168 deletions(-) create mode 100644 apps/server/drizzle/0017_ambitious_gideon.sql create mode 100644 apps/server/drizzle/meta/0017_snapshot.json create mode 100644 apps/xtracter/src/content/fanbox.ts diff --git a/apps/server/drizzle/0017_ambitious_gideon.sql b/apps/server/drizzle/0017_ambitious_gideon.sql new file mode 100644 index 000000000..b153fdda8 --- /dev/null +++ b/apps/server/drizzle/0017_ambitious_gideon.sql @@ -0,0 +1,22 @@ +CREATE TYPE "public"."author_platform" AS ENUM('twitter', 'pixiv-fanbox', 'danbooru');--> statement-breakpoint +CREATE TABLE "author_accounts" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "author_id" uuid NOT NULL, + "platform" "author_platform" NOT NULL, + "account_id" text NOT NULL, + "profile_url" text, + "created_at" timestamp DEFAULT now() NOT NULL, + "updated_at" timestamp DEFAULT now() NOT NULL +); +--> statement-breakpoint +ALTER TABLE "author_accounts" ADD CONSTRAINT "author_accounts_author_id_authors_id_fk" FOREIGN KEY ("author_id") REFERENCES "public"."authors"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +CREATE INDEX "idx_author_accounts_author_id" ON "author_accounts" USING btree ("author_id");--> statement-breakpoint +CREATE UNIQUE INDEX "idx_author_accounts_platform_account_unique" ON "author_accounts" USING btree ("platform","account_id");--> statement-breakpoint +INSERT INTO "author_accounts" ("author_id", "platform", "account_id") +SELECT DISTINCT ON ("account_id") + "id", + 'twitter'::"author_platform", + "account_id" +FROM "authors" +WHERE "account_id" IS NOT NULL AND "account_id" <> '' +ORDER BY "account_id", "created_at", "id"; diff --git a/apps/server/drizzle/meta/0017_snapshot.json b/apps/server/drizzle/meta/0017_snapshot.json new file mode 100644 index 000000000..f13b519b0 --- /dev/null +++ b/apps/server/drizzle/meta/0017_snapshot.json @@ -0,0 +1,3009 @@ +{ + "id": "1928bf68-5088-4c28-9755-00116d14c19c", + "prevId": "ec9e26e5-be2a-4d22-a83c-b8a89b260acf", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.author_accounts": { + "name": "author_accounts", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "author_id": { + "name": "author_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "platform": { + "name": "platform", + "type": "author_platform", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "profile_url": { + "name": "profile_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "idx_author_accounts_author_id": { + "name": "idx_author_accounts_author_id", + "columns": [ + { + "expression": "author_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_author_accounts_platform_account_unique": { + "name": "idx_author_accounts_platform_account_unique", + "columns": [ + { + "expression": "platform", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "account_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "author_accounts_author_id_authors_id_fk": { + "name": "author_accounts_author_id_authors_id_fk", + "tableFrom": "author_accounts", + "tableTo": "authors", + "columnsFrom": [ + "author_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.authors": { + "name": "authors", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "idx_authors_account_id": { + "name": "idx_authors_account_id", + "columns": [ + { + "expression": "account_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_authors_name": { + "name": "idx_authors_name", + "columns": [ + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.categories": { + "name": "categories", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "default": "''" + }, + "color": { + "name": "color", + "type": "text", + "primaryKey": false, + "notNull": false, + "default": "'#808080'" + }, + "parent_id": { + "name": "parent_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'manual'" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "categories_parent_id_categories_id_fk": { + "name": "categories_parent_id_categories_id_fk", + "tableFrom": "categories", + "tableTo": "categories", + "columnsFrom": [ + "parent_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "categories_name_unique": { + "name": "categories_name_unique", + "nullsNotDistinct": false, + "columns": [ + "name" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.character_ips": { + "name": "character_ips", + "schema": "", + "columns": { + "character_id": { + "name": "character_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "ip_id": { + "name": "ip_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'manual'" + } + }, + "indexes": { + "idx_character_ips_ip_id_character_id": { + "name": "idx_character_ips_ip_id_character_id", + "columns": [ + { + "expression": "ip_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "character_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "character_ips_character_id_characters_id_fk": { + "name": "character_ips_character_id_characters_id_fk", + "tableFrom": "character_ips", + "tableTo": "characters", + "columnsFrom": [ + "character_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "character_ips_ip_id_ips_id_fk": { + "name": "character_ips_ip_id_ips_id_fk", + "tableFrom": "character_ips", + "tableTo": "ips", + "columnsFrom": [ + "ip_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "character_ips_character_id_ip_id_pk": { + "name": "character_ips_character_id_ip_id_pk", + "columns": [ + "character_id", + "ip_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.characters": { + "name": "characters", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "default": "''" + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'manual'" + }, + "aliases": { + "name": "aliases", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "characters_name_unique": { + "name": "characters_name_unique", + "nullsNotDistinct": false, + "columns": [ + "name" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.collections": { + "name": "collections", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "default": "''" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "collections_user_id_users_id_fk": { + "name": "collections_user_id_users_id_fk", + "tableFrom": "collections", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.ips": { + "name": "ips", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "default": "''" + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'manual'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "ips_name_unique": { + "name": "ips_name_unique", + "nullsNotDistinct": false, + "columns": [ + "name" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.jobs": { + "name": "jobs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_id": { + "name": "source_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "job_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "result": { + "name": "result", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "parent_id": { + "name": "parent_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "idx_jobs_pending_created": { + "name": "idx_jobs_pending_created", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"jobs\".\"status\" = 'pending' AND \"jobs\".\"type\" <> 'import_request'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_jobs_pending_type_created": { + "name": "idx_jobs_pending_type_created", + "columns": [ + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"jobs\".\"status\" = 'pending' AND \"jobs\".\"type\" <> 'import_request'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_jobs_pending_lancedb_source": { + "name": "idx_jobs_pending_lancedb_source", + "columns": [ + { + "expression": "source_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"jobs\".\"status\" = 'pending'\n\t\t\t\t\tAND \"jobs\".\"type\" IN ('sync_lancedb', 'sync_lancedb_full', 'sync_lancedb_delta')\n\t\t\t\t\tAND \"jobs\".\"source_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_jobs_active_lancedb_source": { + "name": "idx_jobs_active_lancedb_source", + "columns": [ + { + "expression": "source_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"jobs\".\"status\" = 'in_progress'\n\t\t\t\t\tAND \"jobs\".\"type\" IN ('sync_lancedb', 'sync_lancedb_full', 'sync_lancedb_delta')\n\t\t\t\t\tAND \"jobs\".\"source_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "jobs_source_id_media_sources_id_fk": { + "name": "jobs_source_id_media_sources_id_fk", + "tableFrom": "jobs", + "tableTo": "media_sources", + "columnsFrom": [ + "source_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "jobs_parent_id_jobs_id_fk": { + "name": "jobs_parent_id_jobs_id_fk", + "tableFrom": "jobs", + "tableTo": "jobs", + "columnsFrom": [ + "parent_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.lancedb_sync_dirty": { + "name": "lancedb_sync_dirty", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "source_id": { + "name": "source_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "media_id": { + "name": "media_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "operation": { + "name": "operation", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'upsert'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "idx_lancedb_sync_dirty_source_updated": { + "name": "idx_lancedb_sync_dirty_source_updated", + "columns": [ + { + "expression": "source_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "updated_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "lancedb_sync_dirty_source_id_media_sources_id_fk": { + "name": "lancedb_sync_dirty_source_id_media_sources_id_fk", + "tableFrom": "lancedb_sync_dirty", + "tableTo": "media_sources", + "columnsFrom": [ + "source_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "lancedb_sync_dirty_source_media_unique": { + "name": "lancedb_sync_dirty_source_media_unique", + "nullsNotDistinct": false, + "columns": [ + "source_id", + "media_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.media_authors": { + "name": "media_authors", + "schema": "", + "columns": { + "media_id": { + "name": "media_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "author_id": { + "name": "author_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "idx_media_authors_author_id_media_id": { + "name": "idx_media_authors_author_id_media_id", + "columns": [ + { + "expression": "author_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "media_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "media_authors_media_id_media_id_fk": { + "name": "media_authors_media_id_media_id_fk", + "tableFrom": "media_authors", + "tableTo": "media", + "columnsFrom": [ + "media_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "media_authors_author_id_authors_id_fk": { + "name": "media_authors_author_id_authors_id_fk", + "tableFrom": "media_authors", + "tableTo": "authors", + "columnsFrom": [ + "author_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "media_authors_media_id_author_id_pk": { + "name": "media_authors_media_id_author_id_pk", + "columns": [ + "media_id", + "author_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.media_categories": { + "name": "media_categories", + "schema": "", + "columns": { + "media_id": { + "name": "media_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "category_id": { + "name": "category_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "idx_media_categories_category_id_media_id": { + "name": "idx_media_categories_category_id_media_id", + "columns": [ + { + "expression": "category_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "media_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "media_categories_media_id_media_id_fk": { + "name": "media_categories_media_id_media_id_fk", + "tableFrom": "media_categories", + "tableTo": "media", + "columnsFrom": [ + "media_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "media_categories_category_id_categories_id_fk": { + "name": "media_categories_category_id_categories_id_fk", + "tableFrom": "media_categories", + "tableTo": "categories", + "columnsFrom": [ + "category_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "media_categories_media_id_category_id_pk": { + "name": "media_categories_media_id_category_id_pk", + "columns": [ + "media_id", + "category_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.media_characters": { + "name": "media_characters", + "schema": "", + "columns": { + "media_id": { + "name": "media_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "character_id": { + "name": "character_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "confidence": { + "name": "confidence", + "type": "real", + "primaryKey": false, + "notNull": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'manual'" + } + }, + "indexes": { + "idx_media_characters_character_id_media_id": { + "name": "idx_media_characters_character_id_media_id", + "columns": [ + { + "expression": "character_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "media_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "media_characters_media_id_media_id_fk": { + "name": "media_characters_media_id_media_id_fk", + "tableFrom": "media_characters", + "tableTo": "media", + "columnsFrom": [ + "media_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "media_characters_character_id_characters_id_fk": { + "name": "media_characters_character_id_characters_id_fk", + "tableFrom": "media_characters", + "tableTo": "characters", + "columnsFrom": [ + "character_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "media_characters_media_id_character_id_pk": { + "name": "media_characters_media_id_character_id_pk", + "columns": [ + "media_id", + "character_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.media_collections": { + "name": "media_collections", + "schema": "", + "columns": { + "collection_id": { + "name": "collection_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "media_id": { + "name": "media_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "display_order": { + "name": "display_order", + "type": "integer", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "idx_media_collections_media_id": { + "name": "idx_media_collections_media_id", + "columns": [ + { + "expression": "media_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "media_collections_collection_id_collections_id_fk": { + "name": "media_collections_collection_id_collections_id_fk", + "tableFrom": "media_collections", + "tableTo": "collections", + "columnsFrom": [ + "collection_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "media_collections_media_id_media_id_fk": { + "name": "media_collections_media_id_media_id_fk", + "tableFrom": "media_collections", + "tableTo": "media", + "columnsFrom": [ + "media_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "media_collections_collection_id_media_id_pk": { + "name": "media_collections_collection_id_media_id_pk", + "columns": [ + "collection_id", + "media_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.media_details": { + "name": "media_details", + "schema": "", + "columns": { + "media_id": { + "name": "media_id", + "type": "uuid", + "primaryKey": true, + "notNull": true + }, + "rating": { + "name": "rating", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "favorite": { + "name": "favorite", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "view_count": { + "name": "view_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "last_viewed_at": { + "name": "last_viewed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "'1970-01-01 00:00:00'" + } + }, + "indexes": { + "idx_media_details_rating": { + "name": "idx_media_details_rating", + "columns": [ + { + "expression": "rating", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_media_details_favorite": { + "name": "idx_media_details_favorite", + "columns": [ + { + "expression": "favorite", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_media_details_view_count": { + "name": "idx_media_details_view_count", + "columns": [ + { + "expression": "view_count", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "media_details_media_id_media_id_fk": { + "name": "media_details_media_id_media_id_fk", + "tableFrom": "media_details", + "tableTo": "media", + "columnsFrom": [ + "media_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.media_generation_info": { + "name": "media_generation_info", + "schema": "", + "columns": { + "media_id": { + "name": "media_id", + "type": "uuid", + "primaryKey": true, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "prompt": { + "name": "prompt", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "negative_prompt": { + "name": "negative_prompt", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workflow": { + "name": "workflow", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "loras": { + "name": "loras", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "vae": { + "name": "vae", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "hypernetworks": { + "name": "hypernetworks", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "embeddings": { + "name": "embeddings", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "ai_generated": { + "name": "ai_generated", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "model_name": { + "name": "model_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "default": "''" + }, + "seed": { + "name": "seed", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": -1 + }, + "cfg_scale": { + "name": "cfg_scale", + "type": "real", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "steps": { + "name": "steps", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + } + }, + "indexes": { + "idx_media_generation_info_metadata": { + "name": "idx_media_generation_info_metadata", + "columns": [ + { + "expression": "metadata", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_media_generation_info_ai_generated": { + "name": "idx_media_generation_info_ai_generated", + "columns": [ + { + "expression": "ai_generated", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_media_generation_info_model_name": { + "name": "idx_media_generation_info_model_name", + "columns": [ + { + "expression": "model_name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "media_generation_info_media_id_media_id_fk": { + "name": "media_generation_info_media_id_media_id_fk", + "tableFrom": "media_generation_info", + "tableTo": "media", + "columnsFrom": [ + "media_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.media_ips": { + "name": "media_ips", + "schema": "", + "columns": { + "media_id": { + "name": "media_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "ip_id": { + "name": "ip_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "confidence": { + "name": "confidence", + "type": "real", + "primaryKey": false, + "notNull": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'manual'" + } + }, + "indexes": { + "idx_media_ips_ip_id_media_id": { + "name": "idx_media_ips_ip_id_media_id", + "columns": [ + { + "expression": "ip_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "media_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "media_ips_media_id_media_id_fk": { + "name": "media_ips_media_id_media_id_fk", + "tableFrom": "media_ips", + "tableTo": "media", + "columnsFrom": [ + "media_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "media_ips_ip_id_ips_id_fk": { + "name": "media_ips_ip_id_ips_id_fk", + "tableFrom": "media_ips", + "tableTo": "ips", + "columnsFrom": [ + "ip_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "media_ips_media_id_ip_id_pk": { + "name": "media_ips_media_id_ip_id_pk", + "columns": [ + "media_id", + "ip_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.media_projects": { + "name": "media_projects", + "schema": "", + "columns": { + "media_id": { + "name": "media_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "project_id": { + "name": "project_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "idx_media_projects_project_id_media_id": { + "name": "idx_media_projects_project_id_media_id", + "columns": [ + { + "expression": "project_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "media_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "media_projects_media_id_media_id_fk": { + "name": "media_projects_media_id_media_id_fk", + "tableFrom": "media_projects", + "tableTo": "media", + "columnsFrom": [ + "media_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "media_projects_project_id_projects_id_fk": { + "name": "media_projects_project_id_projects_id_fk", + "tableFrom": "media_projects", + "tableTo": "projects", + "columnsFrom": [ + "project_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "media_projects_media_id_project_id_pk": { + "name": "media_projects_media_id_project_id_pk", + "columns": [ + "media_id", + "project_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.media_relations": { + "name": "media_relations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "parent_media_id": { + "name": "parent_media_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "child_media_id": { + "name": "child_media_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "relation_type": { + "name": "relation_type", + "type": "media_relation_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "order_index": { + "name": "order_index", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "idx_media_relations_child": { + "name": "idx_media_relations_child", + "columns": [ + { + "expression": "child_media_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_media_relations_type": { + "name": "idx_media_relations_type", + "columns": [ + { + "expression": "relation_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "media_relations_parent_media_id_media_id_fk": { + "name": "media_relations_parent_media_id_media_id_fk", + "tableFrom": "media_relations", + "tableTo": "media", + "columnsFrom": [ + "parent_media_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "media_relations_child_media_id_media_id_fk": { + "name": "media_relations_child_media_id_media_id_fk", + "tableFrom": "media_relations", + "tableTo": "media", + "columnsFrom": [ + "child_media_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "parent_child_type_unique": { + "name": "parent_child_type_unique", + "nullsNotDistinct": false, + "columns": [ + "parent_media_id", + "child_media_id", + "relation_type" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.media_sources": { + "name": "media_sources", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "type": { + "name": "type", + "type": "media_source_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "connection_info": { + "name": "connection_info", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.media_sync": { + "name": "media_sync", + "schema": "", + "columns": { + "media_id": { + "name": "media_id", + "type": "uuid", + "primaryKey": true, + "notNull": true + }, + "sync_status": { + "name": "sync_status", + "type": "media_sync_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": false, + "default": "'synced'" + }, + "backup_urls": { + "name": "backup_urls", + "type": "text[]", + "primaryKey": false, + "notNull": false, + "default": "'{}'" + }, + "last_synced_at": { + "name": "last_synced_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "sync_attempts": { + "name": "sync_attempts", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "media_sync_media_id_media_id_fk": { + "name": "media_sync_media_id_media_id_fk", + "tableFrom": "media_sync", + "tableTo": "media", + "columnsFrom": [ + "media_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.media_tags": { + "name": "media_tags", + "schema": "", + "columns": { + "media_id": { + "name": "media_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "tag_id": { + "name": "tag_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "tag_type": { + "name": "tag_type", + "type": "tag_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'positive'" + }, + "confidence": { + "name": "confidence", + "type": "real", + "primaryKey": false, + "notNull": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'manual'" + } + }, + "indexes": { + "idx_media_tags_tag_id_tag_type_media_id": { + "name": "idx_media_tags_tag_id_tag_type_media_id", + "columns": [ + { + "expression": "tag_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "tag_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "media_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "media_tags_media_id_media_id_fk": { + "name": "media_tags_media_id_media_id_fk", + "tableFrom": "media_tags", + "tableTo": "media", + "columnsFrom": [ + "media_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "media_tags_tag_id_tags_id_fk": { + "name": "media_tags_tag_id_tags_id_fk", + "tableFrom": "media_tags", + "tableTo": "tags", + "columnsFrom": [ + "tag_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "media_tags_media_id_tag_id_tag_type_pk": { + "name": "media_tags_media_id_tag_id_tag_type_pk", + "columns": [ + "media_id", + "tag_id", + "tag_type" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.media_technical_info": { + "name": "media_technical_info", + "schema": "", + "columns": { + "media_id": { + "name": "media_id", + "type": "uuid", + "primaryKey": true, + "notNull": true + }, + "color_profile": { + "name": "color_profile", + "type": "text", + "primaryKey": false, + "notNull": false, + "default": "''" + }, + "exif_data": { + "name": "exif_data", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'{}'" + }, + "hash_md5": { + "name": "hash_md5", + "type": "text", + "primaryKey": false, + "notNull": false, + "default": "''" + }, + "hash_perceptual": { + "name": "hash_perceptual", + "type": "text", + "primaryKey": false, + "notNull": false, + "default": "''" + }, + "duration_seconds": { + "name": "duration_seconds", + "type": "real", + "primaryKey": false, + "notNull": false + }, + "frame_rate": { + "name": "frame_rate", + "type": "real", + "primaryKey": false, + "notNull": false + }, + "bitrate_kbps": { + "name": "bitrate_kbps", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "video_codec": { + "name": "video_codec", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "audio_codec": { + "name": "audio_codec", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "idx_media_technical_info_hash_md5": { + "name": "idx_media_technical_info_hash_md5", + "columns": [ + { + "expression": "hash_md5", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "media_technical_info_media_id_media_id_fk": { + "name": "media_technical_info_media_id_media_id_fk", + "tableFrom": "media_technical_info", + "tableTo": "media", + "columnsFrom": [ + "media_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.media_urls": { + "name": "media_urls", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "media_id": { + "name": "media_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "idx_media_urls_media_id": { + "name": "idx_media_urls_media_id", + "columns": [ + { + "expression": "media_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_media_urls_url": { + "name": "idx_media_urls_url", + "columns": [ + { + "expression": "url", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_media_urls_media_id_url_unique": { + "name": "idx_media_urls_media_id_url_unique", + "columns": [ + { + "expression": "media_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "url", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "media_urls_media_id_media_id_fk": { + "name": "media_urls_media_id_media_id_fk", + "tableFrom": "media_urls", + "tableTo": "media", + "columnsFrom": [ + "media_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.media": { + "name": "media", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "source_id": { + "name": "source_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "file_path": { + "name": "file_path", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "file_name": { + "name": "file_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "media_type": { + "name": "media_type", + "type": "media_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "width": { + "name": "width", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "height": { + "name": "height", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "file_size": { + "name": "file_size", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "modified_at": { + "name": "modified_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "indexed_at": { + "name": "indexed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "status": { + "name": "status", + "type": "media_organization_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'active'" + } + }, + "indexes": { + "idx_media_source_id": { + "name": "idx_media_source_id", + "columns": [ + { + "expression": "source_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_media_file_size": { + "name": "idx_media_file_size", + "columns": [ + { + "expression": "file_size", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_media_file_name": { + "name": "idx_media_file_name", + "columns": [ + { + "expression": "file_name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_media_created_at": { + "name": "idx_media_created_at", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_media_description": { + "name": "idx_media_description", + "columns": [ + { + "expression": "description", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "media_source_id_media_sources_id_fk": { + "name": "media_source_id_media_sources_id_fk", + "tableFrom": "media", + "tableTo": "media_sources", + "columnsFrom": [ + "source_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "source_id_file_path_unique": { + "name": "source_id_file_path_unique", + "nullsNotDistinct": false, + "columns": [ + "source_id", + "file_path" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.presets": { + "name": "presets", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "sort": { + "name": "sort", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "order": { + "name": "order", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "mode": { + "name": "mode", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "presets_name_unique": { + "name": "presets_name_unique", + "nullsNotDistinct": false, + "columns": [ + "name" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.projects": { + "name": "projects", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "default": "''" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "archived_at": { + "name": "archived_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "idx_projects_name": { + "name": "idx_projects_name", + "columns": [ + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "projects_name_unique": { + "name": "projects_name_unique", + "nullsNotDistinct": false, + "columns": [ + "name" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.similar_media": { + "name": "similar_media", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "media1_id": { + "name": "media1_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "media2_id": { + "name": "media2_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "similarity_score": { + "name": "similarity_score", + "type": "real", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "algorithm": { + "name": "algorithm", + "type": "text", + "primaryKey": false, + "notNull": false, + "default": "'perceptual'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "idx_similar_media_score": { + "name": "idx_similar_media_score", + "columns": [ + { + "expression": "similarity_score", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "similar_media_media1_id_media_id_fk": { + "name": "similar_media_media1_id_media_id_fk", + "tableFrom": "similar_media", + "tableTo": "media", + "columnsFrom": [ + "media1_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "similar_media_media2_id_media_id_fk": { + "name": "similar_media_media2_id_media_id_fk", + "tableFrom": "similar_media", + "tableTo": "media", + "columnsFrom": [ + "media2_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "media1Id_media2Id_algorithm_unique": { + "name": "media1Id_media2Id_algorithm_unique", + "nullsNotDistinct": false, + "columns": [ + "media1_id", + "media2_id", + "algorithm" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.tags": { + "name": "tags", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "attribute": { + "name": "attribute", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "color": { + "name": "color", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'manual'" + }, + "author_id": { + "name": "author_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "idx_tags_author_id": { + "name": "idx_tags_author_id", + "columns": [ + { + "expression": "author_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "tags_author_id_authors_id_fk": { + "name": "tags_author_id_authors_id_fk", + "tableFrom": "tags", + "tableTo": "authors", + "columnsFrom": [ + "author_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "tags_name_unique": { + "name": "tags_name_unique", + "nullsNotDistinct": false, + "columns": [ + "name" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.users": { + "name": "users", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "users_email_unique": { + "name": "users_email_unique", + "nullsNotDistinct": false, + "columns": [ + "email" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.view_history": { + "name": "view_history", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "media_id": { + "name": "media_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "viewed_at": { + "name": "viewed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false, + "default": "''" + } + }, + "indexes": {}, + "foreignKeys": { + "view_history_media_id_media_id_fk": { + "name": "view_history_media_id_media_id_fk", + "tableFrom": "view_history", + "tableTo": "media", + "columnsFrom": [ + "media_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.author_platform": { + "name": "author_platform", + "schema": "public", + "values": [ + "twitter", + "pixiv-fanbox", + "danbooru" + ] + }, + "public.job_status": { + "name": "job_status", + "schema": "public", + "values": [ + "pending", + "in_progress", + "completed", + "failed" + ] + }, + "public.media_organization_status": { + "name": "media_organization_status", + "schema": "public", + "values": [ + "active", + "archived", + "deleted" + ] + }, + "public.media_relation_type": { + "name": "media_relation_type", + "schema": "public", + "values": [ + "variant", + "version", + "page", + "derivative", + "edit", + "source" + ] + }, + "public.media_source_type": { + "name": "media_source_type", + "schema": "public", + "values": [ + "local", + "sftp", + "s3" + ] + }, + "public.media_sync_status": { + "name": "media_sync_status", + "schema": "public", + "values": [ + "synced", + "pending", + "failed" + ] + }, + "public.media_type": { + "name": "media_type", + "schema": "public", + "values": [ + "image", + "video", + "audio" + ] + }, + "public.tag_type": { + "name": "tag_type", + "schema": "public", + "values": [ + "positive", + "negative" + ] + } + }, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/apps/server/drizzle/meta/_journal.json b/apps/server/drizzle/meta/_journal.json index d9897d5bb..958c43744 100644 --- a/apps/server/drizzle/meta/_journal.json +++ b/apps/server/drizzle/meta/_journal.json @@ -120,6 +120,13 @@ "when": 1782395330635, "tag": "0016_nebulous_blade", "breakpoints": true + }, + { + "idx": 17, + "version": "7", + "when": 1782830770400, + "tag": "0017_ambitious_gideon", + "breakpoints": true } ] } \ No newline at end of file diff --git a/apps/server/src/application/services/backup-service.ts b/apps/server/src/application/services/backup-service.ts index 10c7f2960..c12f414fb 100644 --- a/apps/server/src/application/services/backup-service.ts +++ b/apps/server/src/application/services/backup-service.ts @@ -1,6 +1,7 @@ import fs from "node:fs/promises"; import path from "node:path"; import { + type AuthorPlatform, type MediaDumpItem, mediaDumpItemSchema, } from "@solid-imager/core/domain/media/schemas"; @@ -11,6 +12,7 @@ import { and, asc, eq, gt, inArray, lt, sql } from "drizzle-orm"; import type { PgColumn } from "drizzle-orm/pg-core"; import { db } from "~/infrastructure/db"; import { + authorAccounts, authors, characterIps, characters, @@ -86,7 +88,14 @@ interface MediaListQueryItem { source: string; }[]; authors: { - author: { name: string; accountId: string | null }; + author: { + name: string; + accountId: string | null; + accounts?: { + platform: AuthorPlatform; + accountId: string; + }[]; + }; }[]; characters: { character: { @@ -394,6 +403,33 @@ export const BackupService = { authorData, _tx, ); + const restoredAuthorAccounts = new Map< + string, + { + authorId: string; + platform: AuthorPlatform; + accountId: string; + } + >(); + for (const item of validItems) { + for (const author of item.authors ?? []) { + if (!(author.platform && author.accountId)) continue; + const authorId = authorMap.get(author.name); + if (!authorId) continue; + const key = `${author.platform}:${author.accountId}`; + restoredAuthorAccounts.set(key, { + authorId, + platform: author.platform, + accountId: author.accountId, + }); + } + } + if (restoredAuthorAccounts.size > 0) { + await (_tx ?? db) + .insert(authorAccounts) + .values([...restoredAuthorAccounts.values()]) + .onConflictDoNothing(); + } const projectMap = await this._ensureMasterData( projects, projects.id, @@ -1213,7 +1249,7 @@ export const BackupService = { generationInfo: true, urls: true, tags: { with: { tag: true } }, - authors: { with: { author: true } }, + authors: { with: { author: { with: { accounts: true } } } }, characters: { with: { character: { @@ -1323,7 +1359,7 @@ export const BackupService = { generationInfo: true, urls: true, tags: { with: { tag: true } }, - authors: { with: { author: true } }, + authors: { with: { author: { with: { accounts: true } } } }, characters: { with: { character: { with: { ips: { with: { ip: true } } } } }, }, @@ -1407,7 +1443,9 @@ export const BackupService = { generationInfo: true, urls: true, tags: { with: { tag: true } }, - authors: { with: { author: true } }, + authors: { + with: { author: { with: { accounts: true } } }, + }, characters: { with: { character: { with: { ips: { with: { ip: true } } } }, @@ -1474,7 +1512,9 @@ export const BackupService = { generationInfo: true, urls: true, tags: { with: { tag: true } }, - authors: { with: { author: true } }, + authors: { + with: { author: { with: { accounts: true } } }, + }, characters: { with: { character: { with: { ips: { with: { ip: true } } } }, @@ -1623,7 +1663,7 @@ export const BackupService = { generationInfo: true, urls: true, tags: { with: { tag: true } }, - authors: { with: { author: true } }, + authors: { with: { author: { with: { accounts: true } } } }, characters: { with: { character: { with: { ips: { with: { ip: true } } } }, @@ -1706,6 +1746,9 @@ export const BackupService = { const simpleAuthors = (media.authors || []).map((ma) => ({ name: ma.author?.name || "", accountId: ma.author?.accountId ?? null, + platform: ma.author?.accounts?.find( + (account) => account.accountId === ma.author?.accountId, + )?.platform, })); // Extract characters diff --git a/apps/server/src/infrastructure/jobs/download-jobs.ts b/apps/server/src/infrastructure/jobs/download-jobs.ts index 725184049..54a8d5d98 100644 --- a/apps/server/src/infrastructure/jobs/download-jobs.ts +++ b/apps/server/src/infrastructure/jobs/download-jobs.ts @@ -754,6 +754,7 @@ async function updateExistingMediaWithMetadata( authors: item.authors?.map((a) => ({ name: a.name, accountId: a.accountId ?? null, + ...(a.platform ? { platform: a.platform } : {}), })), // We can also update other metadata if needed, consistent with registerMedia tags: item.tags, @@ -794,6 +795,7 @@ async function registerMedia( authors: item.authors?.map((a) => ({ name: a.name, accountId: a.accountId ?? null, + ...(a.platform ? { platform: a.platform } : {}), })), tags: item.tags, characters: item.characters, diff --git a/apps/server/src/tests/integration/backup/backup-service.test.ts b/apps/server/src/tests/integration/backup/backup-service.test.ts index 03171a016..998a8312a 100644 --- a/apps/server/src/tests/integration/backup/backup-service.test.ts +++ b/apps/server/src/tests/integration/backup/backup-service.test.ts @@ -10,6 +10,7 @@ import { import { BackupService } from "~/application/services/backup-service"; import { db } from "~/infrastructure/db"; import { + authorAccounts, authors, characterIps, characters, @@ -108,8 +109,13 @@ describe("BackupService Integration", () => { .values({ characterId: character.id, ipId: ip.id, source: "manual" }); const [author] = await db .insert(authors) - .values({ name: "Dump Author" }) + .values({ name: "Dump Author", accountId: "dump-author" }) .returning(); + await db.insert(authorAccounts).values({ + authorId: author.id, + platform: "pixiv-fanbox", + accountId: "dump-author", + }); await db .insert(mediaProjects) @@ -166,6 +172,7 @@ describe("BackupService Integration", () => { expect(item.authors).toHaveLength(1); expect(item.authors[0].name).toBe("Dump Author"); + expect(item.authors[0].platform).toBe("pixiv-fanbox"); expect(item.sourceUrls).toHaveLength(1); expect(item.sourceUrls[0]).toBe("https://example.com/source"); @@ -194,7 +201,13 @@ describe("BackupService Integration", () => { { name: "Restore Character", confidence: confidenceRestoreChar }, ], ips: [{ name: "Restore IP" }], - authors: [{ name: "Restore Author", accountId: "test_account_123" }], + authors: [ + { + name: "Restore Author", + accountId: "test_account_123", + platform: "pixiv-fanbox", + }, + ], sourceUrls: ["https://example.com/restore"], }; @@ -214,7 +227,7 @@ describe("BackupService Integration", () => { projects: { with: { project: true } }, characters: true, ips: { with: { ip: true } }, - authors: { with: { author: true } }, + authors: { with: { author: { with: { accounts: true } } } }, urls: true, }, }); @@ -235,6 +248,12 @@ describe("BackupService Integration", () => { expect(restoredMedia?.authors).toHaveLength(1); expect(restoredMedia?.authors[0].author.name).toBe("Restore Author"); expect(restoredMedia?.authors[0].author.accountId).toBe("test_account_123"); + expect(restoredMedia?.authors[0].author.accounts).toEqual([ + expect.objectContaining({ + platform: "pixiv-fanbox", + accountId: "test_account_123", + }), + ]); expect(restoredMedia?.urls).toHaveLength(1); expect(restoredMedia?.urls[0].url).toBe("https://example.com/restore"); diff --git a/apps/server/src/tests/integration/repository/author-dedupe.test.ts b/apps/server/src/tests/integration/repository/author-dedupe.test.ts index 6857ea712..c0aaf7496 100644 --- a/apps/server/src/tests/integration/repository/author-dedupe.test.ts +++ b/apps/server/src/tests/integration/repository/author-dedupe.test.ts @@ -1,8 +1,8 @@ -import { eq } from "drizzle-orm"; +import { and, eq } from "drizzle-orm"; import { migrate } from "drizzle-orm/pglite/migrator"; import { afterEach, beforeAll, describe, expect, it } from "vite-plus/test"; import { db } from "~/infrastructure/db"; -import { authors } from "~/infrastructure/db/schema"; +import { authorAccounts, authors } from "~/infrastructure/db/schema"; import { AuthorRepository } from "~/infrastructure/repositories/author-repository"; describe("AuthorRepository Deduplication", () => { @@ -45,4 +45,45 @@ describe("AuthorRepository Deduplication", () => { .where(eq(authors.name, "Duplicate Tester")); expect(count).toHaveLength(1); }); + + it("deduplicates an author by platform and account ID", async () => { + const author1 = await AuthorRepository.create({ + name: "Original Name", + accountId: "@creator", + platform: "twitter", + }); + const author2 = await AuthorRepository.create({ + name: "Updated Display Name", + accountId: "@creator", + platform: "twitter", + }); + + expect(author2.id).toBe(author1.id); + const accounts = await db + .select() + .from(authorAccounts) + .where( + and( + eq(authorAccounts.platform, "twitter"), + eq(authorAccounts.accountId, "@creator"), + ), + ); + expect(accounts).toHaveLength(1); + expect(accounts[0]?.authorId).toBe(author1.id); + }); + + it("keeps identical account IDs on different platforms separate", async () => { + const twitterAuthor = await AuthorRepository.create({ + name: "Twitter Creator", + accountId: "creator", + platform: "twitter", + }); + const fanboxAuthor = await AuthorRepository.create({ + name: "FANBOX Creator", + accountId: "creator", + platform: "pixiv-fanbox", + }); + + expect(fanboxAuthor.id).not.toBe(twitterAuthor.id); + }); }); diff --git a/apps/server/src/tests/unit/application/services/lancedb-dump-service.test.ts b/apps/server/src/tests/unit/application/services/lancedb-dump-service.test.ts index db1fa9f95..2134c82db 100644 --- a/apps/server/src/tests/unit/application/services/lancedb-dump-service.test.ts +++ b/apps/server/src/tests/unit/application/services/lancedb-dump-service.test.ts @@ -7,13 +7,18 @@ const { mockCreateTable, mockOpenTable, mockOptimize } = vi.hoisted(() => ({ mockOptimize: vi.fn(), })); -function createTable(rows: Record[] = []) { +function createTable( + rows: Record[] = [], + schemaFields: { name: string }[] = [], +) { return { add: vi.fn(async (newRows: Record[]) => rows.push(...newRows), ), + addColumns: vi.fn(), delete: vi.fn(), optimize: mockOptimize, + schema: vi.fn(async () => ({ fields: schemaFields })), query: vi.fn(() => ({ limit: (limit: number) => ({ offset: (offset: number) => ({ @@ -32,8 +37,12 @@ function createMockConnect( ) { const tables = new Map>(); mockCreateTable.mockImplementation( - async (name: string, rows: Record[]) => { - const table = createTable([...rows]); + async ( + name: string, + rows: Record[], + options: { schema?: { fields?: { name: string }[] } }, + ) => { + const table = createTable([...rows], options.schema?.fields ?? []); tables.set(name, table); return table; }, @@ -70,7 +79,9 @@ describe("LanceDB Dump Service", () => { height: 100, fileSize: 1024, tags: [{ name: "tag-a", type: "positive", confidence: 0.9 }], - authors: [{ name: "author-a", accountId: "acct" }], + authors: [ + { name: "author-a", accountId: "acct", platform: "pixiv-fanbox" }, + ], characters: [{ name: "char-a", linkedIps: ["ip-a"] }], ips: [{ name: "ip-a" }], projects: [{ name: "project-a" }], @@ -103,6 +114,11 @@ describe("LanceDB Dump Service", () => { fileName: "image1.png", }); expect(mediaRows[0]).not.toHaveProperty("imageData"); + expect(mockCreateTable.mock.calls[2][1][0]).toMatchObject({ + name: "author-a", + accountId: "acct", + platform: "pixiv-fanbox", + }); }); it("reads v2 tables back into MediaDumpItem chunks", async () => { @@ -123,7 +139,12 @@ describe("LanceDB Dump Service", () => { ], media_tags: [{ mediaId: "item-1", name: "tag-a", type: "positive" }], media_authors: [ - { mediaId: "item-1", name: "author-a", accountId: "acct" }, + { + mediaId: "item-1", + name: "author-a", + accountId: "acct", + platform: "pixiv-fanbox", + }, ], media_characters: [{ mediaId: "item-1", name: "char-a" }], media_character_ips: [ @@ -155,7 +176,9 @@ describe("LanceDB Dump Service", () => { id: "item-1", filePath: "test/image1.png", tags: [{ name: "tag-a", type: "positive" }], - authors: [{ name: "author-a", accountId: "acct" }], + authors: [ + { name: "author-a", accountId: "acct", platform: "pixiv-fanbox" }, + ], characters: [{ name: "char-a", linkedIps: ["ip-a"] }], ips: [{ name: "ip-a" }], projects: [{ name: "project-a" }], @@ -194,4 +217,34 @@ describe("LanceDB Dump Service", () => { expect.objectContaining({ id: "new-media", filePath: "new.png" }), ]); }); + + it("adds the author platform column before syncing an older LanceDB cache", async () => { + const { createLanceDbDumpService } = await import( + "@solid-imager/application/services/lancedb-dump-service" + ); + const service = createLanceDbDumpService({ connect: createMockConnect() }); + + await service.syncLanceDBDelta("/tmp/lancedb-v3", { + itemsToUpsert: [ + { + id: "new-media", + filePath: "new.png", + fileName: "new.png", + mediaType: "image", + width: 10, + height: 20, + authors: [ + { + name: "creator", + accountId: "creator", + platform: "pixiv-fanbox", + }, + ], + }, + ], + }); + + const authorTable = await mockOpenTable.mock.results[2].value; + expect(authorTable.addColumns).toHaveBeenCalledTimes(1); + }); }); diff --git a/apps/server/src/tests/unit/application/services/media-processing-service.test.ts b/apps/server/src/tests/unit/application/services/media-processing-service.test.ts index ee92dc675..f8c9c9af3 100644 --- a/apps/server/src/tests/unit/application/services/media-processing-service.test.ts +++ b/apps/server/src/tests/unit/application/services/media-processing-service.test.ts @@ -141,7 +141,7 @@ describe("MediaProcessingService", () => { }); expect(mockAuthorRepo.findOrCreateBulk).toHaveBeenCalledWith( - ["Author Name"], + [{ name: "Author Name", accountId: "acc-123" }], undefined, ); expect(mockAuthorRepo.addMediaBulk).toHaveBeenCalledWith( diff --git a/apps/server/src/tests/unit/domain/media/schemas.test.ts b/apps/server/src/tests/unit/domain/media/schemas.test.ts index 687ee4660..2d8317807 100644 --- a/apps/server/src/tests/unit/domain/media/schemas.test.ts +++ b/apps/server/src/tests/unit/domain/media/schemas.test.ts @@ -10,7 +10,9 @@ describe("downloadItemSchema", () => { "https://example.com/image.jpg", ], description: "Test description", - authors: [{ name: "Test Author", accountId: "@test" }], + authors: [ + { name: "Test Author", accountId: "@test", platform: "twitter" }, + ], tags: [{ name: "tag1", type: "positive" }], cookies: [{ name: "session", value: "123" }], userAgent: "Mozilla/5.0", @@ -20,6 +22,21 @@ describe("downloadItemSchema", () => { expect(result.success).toBe(true); }); + it("rejects an unsupported author platform", () => { + const result = downloadItemSchema.safeParse({ + targetUrl: "https://example.com/image.jpg", + authors: [ + { + name: "Test Author", + accountId: "test", + platform: "unsupported", + }, + ], + }); + + expect(result.success).toBe(false); + }); + it("should allow missing targetUrl (for restore scenarios)", () => { const backupItem = { sourceUrls: ["https://example.com/tweet"], diff --git a/apps/server/src/tests/unit/infrastructure/jobs/download-jobs.test.ts b/apps/server/src/tests/unit/infrastructure/jobs/download-jobs.test.ts index 0af61ec41..d4e390f24 100644 --- a/apps/server/src/tests/unit/infrastructure/jobs/download-jobs.test.ts +++ b/apps/server/src/tests/unit/infrastructure/jobs/download-jobs.test.ts @@ -87,8 +87,8 @@ vi.mock("node:child_process", () => ({ const fetchMock = vi.fn(); global.fetch = fetchMock as any; -// Regex for download filename pattern (unified format: {author}_{date}_{id}.{ext}) -const DOWNLOAD_FILENAME_PATTERN = /^user_.*123\.jpg$/; +// Unified format: {author}_{platform}_{postId}_{assetId}.{ext} +const DOWNLOAD_FILENAME_PATTERN = /^user_twitter_123_image\.jpg$/; describe("processDownloadJob", () => { beforeEach(async () => { diff --git a/apps/xtracter/README.md b/apps/xtracter/README.md index 10b4177d6..b850543b8 100644 --- a/apps/xtracter/README.md +++ b/apps/xtracter/README.md @@ -1,14 +1,14 @@ # xtracter -xtracter is a chrome extension that extracts images from a X(twitter) and save them to a local directory. +xtracter is a chrome extension that extracts images from X (Twitter), pixivFANBOX, and Danbooru and saves them to a local directory. ## Features - Xのタイムライン上にある画像の右上にダウンロードボタンを追加、ローカルに保存可能にする。 +- pixivFANBOXの記事画像の右上にダウンロードボタンを追加し、画像CDN URLと元記事URLを保存する。 - タイムライン上にある画像の情報をJSONファイルに出力する。 - json ファイルには画像のソースURL、元ポストのURL、元ポストの文面、元ポストの投稿日時、元ポストの投稿者の名前、元ポストの投稿者のIDが含まれる ## Development 開発言語はTS、ビルドシステムにはviteを使用する - diff --git a/apps/xtracter/public/manifest.json b/apps/xtracter/public/manifest.json index 9d1d3ea23..e474b3365 100644 --- a/apps/xtracter/public/manifest.json +++ b/apps/xtracter/public/manifest.json @@ -15,6 +15,8 @@ "https://twimg.com/*", "https://pbs.twimg.com/*", "https://video.twimg.com/*", + "https://*.fanbox.cc/*", + "https://downloads.fanbox.cc/*", "https://danbooru.donmai.us/*", "http://localhost:*/*", "http://127.0.0.1:*/*", @@ -33,6 +35,7 @@ "matches": [ "https://twitter.com/*", "https://x.com/*", + "https://*.fanbox.cc/*", "https://danbooru.donmai.us/*" ], "js": ["content.js"], diff --git a/apps/xtracter/src/content/danbooru.ts b/apps/xtracter/src/content/danbooru.ts index f459cafea..14ebe01a1 100644 --- a/apps/xtracter/src/content/danbooru.ts +++ b/apps/xtracter/src/content/danbooru.ts @@ -191,7 +191,11 @@ function parseDanbooruApiMetadata( // Parse artists for (const name of parseTagsFromApiString(data.tag_string_artist)) { - authors.push({ name, accountId: twitterAccountId }); + authors.push({ + name, + accountId: twitterAccountId ?? name, + platform: twitterAccountId ? "twitter" : "danbooru", + }); } // Parse copyrights (IPs) @@ -267,6 +271,12 @@ function extractDanbooruMetadata(container: HTMLElement): TweetMetadata | null { if (!author.accountId) { author.accountId = twitterAccountId; } + author.platform = "twitter"; + } + } else { + for (const author of authors) { + author.accountId = author.accountId ?? author.name; + author.platform = "danbooru"; } } diff --git a/apps/xtracter/src/content/fanbox.ts b/apps/xtracter/src/content/fanbox.ts new file mode 100644 index 000000000..3033c559f --- /dev/null +++ b/apps/xtracter/src/content/fanbox.ts @@ -0,0 +1,111 @@ +import type { Author, TweetMetadata } from "@ext/schema"; +import { querySelectorAllTyped } from "../utils/dom-utils"; + +const PROCESSED_IMAGE_CLASS = "xtracter-fanbox-image-processed"; +const FANBOX_IMAGE_HOSTNAME = "downloads.fanbox.cc"; +const FANBOX_POST_PATH = /^\/posts\/(\d+)/; + +export function processFanboxMedia( + processedMetadata: Map, + createButtonContainer: ( + metadata: TweetMetadata, + type: "IMAGE" | "VIDEO", + ) => HTMLDivElement, +) { + const postUrl = getPostUrl(); + if (!postUrl) { + return; + } + + const postId = new URL(postUrl).pathname.match(FANBOX_POST_PATH)?.[1]; + if (!postId) { + return; + } + + const images = querySelectorAllTyped(document, "img"); + for (const imageElement of images) { + const targetUrl = getFanboxImageUrl(imageElement, postId); + if (!targetUrl || imageElement.dataset.xtracterProcessed === "true") { + continue; + } + + const container = imageElement.parentElement; + if (!container) { + continue; + } + + const metadata = extractMetadata(targetUrl, postUrl); + processedMetadata.set(targetUrl, metadata); + + if (window.getComputedStyle(container).position === "static") { + container.style.position = "relative"; + } + container.classList.add(PROCESSED_IMAGE_CLASS); + imageElement.dataset.xtracterProcessed = "true"; + container.appendChild(createButtonContainer(metadata, "IMAGE")); + } +} + +function getPostUrl(): string | null { + const match = window.location.pathname.match(FANBOX_POST_PATH); + if (!match) { + return null; + } + + return `${window.location.origin}/posts/${match[1]}`; +} + +function getFanboxImageUrl( + imageElement: HTMLImageElement, + postId: string, +): string | null { + const linkedUrl = imageElement.closest("a")?.href; + const candidates = [linkedUrl, imageElement.currentSrc, imageElement.src]; + + for (const candidate of candidates) { + if (!candidate) { + continue; + } + + try { + const url = new URL(candidate); + if ( + url.hostname === FANBOX_IMAGE_HOSTNAME && + url.pathname.startsWith(`/images/post/${postId}/`) + ) { + return url.toString(); + } + } catch { + // Ignore malformed candidates and try the next image source. + } + } + + return null; +} + +function extractMetadata(targetUrl: string, postUrl: string): TweetMetadata { + const creatorId = window.location.hostname.endsWith(".fanbox.cc") + ? window.location.hostname.slice(0, -".fanbox.cc".length) + : ""; + const authorName = + document.querySelector('a[href="/"] h1')?.innerText.trim() || + creatorId; + const authors: Author[] = []; + + if (authorName || creatorId) { + authors.push({ + name: authorName || creatorId, + accountId: creatorId || undefined, + platform: "pixiv-fanbox", + }); + } + + return { + targetUrl, + sourceUrls: [targetUrl, postUrl], + description: + document.querySelector("article h1")?.textContent?.trim() ?? "", + authors, + userAgent: navigator.userAgent, + }; +} diff --git a/apps/xtracter/src/content/index.ts b/apps/xtracter/src/content/index.ts index 4e7dd1640..7ce30c9f3 100644 --- a/apps/xtracter/src/content/index.ts +++ b/apps/xtracter/src/content/index.ts @@ -1,5 +1,6 @@ import type { TweetMetadata } from "@ext/schema"; import { processDanbooruMedia } from "./danbooru"; +import { processFanboxMedia } from "./fanbox"; import { processTwitterMedia } from "./twitter"; const OBSERVER_CONFIG = { childList: true, subtree: true }; @@ -180,6 +181,8 @@ function processMedia() { const hostname = window.location.hostname; if (hostname.includes("twitter.com") || hostname.includes("x.com")) { processTwitterMedia(processedMetadata, createButtonContainer); + } else if (hostname === "fanbox.cc" || hostname.endsWith(".fanbox.cc")) { + processFanboxMedia(processedMetadata, createButtonContainer); } else if (hostname.includes("danbooru.donmai.us")) { processDanbooruMedia(createButtonContainer, createAsyncButtonContainer); } diff --git a/apps/xtracter/src/content/twitter.ts b/apps/xtracter/src/content/twitter.ts index 3c60a028b..c4778afdd 100644 --- a/apps/xtracter/src/content/twitter.ts +++ b/apps/xtracter/src/content/twitter.ts @@ -176,6 +176,7 @@ function extractMetadata( authors.push({ name: authorName || authorId, accountId: authorId, + platform: "twitter", }); } diff --git a/apps/xtracter/src/schema.ts b/apps/xtracter/src/schema.ts index 4d4345414..211b44066 100644 --- a/apps/xtracter/src/schema.ts +++ b/apps/xtracter/src/schema.ts @@ -3,6 +3,7 @@ import { z } from "zod"; export const authorSchema = z.object({ name: z.string(), accountId: z.string().nullable().optional(), + platform: z.enum(["twitter", "pixiv-fanbox", "danbooru"]).optional(), }); export type Author = z.infer; diff --git a/packages/application/src/services/lancedb-dump-service.ts b/packages/application/src/services/lancedb-dump-service.ts index a982beae2..8f2665062 100644 --- a/packages/application/src/services/lancedb-dump-service.ts +++ b/packages/application/src/services/lancedb-dump-service.ts @@ -14,7 +14,7 @@ import type { const ROW_CHUNK_SIZE = 5000; const READ_CHUNK_SIZE = 5000; -const LANCEDB_DUMP_VERSION = 3; +const LANCEDB_DUMP_VERSION = 4; type JsonValue = | string @@ -126,6 +126,7 @@ async function createSchemas(): Promise> { ["mediaId", utf8()], ["name", utf8()], ["accountId", utf8()], + ["platform", utf8()], ]), media_characters: await createSchema([ ["key", utf8(), false], @@ -222,10 +223,11 @@ function itemsToRows(items: MediaDumpItem[]): TableRows { }); for (const author of item.authors ?? []) rows.authors.push({ - key: rowKey(mediaId, author.name, author.accountId), + key: rowKey(mediaId, author.name, author.platform, author.accountId), mediaId, name: author.name, accountId: author.accountId ?? null, + platform: author.platform ?? null, }); for (const character of item.characters ?? []) { rows.characters.push({ @@ -379,6 +381,16 @@ async function openTables(db: Connection): Promise { }; } +async function migrateLanceDbSchema(tables: LanceTables): Promise { + const authorSchema = await tables.media_authors.schema(); + if (!authorSchema.fields.some((field) => field.name === "platform")) { + const arrow = await import("apache-arrow"); + await tables.media_authors.addColumns( + new arrow.Field("platform", new arrow.Utf8(), true), + ); + } +} + function sqlInList(values: string[]): string { return values.map((value) => `'${escapeLanceSqlString(value)}'`).join(","); } @@ -548,6 +560,12 @@ function rowsToItems( authors: (authors.get(id) ?? []).map((row) => ({ name: safeString(row.name) ?? "", accountId: safeString(row.accountId), + platform: + row.platform === "twitter" || + row.platform === "pixiv-fanbox" || + row.platform === "danbooru" + ? row.platform + : undefined, })), characters: (characters.get(id) ?? []).map((row) => { const name = safeString(row.name) ?? ""; @@ -772,6 +790,7 @@ export function createLanceDbDumpService(deps?: { const connect = await getConnect(); const db = await connect(lanceDbDir); const tables = await openTables(db); + await migrateLanceDbSchema(tables); await deleteMediaRows(tables, targetIds); await addRowsToTables(tables, itemsToRows(itemsToUpsert)); diff --git a/packages/application/src/services/media-processing-service.ts b/packages/application/src/services/media-processing-service.ts index a0e40819f..55a63b9a2 100644 --- a/packages/application/src/services/media-processing-service.ts +++ b/packages/application/src/services/media-processing-service.ts @@ -345,9 +345,8 @@ export class MediaProcessingServiceImpl implements IMediaProcessingService { if (authors.length === 0) { return; } - const names = authors.map((a) => a.name); try { - const allAuthors = await this.authorRepo.findOrCreateBulk(names, tx); + const allAuthors = await this.authorRepo.findOrCreateBulk(authors, tx); const authorIds = allAuthors.map((a) => a.id); await this.authorRepo.addMediaBulk(mediaId, authorIds, tx); } catch (e) { diff --git a/packages/core/src/domain/authors/schemas.ts b/packages/core/src/domain/authors/schemas.ts index 2311bd3a0..b29a3bb7c 100644 --- a/packages/core/src/domain/authors/schemas.ts +++ b/packages/core/src/domain/authors/schemas.ts @@ -1,4 +1,5 @@ import { z } from "zod"; +import { authorPlatformSchema } from "../media/schemas"; export const authorSchema = z.object({ id: z.string().uuid(), @@ -9,3 +10,15 @@ export const authorSchema = z.object({ }); export type Author = z.infer; + +export const authorAccountSchema = z.object({ + id: z.string().uuid(), + authorId: z.string().uuid(), + platform: authorPlatformSchema, + accountId: z.string().min(1), + profileUrl: z.string().url().nullable(), + createdAt: z.coerce.date(), + updatedAt: z.coerce.date(), +}); + +export type AuthorAccount = z.infer; diff --git a/packages/core/src/domain/media/schemas.ts b/packages/core/src/domain/media/schemas.ts index 06c0e675d..c858bee42 100644 --- a/packages/core/src/domain/media/schemas.ts +++ b/packages/core/src/domain/media/schemas.ts @@ -12,6 +12,13 @@ import { z } from "zod"; export const mediaTypeSchema = z.enum(["image", "video", "audio"]); export type MediaType = z.infer; +export const authorPlatformSchema = z.enum([ + "twitter", + "pixiv-fanbox", + "danbooru", +]); +export type AuthorPlatform = z.infer; + /** * Zod schema for validating the request body when adding new media. * Ensures all required fields for new media are present and correctly formatted. @@ -71,6 +78,7 @@ export const updateMediaRequestSchema = z.object({ z.object({ name: z.string(), accountId: z.string().optional().nullable(), + platform: authorPlatformSchema.optional(), }), ) .optional(), @@ -183,6 +191,7 @@ export type Author = z.infer; export const newAuthorSchema = z.object({ name: z.string(), accountId: z.string().nullable().optional(), + platform: authorPlatformSchema.optional(), }); export type NewAuthor = z.infer; @@ -412,6 +421,7 @@ export const mediaMetadataContextSchema = z.object({ z.object({ name: z.string(), accountId: z.string().nullable().optional(), + platform: authorPlatformSchema.optional(), }), ) .optional(), diff --git a/packages/core/src/domain/media/utils/filename-utils.ts b/packages/core/src/domain/media/utils/filename-utils.ts index a8ba3ad8a..d58d93afb 100644 --- a/packages/core/src/domain/media/utils/filename-utils.ts +++ b/packages/core/src/domain/media/utils/filename-utils.ts @@ -1,4 +1,4 @@ -import type { DownloadItem } from "../schemas"; +import type { AuthorPlatform, DownloadItem } from "../schemas"; const TWITTER_ID_PREFIX_LENGTH = 1; const FILENAME_SANITIZE_REGEX = /[/\\?%*:|"<>.]/g; @@ -22,9 +22,10 @@ export function sanitizeFilenamePart(part: string): string { const TWITTER_STATUS_REGEX = /\/status\/(\d+)/; const DANBOORU_POST_REGEX = /\/posts\/(\d+)/; +const FANBOX_POST_REGEX = /\/posts\/(\d+)/; /** - * Extract a unique ID from common media source URLs (Twitter, Danbooru). + * Extract a unique post ID from supported media source URLs. */ export function extractIdFromUrl(url: string): string | null { try { @@ -42,6 +43,14 @@ export function extractIdFromUrl(url: string): string | null { const match = urlObj.pathname.match(DANBOORU_POST_REGEX); return match ? match[1] : null; } + // pixivFANBOX: https://creator.fanbox.cc/posts/123456 + if ( + urlObj.hostname === "fanbox.cc" || + urlObj.hostname.endsWith(".fanbox.cc") + ) { + const match = urlObj.pathname.match(FANBOX_POST_REGEX); + return match ? match[1] : null; + } } catch (_e) { return null; } @@ -50,35 +59,24 @@ export function extractIdFromUrl(url: string): string | null { /** * Generate a unified filename based on media metadata. - * Format: {authorId}_{date}_{contentId}.{extension} + * Format: {accountId}_{platform}_{postId}_{assetId}.{extension} */ export function generateMediaFilename( item: DownloadItem, extension: string, ): string { const authorId = getAuthorPart(item); - const dateStr = getDatePart(item); + const platform = getPlatformPart(item); const contentId = getContentIdPart(item); + const assetId = getAssetIdPart(item); - let base = ""; - if (authorId || dateStr || contentId) { - const parts: string[] = []; - if (authorId) { - parts.push(authorId); - } - if (dateStr) { - parts.push(dateStr); - } - if (contentId) { - parts.push(contentId); - } - base = parts.join("_"); - } else { - base = getFallbackBase(item); - } + const parts = [authorId, platform, contentId, assetId].filter( + (part) => part.length > 0, + ); + let base = parts.join("_"); if (!base) { - base = "media"; + base = getFallbackBase(item) || "media"; } return `${base}${sanitizeExtension(extension)}`; @@ -95,21 +93,45 @@ function getAuthorPart(item: DownloadItem): string { return ""; } -function getDatePart(item: DownloadItem): string { - if (item.createdAt) { - const date = new Date(item.createdAt); - if (!Number.isNaN(date.getTime())) { - const DATE_ISO_SUBSTRING_START = 0; - const DATE_ISO_SUBSTRING_END = 10; - return date - .toISOString() - .slice(DATE_ISO_SUBSTRING_START, DATE_ISO_SUBSTRING_END) - .replace(/-/g, ""); - } +function getPlatformPart(item: DownloadItem): string { + const declaredPlatform = item.authors?.[0]?.platform; + if (declaredPlatform) { + return declaredPlatform; + } + + for (const candidate of [...(item.sourceUrls ?? []), item.targetUrl]) { + if (!candidate) continue; + const inferred = inferPlatformFromUrl(candidate); + if (inferred) return inferred; } + return ""; } +function inferPlatformFromUrl(url: string): AuthorPlatform | null { + try { + const hostname = new URL(url).hostname; + if ( + hostname === "twitter.com" || + hostname.endsWith(".twitter.com") || + hostname === "x.com" || + hostname.endsWith(".x.com") || + hostname.endsWith(".twimg.com") + ) { + return "twitter"; + } + if (hostname === "fanbox.cc" || hostname.endsWith(".fanbox.cc")) { + return "pixiv-fanbox"; + } + if (hostname === "danbooru.donmai.us" || hostname.endsWith(".donmai.us")) { + return "danbooru"; + } + } catch { + return null; + } + return null; +} + function getContentIdPart(item: DownloadItem): string { if (item.sourceUrls) { for (const url of item.sourceUrls) { @@ -127,6 +149,21 @@ function getContentIdPart(item: DownloadItem): string { return ""; } +function getAssetIdPart(item: DownloadItem): string { + if (!item.targetUrl) return ""; + + try { + const pathname = new URL(item.targetUrl).pathname; + const filename = pathname.split("/").filter(Boolean).pop() ?? ""; + const extensionIndex = filename.lastIndexOf("."); + const assetId = + extensionIndex > 0 ? filename.slice(0, extensionIndex) : filename; + return sanitizeFilenamePart(assetId); + } catch { + return ""; + } +} + function getFallbackBase(item: DownloadItem): string { if (item.targetUrl) { try { diff --git a/packages/core/src/domain/repositories/author-repository.ts b/packages/core/src/domain/repositories/author-repository.ts index 9071da1d1..91f8b9d85 100644 --- a/packages/core/src/domain/repositories/author-repository.ts +++ b/packages/core/src/domain/repositories/author-repository.ts @@ -37,6 +37,6 @@ export type IAuthorRepository = { tx?: Transaction, ): Promise; - /** Bulk find-or-create authors by name (uses ON CONFLICT DO NOTHING with unique constraint on name). */ - findOrCreateBulk(names: string[], tx?: Transaction): Promise; + /** Bulk find-or-create authors, preferring platform-scoped account identity. */ + findOrCreateBulk(authors: NewAuthor[], tx?: Transaction): Promise; }; diff --git a/packages/core/tests/filename-utils.test.ts b/packages/core/tests/filename-utils.test.ts index 56dd3f274..a5e15cdae 100644 --- a/packages/core/tests/filename-utils.test.ts +++ b/packages/core/tests/filename-utils.test.ts @@ -1,86 +1,117 @@ -import { describe, it, expect } from "vite-plus/test"; -import { generateMediaFilename, sanitizeFilenamePart, extractIdFromUrl } from "../src/domain/media/utils/filename-utils"; +import { describe, expect, it } from "vite-plus/test"; +import { + extractIdFromUrl, + generateMediaFilename, + sanitizeFilenamePart, +} from "../src/domain/media/utils/filename-utils"; import type { DownloadItem } from "../src/domain/media/schemas"; describe("filename-utils", () => { - describe("sanitizeFilenamePart", () => { - it("should remove @ from the beginning", () => { - expect(sanitizeFilenamePart("@user_name")).toBe("user_name"); - }); + describe("sanitizeFilenamePart", () => { + it("should remove @ from the beginning", () => { + expect(sanitizeFilenamePart("@user_name")).toBe("user_name"); + }); - it("should remove invalid characters including dots", () => { - expect(sanitizeFilenamePart("user/name.ext?%")).toBe("usernameext"); - }); + it("should remove invalid characters including dots", () => { + expect(sanitizeFilenamePart("user/name.ext?%")).toBe("usernameext"); + }); - it("should replace spaces with underscores", () => { - expect(sanitizeFilenamePart("user name")).toBe("user_name"); - }); + it("should replace spaces with underscores", () => { + expect(sanitizeFilenamePart("user name")).toBe("user_name"); + }); - it("should trim leading and trailing spaces", () => { - expect(sanitizeFilenamePart(" artist_name ")).toBe("artist_name"); - }); - }); + it("should trim leading and trailing spaces", () => { + expect(sanitizeFilenamePart(" artist_name ")).toBe("artist_name"); + }); + }); - describe("extractIdFromUrl", () => { - it("should extract Twitter status ID", () => { - expect(extractIdFromUrl("https://twitter.com/user/status/123456789?s=20")).toBe("123456789"); - expect(extractIdFromUrl("https://x.com/user/status/987654321")).toBe("987654321"); - }); + describe("extractIdFromUrl", () => { + it("should extract Twitter status ID", () => { + expect( + extractIdFromUrl( + "https://twitter.com/user/status/123456789?s=20", + ), + ).toBe("123456789"); + expect( + extractIdFromUrl("https://x.com/user/status/987654321"), + ).toBe("987654321"); + }); - it("should extract Danbooru post ID", () => { - expect(extractIdFromUrl("https://danbooru.donmai.us/posts/123456")).toBe("123456"); - }); + it("should extract Danbooru post ID", () => { + expect( + extractIdFromUrl("https://danbooru.donmai.us/posts/123456"), + ).toBe("123456"); + }); - it("should return null for other URLs", () => { - expect(extractIdFromUrl("https://google.com")).toBeNull(); - }); - }); + it("should extract pixivFANBOX post ID", () => { + expect( + extractIdFromUrl("https://example-creator.fanbox.cc/posts/12345678"), + ).toBe("12345678"); + }); - describe("generateMediaFilename", () => { - it("should generate unified filename with author, date, and ID", () => { - const item: DownloadItem = { - targetUrl: "https://pbs.twimg.com/media/abc.jpg", - authors: [{ name: "Artist", accountId: "@artist_id" }], - createdAt: "2023-10-27T10:00:00Z", - sourceUrls: ["https://twitter.com/artist_id/status/1717891234"] - }; - expect(generateMediaFilename(item, ".jpg")).toBe("artist_id_20231027_1717891234.jpg"); - }); + it("should return null for other URLs", () => { + expect(extractIdFromUrl("https://google.com")).toBeNull(); + }); + }); - it("should fallback to name if accountId is missing", () => { - const item: DownloadItem = { - targetUrl: "https://pbs.twimg.com/media/abc.jpg", - authors: [{ name: "Artist Name" }], - createdAt: "2023-10-27T10:00:00Z", - sourceUrls: ["https://twitter.com/artist_id/status/1717891234"] - }; - expect(generateMediaFilename(item, ".jpg")).toBe("Artist_Name_20231027_1717891234.jpg"); - }); + describe("generateMediaFilename", () => { + it("should generate a Twitter filename", () => { + const item: DownloadItem = { + targetUrl: "https://pbs.twimg.com/media/abc.jpg", + authors: [ + { + name: "Artist", + accountId: "@artist_id", + platform: "twitter", + }, + ], + sourceUrls: [ + "https://twitter.com/artist_id/status/1717891234", + ], + }; + expect(generateMediaFilename(item, ".jpg")).toBe( + "artist_id_twitter_1717891234_abc.jpg", + ); + }); - it("should handle missing date", () => { - const item: DownloadItem = { - targetUrl: "https://pbs.twimg.com/media/abc.jpg", - authors: [{ name: "Artist" }], - sourceUrls: ["https://danbooru.donmai.us/posts/123456"] - }; - expect(generateMediaFilename(item, ".png")).toBe("Artist_123456.png"); - }); + it("should generate a pixivFANBOX filename", () => { + const item: DownloadItem = { + targetUrl: + "https://downloads.fanbox.cc/images/post/12345678/exampleAssetId.jpeg", + authors: [ + { + name: "Example Creator", + accountId: "example-creator", + platform: "pixiv-fanbox", + }, + ], + sourceUrls: [ + "https://downloads.fanbox.cc/images/post/12345678/exampleAssetId.jpeg", + "https://example-creator.fanbox.cc/posts/12345678", + ], + }; + expect(generateMediaFilename(item, ".jpeg")).toBe( + "example-creator_pixiv-fanbox_12345678_exampleAssetId.jpeg", + ); + }); - it("should handle missing author", () => { - const item: DownloadItem = { - targetUrl: "https://pbs.twimg.com/media/abc.jpg", - createdAt: "2023-10-27T10:00:00Z", - sourceUrls: ["https://danbooru.donmai.us/posts/123456"] - }; - expect(generateMediaFilename(item, ".png")).toBe("20231027_123456.png"); - }); + it("should infer platform for legacy metadata", () => { + const item: DownloadItem = { + targetUrl: "https://cdn.donmai.us/original/hash.png", + authors: [{ name: "Artist" }], + sourceUrls: ["https://danbooru.donmai.us/posts/123456"], + }; + expect(generateMediaFilename(item, ".png")).toBe( + "Artist_danbooru_123456_hash.png", + ); + }); - it("should not have a trailing dot if extension is empty", () => { - const item: DownloadItem = { - targetUrl: "https://example.com/image", - authors: [{ name: "Artist" }] - }; - expect(generateMediaFilename(item, "")).toBe("Artist"); - }); - }); + it("should not have a trailing dot if extension is empty", () => { + const item: DownloadItem = { + targetUrl: "https://example.com/image", + authors: [{ name: "Artist" }], + }; + expect(generateMediaFilename(item, "")).toBe("Artist_image"); + }); + }); }); diff --git a/packages/db/src/repositories/author-repository.ts b/packages/db/src/repositories/author-repository.ts index 63b2a2b89..67607e6bb 100644 --- a/packages/db/src/repositories/author-repository.ts +++ b/packages/db/src/repositories/author-repository.ts @@ -5,7 +5,7 @@ import type { } from "@solid-imager/core/domain/media/schemas"; import type { IAuthorRepository } from "@solid-imager/core/domain/repositories/author-repository"; import { and, eq, inArray } from "drizzle-orm"; -import { authors, mediaAuthors } from "../schema"; +import { authorAccounts, authors, mediaAuthors } from "../schema"; import type { DrizzleExecutor } from "../types"; type AuthorRepositoryOptions = { @@ -62,22 +62,56 @@ export function createAuthorRepository( async create(author: NewAuthor, tx?: unknown): Promise { const client = getExecutor(tx); - const condition = author.accountId - ? eq(authors.accountId, author.accountId) - : eq(authors.name, author.name); - const existing = await client - .select() - .from(authors) - .where(condition) - .limit(1); + if (author.platform && author.accountId) { + const existingByAccount = await client + .select({ author: authors }) + .from(authorAccounts) + .innerJoin(authors, eq(authorAccounts.authorId, authors.id)) + .where( + and( + eq(authorAccounts.platform, author.platform), + eq(authorAccounts.accountId, author.accountId), + ), + ) + .limit(1); + + if (existingByAccount[0]) { + return mapAuthor(existingByAccount[0].author); + } + } else { + const condition = author.accountId + ? eq(authors.accountId, author.accountId) + : eq(authors.name, author.name); + const existing = await client + .select() + .from(authors) + .where(condition) + .limit(1); + + if (existing[0]) { + return mapAuthor(existing[0]); + } + } - if (existing[0]) { - return mapAuthor(existing[0]); + const result = await client + .insert(authors) + .values({ + name: author.name, + accountId: author.accountId, + }) + .returning(); + const created = result[0]; + + if (author.platform && author.accountId) { + await client.insert(authorAccounts).values({ + authorId: created.id, + platform: author.platform, + accountId: author.accountId, + }); } - const result = await client.insert(authors).values(author).returning(); - return mapAuthor(result[0]); + return mapAuthor(created); }, async update( @@ -167,30 +201,25 @@ export function createAuthorRepository( .onConflictDoNothing(); }, - async findOrCreateBulk(names: string[], tx?: unknown): Promise { - if (names.length === 0) return []; - const uniqueNames = [...new Set(names)].filter((n) => n.length > 0); - const client = getExecutor(tx); - - const existing = await client - .select() - .from(authors) - .where(inArray(authors.name, uniqueNames)); - const existingNames = new Set(existing.map((a) => a.name)); - - const newNames = uniqueNames.filter((n) => !existingNames.has(n)); - if (newNames.length > 0) { - await client - .insert(authors) - .values(newNames.map((name) => ({ name }))) - .onConflictDoNothing(); + async findOrCreateBulk( + inputs: NewAuthor[], + tx?: unknown, + ): Promise { + const uniqueInputs = new Map(); + for (const input of inputs) { + if (input.name.length === 0) continue; + const key = + input.platform && input.accountId + ? `${input.platform}:${input.accountId}` + : `name:${input.name}`; + uniqueInputs.set(key, input); } - const all = await client - .select() - .from(authors) - .where(inArray(authors.name, uniqueNames)); - return all.map(mapAuthor); + const result: Author[] = []; + for (const input of uniqueInputs.values()) { + result.push(await this.create(input, tx)); + } + return result; }, }; } diff --git a/packages/db/src/schema.ts b/packages/db/src/schema.ts index bcffafc8b..9f9ede449 100644 --- a/packages/db/src/schema.ts +++ b/packages/db/src/schema.ts @@ -82,6 +82,12 @@ export const mediaRelationTypeEnum = pgEnum("media_relation_type", [ */ export const tagTypeEnum = pgEnum("tag_type", ["positive", "negative"]); +export const authorPlatformEnum = pgEnum("author_platform", [ + "twitter", + "pixiv-fanbox", + "danbooru", +]); + // テーブル /** * Schema for the media_sources table. @@ -742,6 +748,31 @@ export const authors = pgTable( }), ); +/** + * External accounts belonging to an author. + * An author can be linked to multiple platform identities. + */ +export const authorAccounts = pgTable( + "author_accounts", + { + id: uuid("id").primaryKey().default(sql`gen_random_uuid()`), + authorId: uuid("author_id") + .notNull() + .references(() => authors.id, { onDelete: "cascade" }), + platform: authorPlatformEnum("platform").notNull(), + accountId: text("account_id").notNull(), + profileUrl: text("profile_url"), + createdAt: timestamp("created_at").notNull().defaultNow(), + updatedAt: timestamp("updated_at").notNull().defaultNow(), + }, + (table) => ({ + authorIdIndex: index("idx_author_accounts_author_id").on(table.authorId), + platformAccountUnique: uniqueIndex( + "idx_author_accounts_platform_account_unique", + ).on(table.platform, table.accountId), + }), +); + /** * Schema for the media_authors join table. * Many-to-many relationship between media and authors. @@ -1298,10 +1329,19 @@ export const mediaCollectionsRelations = relations( export const authorsRelations = relations(authors, ({ many }) => ({ /** Authorに関連するメディア */ media: many(mediaAuthors), + /** Authorの外部サービス上のアカウント */ + accounts: many(authorAccounts), /** Authorに関連するタグ */ tags: many(tags), })); +export const authorAccountsRelations = relations(authorAccounts, ({ one }) => ({ + author: one(authors, { + fields: [authorAccounts.authorId], + references: [authors.id], + }), +})); + /** * Defines the relations for the media_authors join table. */ From bd5b9ff2de601df3d1574608e37252f443e3c2e6 Mon Sep 17 00:00:00 2001 From: hmjn Date: Wed, 1 Jul 2026 00:29:32 +0900 Subject: [PATCH 06/19] fix(xtracter): address author identity review feedback --- .../repository/author-dedupe.test.ts | 30 +++ apps/xtracter/src/content/fanbox.ts | 28 +-- .../db/src/repositories/author-repository.ts | 220 ++++++++++++------ 3 files changed, 192 insertions(+), 86 deletions(-) diff --git a/apps/server/src/tests/integration/repository/author-dedupe.test.ts b/apps/server/src/tests/integration/repository/author-dedupe.test.ts index c0aaf7496..9bb19478a 100644 --- a/apps/server/src/tests/integration/repository/author-dedupe.test.ts +++ b/apps/server/src/tests/integration/repository/author-dedupe.test.ts @@ -86,4 +86,34 @@ describe("AuthorRepository Deduplication", () => { expect(fanboxAuthor.id).not.toBe(twitterAuthor.id); }); + + it("links a platform account to a legacy author without creating a duplicate", async () => { + const [legacyAuthor] = await db + .insert(authors) + .values({ name: "Legacy Creator", accountId: "legacy-creator" }) + .returning(); + + const resolved = await AuthorRepository.create({ + name: "Legacy Creator", + accountId: "legacy-creator", + platform: "twitter", + }); + + expect(resolved.id).toBe(legacyAuthor.id); + const matchingAuthors = await db + .select() + .from(authors) + .where(eq(authors.accountId, "legacy-creator")); + expect(matchingAuthors).toHaveLength(1); + const accounts = await db + .select() + .from(authorAccounts) + .where(eq(authorAccounts.authorId, legacyAuthor.id)); + expect(accounts).toEqual([ + expect.objectContaining({ + platform: "twitter", + accountId: "legacy-creator", + }), + ]); + }); }); diff --git a/apps/xtracter/src/content/fanbox.ts b/apps/xtracter/src/content/fanbox.ts index 3033c559f..72c7caa68 100644 --- a/apps/xtracter/src/content/fanbox.ts +++ b/apps/xtracter/src/content/fanbox.ts @@ -12,15 +12,12 @@ export function processFanboxMedia( type: "IMAGE" | "VIDEO", ) => HTMLDivElement, ) { - const postUrl = getPostUrl(); - if (!postUrl) { - return; - } - - const postId = new URL(postUrl).pathname.match(FANBOX_POST_PATH)?.[1]; - if (!postId) { + const match = window.location.pathname.match(FANBOX_POST_PATH); + if (!match) { return; } + const postId = match[1]; + const postUrl = `${window.location.origin}/posts/${postId}`; const images = querySelectorAllTyped(document, "img"); for (const imageElement of images) { @@ -46,15 +43,6 @@ export function processFanboxMedia( } } -function getPostUrl(): string | null { - const match = window.location.pathname.match(FANBOX_POST_PATH); - if (!match) { - return null; - } - - return `${window.location.origin}/posts/${match[1]}`; -} - function getFanboxImageUrl( imageElement: HTMLImageElement, postId: string, @@ -84,9 +72,11 @@ function getFanboxImageUrl( } function extractMetadata(targetUrl: string, postUrl: string): TweetMetadata { - const creatorId = window.location.hostname.endsWith(".fanbox.cc") - ? window.location.hostname.slice(0, -".fanbox.cc".length) - : ""; + const hostname = window.location.hostname; + const creatorId = + hostname.endsWith(".fanbox.cc") && !hostname.startsWith("www.") + ? hostname.slice(0, -".fanbox.cc".length) + : ""; const authorName = document.querySelector('a[href="/"] h1')?.innerText.trim() || creatorId; diff --git a/packages/db/src/repositories/author-repository.ts b/packages/db/src/repositories/author-repository.ts index 67607e6bb..4e1aa525b 100644 --- a/packages/db/src/repositories/author-repository.ts +++ b/packages/db/src/repositories/author-repository.ts @@ -1,10 +1,11 @@ +import { randomUUID } from "node:crypto"; import { ResourceNotFoundError } from "@solid-imager/core/domain/errors"; import type { Author, NewAuthor, } from "@solid-imager/core/domain/media/schemas"; import type { IAuthorRepository } from "@solid-imager/core/domain/repositories/author-repository"; -import { and, eq, inArray } from "drizzle-orm"; +import { and, eq, inArray, or, type SQL } from "drizzle-orm"; import { authorAccounts, authors, mediaAuthors } from "../schema"; import type { DrizzleExecutor } from "../types"; @@ -22,6 +23,154 @@ function mapAuthor(row: typeof authors.$inferSelect): Author { }; } +function authorInputKey(author: NewAuthor): string { + return author.platform && author.accountId + ? `${author.platform}:${author.accountId}` + : `name:${author.name}`; +} + +async function findOrCreateAuthorsBulk( + client: DrizzleExecutor, + inputs: NewAuthor[], +): Promise { + const uniqueInputs = new Map(); + for (const input of inputs) { + if (input.name.length > 0) { + uniqueInputs.set(authorInputKey(input), input); + } + } + if (uniqueInputs.size === 0) return []; + + const identityConditions: SQL[] = []; + for (const input of uniqueInputs.values()) { + if (input.platform && input.accountId) { + const condition = and( + eq(authorAccounts.platform, input.platform), + eq(authorAccounts.accountId, input.accountId), + ); + if (condition) identityConditions.push(condition); + } + } + + const resolved = new Map(); + if (identityConditions.length > 0) { + const existingIdentities = await client + .select({ account: authorAccounts, author: authors }) + .from(authorAccounts) + .innerJoin(authors, eq(authorAccounts.authorId, authors.id)) + .where(or(...identityConditions)); + for (const row of existingIdentities) { + resolved.set( + `${row.account.platform}:${row.account.accountId}`, + row.author, + ); + } + } + + const unresolved = [...uniqueInputs.entries()].filter( + ([key]) => !resolved.has(key), + ); + const accountIds = [ + ...new Set( + unresolved.flatMap(([, input]) => + input.accountId ? [input.accountId] : [], + ), + ), + ]; + const names = [...new Set(unresolved.map(([, input]) => input.name))]; + const legacyConditions: SQL[] = []; + if (accountIds.length > 0) { + legacyConditions.push(inArray(authors.accountId, accountIds)); + } + if (names.length > 0) { + legacyConditions.push(inArray(authors.name, names)); + } + + const legacyAuthors = + legacyConditions.length > 0 + ? await client + .select() + .from(authors) + .where(or(...legacyConditions)) + : []; + const legacyByAccount = new Map( + legacyAuthors.flatMap((author) => + author.accountId ? [[author.accountId, author] as const] : [], + ), + ); + const legacyByName = new Map( + legacyAuthors.map((author) => [author.name, author] as const), + ); + const legacyAuthorIds = legacyAuthors.map((author) => author.id); + const linkedLegacyAuthorIds = + legacyAuthorIds.length > 0 + ? new Set( + ( + await client + .select({ authorId: authorAccounts.authorId }) + .from(authorAccounts) + .where(inArray(authorAccounts.authorId, legacyAuthorIds)) + ).map((account) => account.authorId), + ) + : new Set(); + + const accountsToInsert: (typeof authorAccounts.$inferInsert)[] = []; + const authorsToInsert: (typeof authors.$inferInsert)[] = []; + for (const [key, input] of unresolved) { + const legacyByMatchingAccount = input.accountId + ? legacyByAccount.get(input.accountId) + : undefined; + const legacyByMatchingName = legacyByName.get(input.name); + const legacy = + legacyByMatchingAccount ?? + (legacyByMatchingName?.accountId ? undefined : legacyByMatchingName); + const canReuseLegacy = + legacy && (!input.platform || !linkedLegacyAuthorIds.has(legacy.id)); + + const author = + canReuseLegacy && legacy + ? legacy + : { + id: randomUUID(), + name: input.name, + accountId: input.accountId ?? null, + createdAt: new Date(), + updatedAt: new Date(), + }; + if (!canReuseLegacy) { + authorsToInsert.push({ + id: author.id, + name: author.name, + accountId: author.accountId, + }); + } + resolved.set(key, author); + + if (input.platform && input.accountId) { + accountsToInsert.push({ + authorId: author.id, + platform: input.platform, + accountId: input.accountId, + }); + } + } + + if (authorsToInsert.length > 0) { + await client.insert(authors).values(authorsToInsert); + } + if (accountsToInsert.length > 0) { + await client + .insert(authorAccounts) + .values(accountsToInsert) + .onConflictDoNothing(); + } + + return [...uniqueInputs.keys()].flatMap((key) => { + const author = resolved.get(key); + return author ? [mapAuthor(author)] : []; + }); +} + export function createAuthorRepository( getExecutor: (tx?: unknown) => DrizzleExecutor, _options?: AuthorRepositoryOptions, @@ -61,57 +210,8 @@ export function createAuthorRepository( }, async create(author: NewAuthor, tx?: unknown): Promise { - const client = getExecutor(tx); - - if (author.platform && author.accountId) { - const existingByAccount = await client - .select({ author: authors }) - .from(authorAccounts) - .innerJoin(authors, eq(authorAccounts.authorId, authors.id)) - .where( - and( - eq(authorAccounts.platform, author.platform), - eq(authorAccounts.accountId, author.accountId), - ), - ) - .limit(1); - - if (existingByAccount[0]) { - return mapAuthor(existingByAccount[0].author); - } - } else { - const condition = author.accountId - ? eq(authors.accountId, author.accountId) - : eq(authors.name, author.name); - const existing = await client - .select() - .from(authors) - .where(condition) - .limit(1); - - if (existing[0]) { - return mapAuthor(existing[0]); - } - } - - const result = await client - .insert(authors) - .values({ - name: author.name, - accountId: author.accountId, - }) - .returning(); - const created = result[0]; - - if (author.platform && author.accountId) { - await client.insert(authorAccounts).values({ - authorId: created.id, - platform: author.platform, - accountId: author.accountId, - }); - } - - return mapAuthor(created); + const result = await findOrCreateAuthorsBulk(getExecutor(tx), [author]); + return result[0]; }, async update( @@ -205,21 +305,7 @@ export function createAuthorRepository( inputs: NewAuthor[], tx?: unknown, ): Promise { - const uniqueInputs = new Map(); - for (const input of inputs) { - if (input.name.length === 0) continue; - const key = - input.platform && input.accountId - ? `${input.platform}:${input.accountId}` - : `name:${input.name}`; - uniqueInputs.set(key, input); - } - - const result: Author[] = []; - for (const input of uniqueInputs.values()) { - result.push(await this.create(input, tx)); - } - return result; + return findOrCreateAuthorsBulk(getExecutor(tx), inputs); }, }; } From b01d02bf250ad490d233e327e6f245bea2ee5c21 Mon Sep 17 00:00:00 2001 From: hmjn Date: Wed, 1 Jul 2026 00:42:57 +0900 Subject: [PATCH 07/19] fix(search): stabilize author combobox filtering --- .../components/media/pro-search-builder.tsx | 30 +++---------------- .../src/components/media/search-filters.tsx | 28 +++++++---------- packages/ui/src/pro-search-builder.tsx | 23 ++------------ packages/ui/src/search-filters.tsx | 26 +++++----------- 4 files changed, 25 insertions(+), 82 deletions(-) diff --git a/apps/server/src/components/media/pro-search-builder.tsx b/apps/server/src/components/media/pro-search-builder.tsx index 888c1f1a1..2895b79b6 100644 --- a/apps/server/src/components/media/pro-search-builder.tsx +++ b/apps/server/src/components/media/pro-search-builder.tsx @@ -25,10 +25,7 @@ import { SelectTrigger, SelectValue, } from "@solid-imager/ui/select"; -import { - createDebouncedSignal, - parseSelectValue, -} from "@solid-imager/ui/utils"; +import { parseSelectValue } from "@solid-imager/ui/utils"; import { cn } from "@solid-imager/ui/utils/cn"; import { createMemo, Index, Match, Show, Switch } from "solid-js"; @@ -333,25 +330,6 @@ function CriterionBuilder(props: { ? `${author.name}: (twitter)${author.accountId}` : author.name; - const [filterText, setFilterText] = createDebouncedSignal("", 150); - - const filteredItems = createMemo(() => { - const items = autocompleteItems(); - if (!items) return []; - const query = filterText().toLowerCase(); - if (!query) return items.slice(0, 100); - return items - .filter((item) => { - if (!item) return false; - const label = - props.criterion.target === "author" - ? getAuthorLabel(item as Author) - : (item as { name: string }).name; - return label.toLowerCase().includes(query); - }) - .slice(0, 100); - }); - // Helper to determine available items for autocomplete const autocompleteItems = createMemo(() => { switch (props.criterion.target) { @@ -484,7 +462,7 @@ function CriterionBuilder(props: { props.onChange({ ...props.criterion, value: val.name }); } }} - onInputChange={(text) => setFilterText(text)} + defaultFilter="contains" optionLabel={(item) => item ? props.criterion.target === "author" @@ -492,7 +470,7 @@ function CriterionBuilder(props: { : (item as { name: string }).name : "" } - options={filteredItems()} + options={autocompleteItems() ?? []} optionTextValue={(item) => item ? props.criterion.target === "author" @@ -500,7 +478,7 @@ function CriterionBuilder(props: { : (item as { name: string }).name : "" } - optionValue={(item: { name: string }) => item.name} + optionValue={(item) => item.id} placeholder="検索..." triggerMode="focus" value={(autocompleteItems() || []).find( diff --git a/apps/server/src/components/media/search-filters.tsx b/apps/server/src/components/media/search-filters.tsx index 48d800697..72a27321e 100644 --- a/apps/server/src/components/media/search-filters.tsx +++ b/apps/server/src/components/media/search-filters.tsx @@ -16,8 +16,7 @@ import { import { Input } from "@solid-imager/ui/input"; import { Label } from "@solid-imager/ui/label"; import { cn } from "@solid-imager/ui/utils/cn"; -import { createDebouncedSignal } from "@solid-imager/ui/utils/debounce"; -import { createMemo, createSignal, For } from "solid-js"; +import { createSignal, For } from "solid-js"; export type SearchFilterState = { searchQuery: string; @@ -56,23 +55,13 @@ function FilterSection(props: { onSelect: (item: T) => void; onRemove: (id: string) => void; getItemKey: (item: T) => string; + getSelectedItemValue?: (item: T) => string; getItemLabel: (item: T) => string; getItemDescription?: (item: T) => string | undefined | null; placeholder?: string; badgeVariant?: "default" | "destructive" | "secondary" | "outline"; }) { const [value, setValue] = createSignal(null); - const [filterText, setFilterText] = createDebouncedSignal("", 150); - - const filteredItems = createMemo(() => { - const items = props.items; - if (!items) return []; - const query = filterText().toLowerCase(); - if (!query) return items.slice(0, 100); - return items - .filter((item) => props.getItemLabel(item).toLowerCase().includes(query)) - .slice(0, 100); - }); return (
@@ -80,7 +69,10 @@ function FilterSection(props: {
{(id) => { - const item = props.items?.find((i) => props.getItemKey(i) === id); + const item = props.items?.find( + (i) => + (props.getSelectedItemValue?.(i) ?? props.getItemKey(i)) === id, + ); return ( (props: { setValue(() => val); requestAnimationFrame(() => { setValue(null); - setFilterText(""); }); } }} - onInputChange={(text) => setFilterText(text)} + defaultFilter="contains" optionLabel={props.getItemLabel} - options={filteredItems()} + options={props.items ?? []} optionTextValue={props.getItemLabel} optionValue={(item) => props.getItemKey(item)} placeholder={props.placeholder} @@ -259,8 +250,9 @@ export function SearchFilters(props: SearchFiltersProps) { {/* Author Filter */} author.name} + getItemKey={(author) => author.id} getItemLabel={getAuthorLabel} + getSelectedItemValue={(author) => author.name} items={props.authors} label="作者" onRemove={(name) => diff --git a/packages/ui/src/pro-search-builder.tsx b/packages/ui/src/pro-search-builder.tsx index 3063b4d00..dd8bbb276 100644 --- a/packages/ui/src/pro-search-builder.tsx +++ b/packages/ui/src/pro-search-builder.tsx @@ -27,7 +27,6 @@ import { SelectValue, } from "./select"; import { cn } from "./utils/cn"; -import { createDebouncedSignal } from "./utils/debounce"; import { parseSelectValue } from "./utils/parse-select-value"; const TARGET_LABELS: Record = { @@ -313,22 +312,6 @@ function CriterionBuilder(props: { ? `${author.name}：(twitter)${author.accountId}` : author.name; - const [filterText, setFilterText] = createDebouncedSignal("", 150); - - const filteredItems = createMemo(() => { - const items = autocompleteItems(); - if (!items) return []; - const query = filterText().toLowerCase(); - if (!query) return items.slice(0, 100); - return items - .filter((item) => { - if (!item) return false; - const label = "accountId" in item ? getAuthorLabel(item) : item.name; - return label.toLowerCase().includes(query); - }) - .slice(0, 100); - }); - const autocompleteItems = createMemo(() => { switch (props.criterion.target) { case "tag": @@ -479,7 +462,7 @@ function CriterionBuilder(props: { props.onChange({ ...props.criterion, value: value.name }); } }} - onInputChange={(text) => setFilterText(text)} + defaultFilter="contains" optionLabel={(item) => item ? "accountId" in item @@ -487,7 +470,7 @@ function CriterionBuilder(props: { : item.name : "" } - options={filteredItems()} + options={autocompleteItems() ?? []} optionTextValue={(item) => item ? "accountId" in item @@ -495,7 +478,7 @@ function CriterionBuilder(props: { : item.name : "" } - optionValue={(item) => item.name} + optionValue={(item) => item.id} placeholder="検索..." triggerMode="focus" value={(autocompleteItems() || []).find( diff --git a/packages/ui/src/search-filters.tsx b/packages/ui/src/search-filters.tsx index c9335d2c9..076bf8d55 100644 --- a/packages/ui/src/search-filters.tsx +++ b/packages/ui/src/search-filters.tsx @@ -4,7 +4,7 @@ import type { Ip } from "@solid-imager/core/domain/ips/schemas"; import type { Project } from "@solid-imager/core/domain/projects/schemas"; import type { SearchState } from "@solid-imager/core/domain/search/schema"; import type { TagResponse } from "@solid-imager/core/domain/tags/schemas"; -import { createMemo, createSignal, For } from "solid-js"; +import { createSignal, For } from "solid-js"; import type { SetStoreFunction } from "solid-js/store"; import { Badge } from "./badge"; import { Button } from "./button"; @@ -19,7 +19,6 @@ import { import { Input } from "./input"; import { Label } from "./label"; import { cn } from "./utils/cn"; -import { createDebouncedSignal } from "./utils/debounce"; type SearchFiltersProps = { state: SearchState; @@ -41,23 +40,13 @@ function FilterSection(props: { onSelect: (item: T) => void; onRemove: (id: string) => void; getItemKey: (item: T) => string; + getSelectedItemValue?: (item: T) => string; getItemLabel: (item: T) => string; getItemDescription?: (item: T) => string | undefined | null; placeholder?: string; badgeVariant?: "default" | "destructive" | "secondary" | "outline"; }) { const [value, setValue] = createSignal(null); - const [filterText, setFilterText] = createDebouncedSignal("", 150); - - const filteredItems = createMemo(() => { - const items = props.items; - if (!items) return []; - const query = filterText().toLowerCase(); - if (!query) return items.slice(0, 100); - return items - .filter((item) => props.getItemLabel(item).toLowerCase().includes(query)) - .slice(0, 100); - }); return (
@@ -66,7 +55,8 @@ function FilterSection(props: { {(id) => { const item = props.items?.find( - (i) => props.getItemKey(i) === id, + (i) => + (props.getSelectedItemValue?.(i) ?? props.getItemKey(i)) === id, ) as T; return ( (props: { setValue(() => val); requestAnimationFrame(() => { setValue(null); - setFilterText(""); }); } }} - onInputChange={(text) => setFilterText(text)} + defaultFilter="contains" optionLabel={props.getItemLabel} - options={filteredItems()} + options={props.items ?? []} optionTextValue={props.getItemLabel} optionValue={(item) => props.getItemKey(item)} placeholder={props.placeholder} @@ -240,8 +229,9 @@ export function SearchFilters(props: SearchFiltersProps) { author.name} + getItemKey={(author) => author.id} getItemLabel={getAuthorLabel} + getSelectedItemValue={(author) => author.name} items={props.authors} label="作者" onRemove={(name) => From 80245943dd076e8781c93013c6286a69aea3ac8d Mon Sep 17 00:00:00 2001 From: hmjn Date: Wed, 1 Jul 2026 00:46:15 +0900 Subject: [PATCH 08/19] chore: add CodeRabbit config for develop branch reviews --- .coderabbit.yaml | 14 +++++ .opencode/package-lock.json | 122 ++++++++++++++++++------------------ 2 files changed, 75 insertions(+), 61 deletions(-) create mode 100644 .coderabbit.yaml diff --git a/.coderabbit.yaml b/.coderabbit.yaml new file mode 100644 index 000000000..85abf65c4 --- /dev/null +++ b/.coderabbit.yaml @@ -0,0 +1,14 @@ +language: ja-JP + +reviews: + profile: chill + auto_review: + enabled: true + auto_incremental_review: true + drafts: false + base_branches: + - develop + ignore_title_keywords: + - WIP + - "DO NOT MERGE" + - "[skip review]" diff --git a/.opencode/package-lock.json b/.opencode/package-lock.json index 95a4816d7..f78ace43b 100644 --- a/.opencode/package-lock.json +++ b/.opencode/package-lock.json @@ -5,13 +5,13 @@ "packages": { "": { "dependencies": { - "@opencode-ai/plugin": "1.14.50" + "@opencode-ai/plugin": "1.16.2" } }, "node_modules/@msgpackr-extract/msgpackr-extract-darwin-arm64": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.3.tgz", - "integrity": "sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw==", + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.4.tgz", + "integrity": "sha512-LCkGo6JDfaBhgST7UpPWgNgLINpcpabaHfyz5OBx75nUYxBsaEPxjnyNjWpeb/xBup/682QnBfRBy2/LvPutZQ==", "cpu": [ "arm64" ], @@ -22,9 +22,9 @@ ] }, "node_modules/@msgpackr-extract/msgpackr-extract-darwin-x64": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.3.tgz", - "integrity": "sha512-mdzd3AVzYKuUmiWOQ8GNhl64/IoFGol569zNRdkLReh6LRLHOXxU4U8eq0JwaD8iFHdVGqSy4IjFL4reoWCDFw==", + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.4.tgz", + "integrity": "sha512-zExlW9zUJKZH/tOtVMttwjKa4Xm/3KcNjnE3dPN92uCktwavMxpgCA3MoJK/DOnTWsQgo224OaST27/mPNAf+w==", "cpu": [ "x64" ], @@ -35,9 +35,9 @@ ] }, "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.3.tgz", - "integrity": "sha512-fg0uy/dG/nZEXfYilKoRe7yALaNmHoYeIoJuJ7KJ+YyU2bvY8vPv27f7UKhGRpY6euFYqEVhxCFZgAUNQBM3nw==", + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.4.tgz", + "integrity": "sha512-Tg3yX65f5GbtXLkrYEHE5oibZG9epyYWas7FogTTEJeDEF9JlXJzKgXaNhT3UXlTOeA+AfZpYZYZ0uPj7Cfquw==", "cpu": [ "arm" ], @@ -48,9 +48,9 @@ ] }, "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm64": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.3.tgz", - "integrity": "sha512-YxQL+ax0XqBJDZiKimS2XQaf+2wDGVa1enVRGzEvLLVFeqa5kx2bWbtcSXgsxjQB7nRqqIGFIcLteF/sHeVtQg==", + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.4.tgz", + "integrity": "sha512-dgX0P/9wGPJeHFBG+ZmhgE6bmtMt7NP5CRBGyyktpopdk/mW4POnrpQsSLtKI1dwpc+pPLuXHDh6vvskyQE/sw==", "cpu": [ "arm64" ], @@ -61,9 +61,9 @@ ] }, "node_modules/@msgpackr-extract/msgpackr-extract-linux-x64": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.3.tgz", - "integrity": "sha512-cvwNfbP07pKUfq1uH+S6KJ7dT9K8WOE4ZiAcsrSes+UY55E/0jLYc+vq+DO7jlmqRb5zAggExKm0H7O/CBaesg==", + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.4.tgz", + "integrity": "sha512-8TNXMEjJc3QEy7R/x1INhgiU+XakDAFUzBhaz7+Rbrs8NH5UQeHQxxmzsSBJGyV6I1jW79undiQm8tOI+D+8FQ==", "cpu": [ "x64" ], @@ -74,9 +74,9 @@ ] }, "node_modules/@msgpackr-extract/msgpackr-extract-win32-x64": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.3.tgz", - "integrity": "sha512-x0fWaQtYp4E6sktbsdAqnehxDgEc/VwM7uLsRCYWaiGu0ykYdZPiS8zCWdnjHwyiumousxfBm4SO31eXqwEZhQ==", + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.4.tgz", + "integrity": "sha512-CmCXPQrkbwExx3j946/PtHWHbYJiCRBRDl4BlkRQcJB/YOwQxJRTpoo7aTsortjgoJ1x7opzTSxn7C+ASSLVjQ==", "cpu": [ "x64" ], @@ -87,19 +87,19 @@ ] }, "node_modules/@opencode-ai/plugin": { - "version": "1.14.50", - "resolved": "https://registry.npmjs.org/@opencode-ai/plugin/-/plugin-1.14.50.tgz", - "integrity": "sha512-2D4k6r8IFaAajEmezOZ3UKmRRGqEw2YzqotH5zLGT434yKtabduEsQgXa0fWlASut4FVT3JURhq5LqeBUV/k4g==", + "version": "1.16.2", + "resolved": "https://registry.npmjs.org/@opencode-ai/plugin/-/plugin-1.16.2.tgz", + "integrity": "sha512-FaZhVXrbz93xsdGLCtarRDTeqFt8AkLfh8B34tFBj6G4HXVmKSgBwVXmtELKKC+08xMtawBC9hshiMbXryv6cg==", "license": "MIT", "dependencies": { - "@opencode-ai/sdk": "1.14.50", - "effect": "4.0.0-beta.65", + "@opencode-ai/sdk": "1.16.2", + "effect": "4.0.0-beta.74", "zod": "4.1.8" }, "peerDependencies": { - "@opentui/core": ">=0.2.9", - "@opentui/keymap": ">=0.2.9", - "@opentui/solid": ">=0.2.9" + "@opentui/core": ">=0.3.2", + "@opentui/keymap": ">=0.3.2", + "@opentui/solid": ">=0.3.2" }, "peerDependenciesMeta": { "@opentui/core": { @@ -114,9 +114,9 @@ } }, "node_modules/@opencode-ai/sdk": { - "version": "1.14.50", - "resolved": "https://registry.npmjs.org/@opencode-ai/sdk/-/sdk-1.14.50.tgz", - "integrity": "sha512-IrTKTFviR4Tj+u0BI8h8XgXIvEpxwkkHqBj6E0aEc4DBErpS3qh2Lkp1xt0OdtCYiLE3wZ1bAIiHOiO66H/7TA==", + "version": "1.16.2", + "resolved": "https://registry.npmjs.org/@opencode-ai/sdk/-/sdk-1.16.2.tgz", + "integrity": "sha512-Z/xZ7q79dYeE0afqIk/yFEcRNGEQFcE+H8ssYivUiy+xGZ1mGwT72jpaQZKBwPn3JH4sRCu4KA2lcktBQfcOjg==", "license": "MIT", "dependencies": { "cross-spawn": "7.0.6" @@ -153,21 +153,21 @@ } }, "node_modules/effect": { - "version": "4.0.0-beta.65", - "resolved": "https://registry.npmjs.org/effect/-/effect-4.0.0-beta.65.tgz", - "integrity": "sha512-QYKvQPAj3CmtsvWkHQww15wX4KG2gNsszDWEcOO5sZCMknp66u6Si/Opmt3wwWCwsyvRmDAdIg+JIz5qzbbFIw==", + "version": "4.0.0-beta.74", + "resolved": "https://registry.npmjs.org/effect/-/effect-4.0.0-beta.74.tgz", + "integrity": "sha512-Yx+Kh12U+i2FmjwEfKs+ePFmpMd43RPD1oGqc/VraSS9bYzvF0Ff3PojwEFEVEewp8xc92Uxu28gTspU4qyvHA==", "license": "MIT", "dependencies": { "@standard-schema/spec": "^1.1.0", - "fast-check": "^4.6.0", + "fast-check": "^4.8.0", "find-my-way-ts": "^0.1.6", - "ini": "^6.0.0", + "ini": "^7.0.0", "kubernetes-types": "^1.30.0", - "msgpackr": "^1.11.9", + "msgpackr": "^2.0.1", "multipasta": "^0.2.7", "toml": "^4.1.1", - "uuid": "^13.0.0", - "yaml": "^2.8.3" + "uuid": "^14.0.0", + "yaml": "^2.9.0" } }, "node_modules/fast-check": { @@ -199,12 +199,12 @@ "license": "MIT" }, "node_modules/ini": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/ini/-/ini-6.0.0.tgz", - "integrity": "sha512-IBTdIkzZNOpqm7q3dRqJvMaldXjDHWkEDfrwGEQTs5eaQMWV+djAhR+wahyNNMAa+qpbDUhBMVt4ZKNwpPm7xQ==", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/ini/-/ini-7.0.0.tgz", + "integrity": "sha512-ifK0CgjALofS5bkrcTy4RaQ9Vx2Knf/eLeIO+NaswQEpH1UblrtTSCIvN71qQDMq0PeQ/SSPojvEJp9vvvfr+w==", "license": "ISC", "engines": { - "node": "^20.17.0 || >=22.9.0" + "node": "^22.22.2 || ^24.15.0 || >=26.0.0" } }, "node_modules/isexe": { @@ -220,18 +220,18 @@ "license": "Apache-2.0" }, "node_modules/msgpackr": { - "version": "1.11.12", - "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.11.12.tgz", - "integrity": "sha512-RBdJ1Un7yGlXWajrkxcSa93nvQ0w4zBf60c0yYv7YtBelP8H2FA7XsfBbMHtXKXUMUxH7zV3Zuozh+kUQWhHvg==", + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-2.0.4.tgz", + "integrity": "sha512-o1C5KRmuRt+apqMr1HuGSqWStZoRBUpEsCsl15uM9VdAF1qHLtvMOU2En747EnTyEl6c4pzPewRMFF31s1CNbA==", "license": "MIT", "optionalDependencies": { - "msgpackr-extract": "^3.0.2" + "msgpackr-extract": "^3.0.4" } }, "node_modules/msgpackr-extract": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-3.0.3.tgz", - "integrity": "sha512-P0efT1C9jIdVRefqjzOQ9Xml57zpOXnIuS+csaB4MdZbTdmGDLo8XhzBG1N7aO11gKDDkJvBLULeFTo46wwreA==", + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-3.0.4.tgz", + "integrity": "sha512-4kmO/MdyUIkLIvTPr8VHLil4AtoKIoniWPIEk5+CDy0xnWC84azhSFmuJ7PxZdsYtiP5kEeQsORAVIeMgxT+Hw==", "hasInstallScript": true, "license": "MIT", "optional": true, @@ -242,12 +242,12 @@ "download-msgpackr-prebuilds": "bin/download-prebuilds.js" }, "optionalDependencies": { - "@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.3", - "@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.3", - "@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.3", - "@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.3", - "@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.3", - "@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.3" + "@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.4", + "@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.4", + "@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.4", + "@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.4", + "@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.4", + "@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.4" } }, "node_modules/multipasta": { @@ -281,9 +281,9 @@ } }, "node_modules/pure-rand": { - "version": "8.4.0", - "resolved": "https://registry.npmjs.org/pure-rand/-/pure-rand-8.4.0.tgz", - "integrity": "sha512-IoM8YF/jY0hiugFo/wOWqfmarlE6J0wc6fDK1PhftMk7MGhVZl88sZimmqBBFomLOCSmcCCpsfj7wXASCpvK9A==", + "version": "8.4.1", + "resolved": "https://registry.npmjs.org/pure-rand/-/pure-rand-8.4.1.tgz", + "integrity": "sha512-c58R2+SPFcSIPXoU834QN/KPDDOSd8sXcSrqf6e83Me6Rrp1EYkxukkjXMVrKvKaADs1SOyNkWdfvLf6zY8qLQ==", "funding": [ { "type": "individual", @@ -327,9 +327,9 @@ } }, "node_modules/uuid": { - "version": "13.0.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-13.0.2.tgz", - "integrity": "sha512-vzi9uRZ926x4XV73S/4qQaTwPXM2JBj6/6lI/byHH1jOpCzb0zDbfytgA9LcN/hzb2l7WQSQnxITOVx5un/wGw==", + "version": "14.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-14.0.1.tgz", + "integrity": "sha512-6ZxzVpzDXDa3bJWaHilVayA+BH/1zmxCJoVgvmqJnid/gPoKHxUrS/aC/T6LGQtNHT+XHG9fXPJB4d+IrU30Ew==", "funding": [ "https://github.com/sponsors/broofa", "https://github.com/sponsors/ctavan" From d5f562fd0c23ccbcf053c093263bfdf3a5ab74b3 Mon Sep 17 00:00:00 2001 From: hmjn Date: Wed, 1 Jul 2026 23:45:48 +0900 Subject: [PATCH 09/19] fix: address PR review findings --- .../application/services/backup-service.ts | 253 +++++++++--------- .../components/media/pro-search-builder.tsx | 4 +- .../src/components/media/search-filters.tsx | 4 +- .../integration/backup/backup-service.test.ts | 82 ++++++ .../repository/author-dedupe.test.ts | 6 + .../services/lancedb-dump-service.test.ts | 43 +++ apps/xtracter/src/content/fanbox.ts | 12 +- .../src/services/lancedb-dump-service.ts | 22 ++ .../db/src/repositories/author-repository.ts | 9 +- packages/ui/src/pro-search-builder.tsx | 4 +- packages/ui/src/search-filters.tsx | 4 +- 11 files changed, 306 insertions(+), 137 deletions(-) diff --git a/apps/server/src/application/services/backup-service.ts b/apps/server/src/application/services/backup-service.ts index c12f414fb..792db8d2b 100644 --- a/apps/server/src/application/services/backup-service.ts +++ b/apps/server/src/application/services/backup-service.ts @@ -8,7 +8,7 @@ import { import { localConnectionSchema } from "@solid-imager/core/domain/sources/schemas"; import { getErrorMessage } from "@solid-imager/core/utils/get-error-message"; import type { Table } from "drizzle-orm"; -import { and, asc, eq, gt, inArray, lt, sql } from "drizzle-orm"; +import { and, asc, eq, gt, inArray, lt, or, sql } from "drizzle-orm"; import type { PgColumn } from "drizzle-orm/pg-core"; import { db } from "~/infrastructure/db"; import { @@ -58,6 +58,16 @@ type JsonValue = const LanceDbDirtyMaxAttempts = 5; +function authorRestoreKey(author: { + name: string; + platform?: AuthorPlatform; + accountId?: string | null; +}): string { + return author.platform && author.accountId + ? `${author.platform}:${author.accountId}` + : `name:${author.name}`; +} + interface MediaListQueryItem { id: string; filePath: string; @@ -334,7 +344,14 @@ export const BackupService = { async _restoreMasterData(validItems: MediaDumpItem[], _tx?: BackupDbClient) { const tagNames = new Set(); - const authorData = new Map(); + const authorData = new Map< + string, + { + name: string; + accountId?: string | null; + platform?: AuthorPlatform; + } + >(); const projectNames = new Set(); const charNames = new Set(); const ipNames = new Set(); @@ -349,9 +366,8 @@ export const BackupService = { } if (item.authors) { for (const a of item.authors) { - // Preserve accountId if available - if (a.name && (!authorData.has(a.name) || a.accountId)) { - authorData.set(a.name, { accountId: a.accountId }); + if (a.name) { + authorData.set(authorRestoreKey(a), a); } } } @@ -395,14 +411,99 @@ export const BackupService = { }, _tx, ); - const authorMap = await this._ensureMasterDataWithExtras( - authors, - authors.id, - authors.name, - authors.accountId, - authorData, - _tx, - ); + const authorInputs = [...authorData.values()]; + const d = _tx ?? db; + const authorMap = new Map(); + for (const input of authorInputs) { + let authorId: string | undefined; + if (input.platform && input.accountId) { + const existingAccount = await d + .select({ authorId: authorAccounts.authorId }) + .from(authorAccounts) + .where( + and( + eq(authorAccounts.platform, input.platform), + eq(authorAccounts.accountId, input.accountId), + ), + ) + .limit(1); + authorId = existingAccount[0]?.authorId; + + if (!authorId) { + const legacyCandidates = await d + .select() + .from(authors) + .where( + or( + eq(authors.accountId, input.accountId), + eq(authors.name, input.name), + ), + ); + const candidateIds = legacyCandidates.map((author) => author.id); + const linkedIds = + candidateIds.length > 0 + ? new Set( + ( + await d + .select({ authorId: authorAccounts.authorId }) + .from(authorAccounts) + .where(inArray(authorAccounts.authorId, candidateIds)) + ).map((account) => account.authorId), + ) + : new Set(); + const reusable = legacyCandidates.find( + (author) => + !linkedIds.has(author.id) && + (author.accountId === input.accountId || + (author.name === input.name && !author.accountId)), + ); + if (reusable) { + authorId = reusable.id; + } else { + const inserted = await d + .insert(authors) + .values({ + name: input.name, + accountId: input.accountId, + }) + .returning(); + authorId = inserted[0]?.id; + } + if (authorId) { + await d + .insert(authorAccounts) + .values({ + authorId, + platform: input.platform, + accountId: input.accountId, + }) + .onConflictDoNothing(); + } + } + } else { + const existing = await d + .select({ id: authors.id }) + .from(authors) + .where(eq(authors.name, input.name)) + .limit(1); + authorId = existing[0]?.id; + if (!authorId) { + const inserted = await d + .insert(authors) + .values({ + name: input.name, + accountId: input.accountId ?? null, + }) + .returning(); + authorId = inserted[0]?.id; + } + } + if (!authorId) continue; + authorMap.set(authorRestoreKey(input), authorId); + if (!authorMap.has(`name:${input.name}`)) { + authorMap.set(`name:${input.name}`, authorId); + } + } const restoredAuthorAccounts = new Map< string, { @@ -414,7 +515,7 @@ export const BackupService = { for (const item of validItems) { for (const author of item.authors ?? []) { if (!(author.platform && author.accountId)) continue; - const authorId = authorMap.get(author.name); + const authorId = authorMap.get(authorRestoreKey(author)); if (!authorId) continue; const key = `${author.platform}:${author.accountId}`; restoredAuthorAccounts.set(key, { @@ -605,7 +706,9 @@ export const BackupService = { if (item.authors) { for (const a of item.authors) { - const authorId = a.name ? authorMap.get(a.name) : undefined; + const authorId = a.name + ? authorMap.get(authorRestoreKey(a)) + : undefined; if (authorId) { mediaAuthorsData.push({ mediaId, authorId }); } @@ -794,106 +897,6 @@ export const BackupService = { return new Map(records.map((r) => [r.name, r.id])); }, - /** - * Ensures master data with extra fields (specifically for authors with accountId). - * Authors table has no unique constraint on name (display names can overlap), - * so we use manual dedup by name and update accountId when available. - */ - async _ensureMasterDataWithExtras( - table: Table, - idColumn: PgColumn, - nameColumn: PgColumn, - accountIdColumn: PgColumn, - dataMap: Map, - _tx?: BackupDbClient, - ): Promise> { - const d = _tx ?? db; - if (dataMap.size === 0) { - return new Map(); - } - - const entries = Array.from(dataMap.entries()); - const nameList = entries.map(([name]) => name); - - // First, find existing authors by name - const existingRecords = (await d - .select({ - id: idColumn, - name: nameColumn, - accountId: accountIdColumn, - }) - .from(table) - .where(inArray(nameColumn, nameList))) as { - id: string; - name: string; - accountId: string | null; - }[]; - - const existingByName = new Map< - string, - { id: string; accountId: string | null } - >(); - for (const r of existingRecords) { - const existing = existingByName.get(r.name); - // Prioritize entry with an accountId, or just take the first one if none have it yet. - if (!existing || (!existing.accountId && r.accountId)) { - existingByName.set(r.name, { id: r.id, accountId: r.accountId }); - } - } - - // Update existing authors with new accountId if provided - // Note: Update ALL authors with matching name (handles duplicates) - for (const [name, data] of entries) { - const existing = existingByName.get(name); - if (existing && data.accountId && existing.accountId !== data.accountId) { - await d - .update(table) - .set({ accountId: data.accountId }) - .where(eq(nameColumn, name)); - } - } - - // Insert new authors that don't exist - const newEntries = entries.filter(([name]) => !existingByName.has(name)); - if (newEntries.length > 0) { - await d.insert(table).values( - newEntries.map(([name, data]) => ({ - name, - accountId: data.accountId || null, - })), - ); - - // Fetch the newly inserted records - const newRecords = (await d - .select({ - id: idColumn, - name: nameColumn, - accountId: accountIdColumn, - }) - .from(table) - .where( - inArray( - nameColumn, - newEntries.map(([name]) => name), - ), - )) as { id: string; name: string; accountId: string | null }[]; - - for (const r of newRecords) { - if (!existingByName.has(r.name)) { - existingByName.set(r.name, { id: r.id, accountId: r.accountId }); - } - } - } - - // Return map of name -> id - return new Map( - Array.from(existingByName.entries()).map(([name, data]) => [ - name, - data.id, - ]), - ); - }, - /** * Imports media data from a ZIP file path. */ @@ -1743,13 +1746,21 @@ export const BackupService = { })); // Extract authors - const simpleAuthors = (media.authors || []).map((ma) => ({ - name: ma.author?.name || "", - accountId: ma.author?.accountId ?? null, - platform: ma.author?.accounts?.find( + const simpleAuthors = (media.authors || []).map((ma) => { + const legacyAccount = ma.author?.accounts?.find( (account) => account.accountId === ma.author?.accountId, - )?.platform, - })); + ); + const account = + legacyAccount ?? + (ma.author?.accounts?.length === 1 + ? ma.author.accounts[0] + : undefined); + return { + name: ma.author?.name || "", + accountId: account?.accountId ?? ma.author?.accountId ?? null, + platform: account?.platform, + }; + }); // Extract characters const simpleCharacters = (media.characters || []).map((mc) => ({ diff --git a/apps/server/src/components/media/pro-search-builder.tsx b/apps/server/src/components/media/pro-search-builder.tsx index 2895b79b6..22f7344dd 100644 --- a/apps/server/src/components/media/pro-search-builder.tsx +++ b/apps/server/src/components/media/pro-search-builder.tsx @@ -326,9 +326,7 @@ function CriterionBuilder(props: { tags?: TagResponse[]; }) { const getAuthorLabel = (author: Author) => - author.accountId - ? `${author.name}: (twitter)${author.accountId}` - : author.name; + author.accountId ? `${author.name}: ${author.accountId}` : author.name; // Helper to determine available items for autocomplete const autocompleteItems = createMemo(() => { diff --git a/apps/server/src/components/media/search-filters.tsx b/apps/server/src/components/media/search-filters.tsx index 72a27321e..97d193689 100644 --- a/apps/server/src/components/media/search-filters.tsx +++ b/apps/server/src/components/media/search-filters.tsx @@ -125,9 +125,7 @@ function FilterSection(props: { } const getAuthorLabel = (author: Author) => - author.accountId - ? `${author.name}：(twitter)${author.accountId}` - : author.name; + author.accountId ? `${author.name}：${author.accountId}` : author.name; export function SearchFilters(props: SearchFiltersProps) { const addTag = (tagName: string) => { diff --git a/apps/server/src/tests/integration/backup/backup-service.test.ts b/apps/server/src/tests/integration/backup/backup-service.test.ts index 998a8312a..f21e6f613 100644 --- a/apps/server/src/tests/integration/backup/backup-service.test.ts +++ b/apps/server/src/tests/integration/backup/backup-service.test.ts @@ -178,6 +178,35 @@ describe("BackupService Integration", () => { expect(item.sourceUrls[0]).toBe("https://example.com/source"); }); + it("exports a sole platform account when the legacy account ID is absent", () => { + const [item] = BackupService._transformMediaList([ + { + authors: [ + { + author: { + name: "Account-only Author", + accountId: null, + accounts: [ + { + platform: "pixiv-fanbox", + accountId: "account-only", + }, + ], + }, + }, + ], + }, + ]); + + expect(item?.authors).toEqual([ + { + name: "Account-only Author", + accountId: "account-only", + platform: "pixiv-fanbox", + }, + ]); + }); + it("should restore projects, characters, ips, authors, and sourceUrls from dump item", async () => { // Update source type to s3 to bypass fs.access check await db @@ -259,6 +288,59 @@ describe("BackupService Integration", () => { expect(restoredMedia?.urls[0].url).toBe("https://example.com/restore"); }); + it("restores same-name authors separately by platform account identity", async () => { + await db + .update(mediaSources) + .set({ type: "s3" }) + .where(eq(mediaSources.id, testSourceId)); + + const baseItem = { + mediaType: "image", + width: 100, + height: 100, + fileSize: 1024, + createdAt: new Date().toISOString(), + modifiedAt: new Date().toISOString(), + }; + const result = await BackupService.restoreSource(testSourceId, [ + { + ...baseItem, + filePath: "fanbox.png", + fileName: "fanbox.png", + authors: [ + { + name: "Shared Name", + accountId: "fanbox-creator", + platform: "pixiv-fanbox", + }, + ], + }, + { + ...baseItem, + filePath: "twitter.png", + fileName: "twitter.png", + authors: [ + { + name: "Shared Name", + accountId: "twitter-creator", + platform: "twitter", + }, + ], + }, + ]); + + expect(result.processed).toBe(2); + const restored = await db.query.medias.findMany({ + where: eq(medias.mediaSourceId, testSourceId), + with: { authors: { with: { author: { with: { accounts: true } } } } }, + }); + const authorIds = restored.map((media) => media.authors[0]?.author.id); + expect(new Set(authorIds).size).toBe(2); + expect( + restored.map((media) => media.authors[0]?.author.accounts[0]?.platform), + ).toEqual(expect.arrayContaining(["pixiv-fanbox", "twitter"])); + }); + it("should infer character-IP relationships from media when linkedIps is not provided", async () => { // Update source type to s3 to bypass fs.access check await db diff --git a/apps/server/src/tests/integration/repository/author-dedupe.test.ts b/apps/server/src/tests/integration/repository/author-dedupe.test.ts index 9bb19478a..dba14ae99 100644 --- a/apps/server/src/tests/integration/repository/author-dedupe.test.ts +++ b/apps/server/src/tests/integration/repository/author-dedupe.test.ts @@ -22,6 +22,12 @@ describe("AuthorRepository Deduplication", () => { await db.delete(authors); }); + it("rejects an empty author name", async () => { + await expect( + AuthorRepository.create({ name: "", accountId: null }), + ).rejects.toThrow("Author name cannot be empty"); + }); + it("should NOT create duplicate authors when accountId is missing but name matches", async () => { // 1. Create first author without accountId const author1 = await AuthorRepository.create({ diff --git a/apps/server/src/tests/unit/application/services/lancedb-dump-service.test.ts b/apps/server/src/tests/unit/application/services/lancedb-dump-service.test.ts index 2134c82db..b5afde635 100644 --- a/apps/server/src/tests/unit/application/services/lancedb-dump-service.test.ts +++ b/apps/server/src/tests/unit/application/services/lancedb-dump-service.test.ts @@ -1,3 +1,6 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; import type { MediaDumpItem } from "@solid-imager/core/domain/media/schemas"; import { beforeEach, describe, expect, it, vi } from "vite-plus/test"; @@ -247,4 +250,44 @@ describe("LanceDB Dump Service", () => { const authorTable = await mockOpenTable.mock.results[2].value; expect(authorTable.addColumns).toHaveBeenCalledTimes(1); }); + + it("updates the manifest version after delta schema migration", async () => { + const { createLanceDbDumpService } = await import( + "@solid-imager/application/services/lancedb-dump-service" + ); + const service = createLanceDbDumpService({ connect: createMockConnect() }); + const dumpDir = await fs.mkdtemp(path.join(os.tmpdir(), "lancedb-v3-")); + await fs.writeFile( + path.join(dumpDir, "manifest.json"), + JSON.stringify({ + format: "solid-imager-lancedb", + version: 3, + includeImages: false, + }), + ); + + try { + await service.syncLanceDBDelta(dumpDir, { + itemsToUpsert: [ + { + id: "new-media", + filePath: "new.png", + fileName: "new.png", + mediaType: "image", + }, + ], + }); + + const manifest = JSON.parse( + await fs.readFile(path.join(dumpDir, "manifest.json"), "utf8"), + ); + expect(manifest).toMatchObject({ + format: "solid-imager-lancedb", + version: 4, + includeImages: false, + }); + } finally { + await fs.rm(dumpDir, { recursive: true, force: true }); + } + }); }); diff --git a/apps/xtracter/src/content/fanbox.ts b/apps/xtracter/src/content/fanbox.ts index 72c7caa68..e44e69154 100644 --- a/apps/xtracter/src/content/fanbox.ts +++ b/apps/xtracter/src/content/fanbox.ts @@ -48,7 +48,12 @@ function getFanboxImageUrl( postId: string, ): string | null { const linkedUrl = imageElement.closest("a")?.href; - const candidates = [linkedUrl, imageElement.currentSrc, imageElement.src]; + const candidates = [ + linkedUrl, + imageElement.dataset.src, + imageElement.currentSrc, + imageElement.src, + ]; for (const candidate of candidates) { if (!candidate) { @@ -78,8 +83,9 @@ function extractMetadata(targetUrl: string, postUrl: string): TweetMetadata { ? hostname.slice(0, -".fanbox.cc".length) : ""; const authorName = - document.querySelector('a[href="/"] h1')?.innerText.trim() || - creatorId; + document + .querySelector('a[href="/"] h1') + ?.textContent?.trim() || creatorId; const authors: Author[] = []; if (authorName || creatorId) { diff --git a/packages/application/src/services/lancedb-dump-service.ts b/packages/application/src/services/lancedb-dump-service.ts index 8f2665062..1e778262a 100644 --- a/packages/application/src/services/lancedb-dump-service.ts +++ b/packages/application/src/services/lancedb-dump-service.ts @@ -16,6 +16,27 @@ const ROW_CHUNK_SIZE = 5000; const READ_CHUNK_SIZE = 5000; const LANCEDB_DUMP_VERSION = 4; +async function updateLanceDbManifestVersion(lanceDbDir: string): Promise { + const manifestPath = path.join(lanceDbDir, "manifest.json"); + let manifest: unknown; + try { + manifest = JSON.parse(await fs.readFile(manifestPath, "utf8")); + } catch { + return; + } + if ( + typeof manifest !== "object" || + manifest === null || + Array.isArray(manifest) + ) { + return; + } + await fs.writeFile( + manifestPath, + JSON.stringify({ ...manifest, version: LANCEDB_DUMP_VERSION }, null, 2), + ); +} + type JsonValue = | string | number @@ -793,6 +814,7 @@ export function createLanceDbDumpService(deps?: { await migrateLanceDbSchema(tables); await deleteMediaRows(tables, targetIds); await addRowsToTables(tables, itemsToRows(itemsToUpsert)); + await updateLanceDbManifestVersion(lanceDbDir); if (options.optimize) { await Promise.all( diff --git a/packages/db/src/repositories/author-repository.ts b/packages/db/src/repositories/author-repository.ts index 4e1aa525b..a46d7ebcb 100644 --- a/packages/db/src/repositories/author-repository.ts +++ b/packages/db/src/repositories/author-repository.ts @@ -210,8 +210,15 @@ export function createAuthorRepository( }, async create(author: NewAuthor, tx?: unknown): Promise { + if (author.name.trim().length === 0) { + throw new Error("Author name cannot be empty"); + } const result = await findOrCreateAuthorsBulk(getExecutor(tx), [author]); - return result[0]; + const created = result[0]; + if (!created) { + throw new Error("Failed to create author"); + } + return created; }, async update( diff --git a/packages/ui/src/pro-search-builder.tsx b/packages/ui/src/pro-search-builder.tsx index dd8bbb276..50edad0c0 100644 --- a/packages/ui/src/pro-search-builder.tsx +++ b/packages/ui/src/pro-search-builder.tsx @@ -308,9 +308,7 @@ function CriterionBuilder(props: { authors?: Author[]; }) { const getAuthorLabel = (author: Author) => - author.accountId - ? `${author.name}：(twitter)${author.accountId}` - : author.name; + author.accountId ? `${author.name}：${author.accountId}` : author.name; const autocompleteItems = createMemo(() => { switch (props.criterion.target) { diff --git a/packages/ui/src/search-filters.tsx b/packages/ui/src/search-filters.tsx index 076bf8d55..2e22d4226 100644 --- a/packages/ui/src/search-filters.tsx +++ b/packages/ui/src/search-filters.tsx @@ -110,9 +110,7 @@ function FilterSection(props: { } const getAuthorLabel = (author: Author) => - author.accountId - ? `${author.name}：(twitter)${author.accountId}` - : author.name; + author.accountId ? `${author.name}：${author.accountId}` : author.name; export function SearchFilters(props: SearchFiltersProps) { const addTag = (tagName: string) => { From 1b2675831ee6e61cacbd65f659f59ffbfc1428d7 Mon Sep 17 00:00:00 2001 From: hmjn Date: Wed, 1 Jul 2026 23:56:01 +0900 Subject: [PATCH 10/19] fix(ui): restore combobox filtering limits --- .../components/media/pro-search-builder.tsx | 26 ++++++++++++++++--- .../src/components/media/search-filters.tsx | 18 ++++++++++--- packages/ui/src/pro-search-builder.tsx | 19 ++++++++++++-- packages/ui/src/search-filters.tsx | 18 ++++++++++--- 4 files changed, 70 insertions(+), 11 deletions(-) diff --git a/apps/server/src/components/media/pro-search-builder.tsx b/apps/server/src/components/media/pro-search-builder.tsx index 22f7344dd..8ea5a0c13 100644 --- a/apps/server/src/components/media/pro-search-builder.tsx +++ b/apps/server/src/components/media/pro-search-builder.tsx @@ -25,7 +25,10 @@ import { SelectTrigger, SelectValue, } from "@solid-imager/ui/select"; -import { parseSelectValue } from "@solid-imager/ui/utils"; +import { + createDebouncedSignal, + parseSelectValue, +} from "@solid-imager/ui/utils"; import { cn } from "@solid-imager/ui/utils/cn"; import { createMemo, Index, Match, Show, Switch } from "solid-js"; @@ -345,6 +348,23 @@ function CriterionBuilder(props: { return; } }); + const [filterText, setFilterText] = createDebouncedSignal("", 150); + const filteredItems = createMemo(() => { + const items = autocompleteItems(); + if (!items) return []; + const query = filterText().toLowerCase(); + if (!query) return items.slice(0, 100); + return items + .filter((item) => { + if (!item) return false; + const label = + props.criterion.target === "author" + ? getAuthorLabel(item as Author) + : (item as { name: string }).name; + return label.toLowerCase().includes(query); + }) + .slice(0, 100); + }); const getValidOperators = (target: string) => { // Fast path checks using sets or direct includes @@ -460,7 +480,7 @@ function CriterionBuilder(props: { props.onChange({ ...props.criterion, value: val.name }); } }} - defaultFilter="contains" + onInputChange={(text) => setFilterText(text)} optionLabel={(item) => item ? props.criterion.target === "author" @@ -468,7 +488,7 @@ function CriterionBuilder(props: { : (item as { name: string }).name : "" } - options={autocompleteItems() ?? []} + options={filteredItems()} optionTextValue={(item) => item ? props.criterion.target === "author" diff --git a/apps/server/src/components/media/search-filters.tsx b/apps/server/src/components/media/search-filters.tsx index 97d193689..b9c029206 100644 --- a/apps/server/src/components/media/search-filters.tsx +++ b/apps/server/src/components/media/search-filters.tsx @@ -16,7 +16,8 @@ import { import { Input } from "@solid-imager/ui/input"; import { Label } from "@solid-imager/ui/label"; import { cn } from "@solid-imager/ui/utils/cn"; -import { createSignal, For } from "solid-js"; +import { createDebouncedSignal } from "@solid-imager/ui/utils/debounce"; +import { createMemo, createSignal, For } from "solid-js"; export type SearchFilterState = { searchQuery: string; @@ -62,6 +63,16 @@ function FilterSection(props: { badgeVariant?: "default" | "destructive" | "secondary" | "outline"; }) { const [value, setValue] = createSignal(null); + const [filterText, setFilterText] = createDebouncedSignal("", 150); + const filteredItems = createMemo(() => { + const items = props.items; + if (!items) return []; + const query = filterText().toLowerCase(); + if (!query) return items.slice(0, 100); + return items + .filter((item) => props.getItemLabel(item).toLowerCase().includes(query)) + .slice(0, 100); + }); return (
@@ -103,12 +114,13 @@ function FilterSection(props: { setValue(() => val); requestAnimationFrame(() => { setValue(null); + setFilterText(""); }); } }} - defaultFilter="contains" + onInputChange={(text) => setFilterText(text)} optionLabel={props.getItemLabel} - options={props.items ?? []} + options={filteredItems()} optionTextValue={props.getItemLabel} optionValue={(item) => props.getItemKey(item)} placeholder={props.placeholder} diff --git a/packages/ui/src/pro-search-builder.tsx b/packages/ui/src/pro-search-builder.tsx index 50edad0c0..1e6e93e0c 100644 --- a/packages/ui/src/pro-search-builder.tsx +++ b/packages/ui/src/pro-search-builder.tsx @@ -27,6 +27,7 @@ import { SelectValue, } from "./select"; import { cn } from "./utils/cn"; +import { createDebouncedSignal } from "./utils/debounce"; import { parseSelectValue } from "./utils/parse-select-value"; const TARGET_LABELS: Record = { @@ -326,6 +327,20 @@ function CriterionBuilder(props: { return undefined; } }); + const [filterText, setFilterText] = createDebouncedSignal("", 150); + const filteredItems = createMemo(() => { + const items = autocompleteItems(); + if (!items) return []; + const query = filterText().toLowerCase(); + if (!query) return items.slice(0, 100); + return items + .filter((item) => { + if (!item) return false; + const label = "accountId" in item ? getAuthorLabel(item) : item.name; + return label.toLowerCase().includes(query); + }) + .slice(0, 100); + }); const isNumericTarget = createMemo(() => [ @@ -460,7 +475,7 @@ function CriterionBuilder(props: { props.onChange({ ...props.criterion, value: value.name }); } }} - defaultFilter="contains" + onInputChange={(text) => setFilterText(text)} optionLabel={(item) => item ? "accountId" in item @@ -468,7 +483,7 @@ function CriterionBuilder(props: { : item.name : "" } - options={autocompleteItems() ?? []} + options={filteredItems()} optionTextValue={(item) => item ? "accountId" in item diff --git a/packages/ui/src/search-filters.tsx b/packages/ui/src/search-filters.tsx index 2e22d4226..a384bacaf 100644 --- a/packages/ui/src/search-filters.tsx +++ b/packages/ui/src/search-filters.tsx @@ -4,7 +4,7 @@ import type { Ip } from "@solid-imager/core/domain/ips/schemas"; import type { Project } from "@solid-imager/core/domain/projects/schemas"; import type { SearchState } from "@solid-imager/core/domain/search/schema"; import type { TagResponse } from "@solid-imager/core/domain/tags/schemas"; -import { createSignal, For } from "solid-js"; +import { createMemo, createSignal, For } from "solid-js"; import type { SetStoreFunction } from "solid-js/store"; import { Badge } from "./badge"; import { Button } from "./button"; @@ -19,6 +19,7 @@ import { import { Input } from "./input"; import { Label } from "./label"; import { cn } from "./utils/cn"; +import { createDebouncedSignal } from "./utils/debounce"; type SearchFiltersProps = { state: SearchState; @@ -47,6 +48,16 @@ function FilterSection(props: { badgeVariant?: "default" | "destructive" | "secondary" | "outline"; }) { const [value, setValue] = createSignal(null); + const [filterText, setFilterText] = createDebouncedSignal("", 150); + const filteredItems = createMemo(() => { + const items = props.items; + if (!items) return []; + const query = filterText().toLowerCase(); + if (!query) return items.slice(0, 100); + return items + .filter((item) => props.getItemLabel(item).toLowerCase().includes(query)) + .slice(0, 100); + }); return (
@@ -88,12 +99,13 @@ function FilterSection(props: { setValue(() => val); requestAnimationFrame(() => { setValue(null); + setFilterText(""); }); } }} - defaultFilter="contains" + onInputChange={(text) => setFilterText(text)} optionLabel={props.getItemLabel} - options={props.items ?? []} + options={filteredItems()} optionTextValue={props.getItemLabel} optionValue={(item) => props.getItemKey(item)} placeholder={props.placeholder} From 3ef878210fc1f78d727ac409207af29f21d8618b Mon Sep 17 00:00:00 2001 From: hmjn Date: Thu, 2 Jul 2026 00:15:06 +0900 Subject: [PATCH 11/19] fix(search): update pro search target selection --- .../components/media/pro-search-builder.tsx | 20 +++++++------- packages/ui/src/pro-search-builder.tsx | 27 +++++++++---------- 2 files changed, 23 insertions(+), 24 deletions(-) diff --git a/apps/server/src/components/media/pro-search-builder.tsx b/apps/server/src/components/media/pro-search-builder.tsx index 8ea5a0c13..1c189e630 100644 --- a/apps/server/src/components/media/pro-search-builder.tsx +++ b/apps/server/src/components/media/pro-search-builder.tsx @@ -30,7 +30,7 @@ import { parseSelectValue, } from "@solid-imager/ui/utils"; import { cn } from "@solid-imager/ui/utils/cn"; -import { createMemo, Index, Match, Show, Switch } from "solid-js"; +import { createMemo, For, Match, Show, Switch } from "solid-js"; // Labels for targets const TARGET_LABELS: Record = { @@ -263,37 +263,37 @@ function GroupBuilder(props: {
- + {(child, index) => ( updateChild(index, c)} - onRemove={() => removeChild(index)} + onChange={(c) => updateChild(index(), c)} + onRemove={() => removeChild(index())} projects={props.projects} tags={props.tags} /> } - when={isSearchGroup(child())} + when={isSearchGroup(child)} > updateChild(index, g)} - onRemove={() => removeChild(index)} + onChange={(g) => updateChild(index(), g)} + onRemove={() => removeChild(index())} projects={props.projects} tags={props.tags} /> )} - + {props.group.children.length === 0 && (
条件がありません。「+ 条件」ボタンで追加してください。 diff --git a/packages/ui/src/pro-search-builder.tsx b/packages/ui/src/pro-search-builder.tsx index 1e6e93e0c..5fc7bd924 100644 --- a/packages/ui/src/pro-search-builder.tsx +++ b/packages/ui/src/pro-search-builder.tsx @@ -7,7 +7,7 @@ import type { } from "@solid-imager/core/domain/media/schemas"; import type { Project } from "@solid-imager/core/domain/projects/schemas"; import type { TagResponse } from "@solid-imager/core/domain/tags/schemas"; -import { createMemo, Index, Match, Show, Switch } from "solid-js"; +import { createMemo, For, Match, Show, Switch } from "solid-js"; import { Button } from "./button"; import { Card, CardContent } from "./card"; import { @@ -258,18 +258,17 @@ function GroupBuilder(props: {
- - {(child, index) => { - const c = child(); - return c.type === "group" ? ( + + {(child, index) => + child.type === "group" ? ( updateChild(index, value)} - onRemove={() => removeChild(index)} + onChange={(value) => updateChild(index(), value)} + onRemove={() => removeChild(index())} projects={props.projects} tags={props.tags} /> @@ -277,16 +276,16 @@ function GroupBuilder(props: { updateChild(index, value)} - onRemove={() => removeChild(index)} + onChange={(value) => updateChild(index(), value)} + onRemove={() => removeChild(index())} projects={props.projects} tags={props.tags} /> - ); - }} - + ) + } +
条件がありません。「+ 条件」ボタンで追加してください。 From aafc586dd5773b63d8cfc70aff3e89b99bfcf87c Mon Sep 17 00:00:00 2001 From: hmjn Date: Thu, 2 Jul 2026 00:17:21 +0900 Subject: [PATCH 12/19] fix(lancedb): use SQL transform for schema migration --- .../application/services/lancedb-dump-service.test.ts | 7 ++++++- .../application/src/services/lancedb-dump-service.ts | 10 ++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/apps/server/src/tests/unit/application/services/lancedb-dump-service.test.ts b/apps/server/src/tests/unit/application/services/lancedb-dump-service.test.ts index b5afde635..6d3b2a363 100644 --- a/apps/server/src/tests/unit/application/services/lancedb-dump-service.test.ts +++ b/apps/server/src/tests/unit/application/services/lancedb-dump-service.test.ts @@ -248,7 +248,12 @@ describe("LanceDB Dump Service", () => { }); const authorTable = await mockOpenTable.mock.results[2].value; - expect(authorTable.addColumns).toHaveBeenCalledTimes(1); + expect(authorTable.addColumns).toHaveBeenCalledWith([ + { + name: "platform", + valueSql: "CAST(NULL AS STRING)", + }, + ]); }); it("updates the manifest version after delta schema migration", async () => { diff --git a/packages/application/src/services/lancedb-dump-service.ts b/packages/application/src/services/lancedb-dump-service.ts index 1e778262a..c840221f6 100644 --- a/packages/application/src/services/lancedb-dump-service.ts +++ b/packages/application/src/services/lancedb-dump-service.ts @@ -405,10 +405,12 @@ async function openTables(db: Connection): Promise { async function migrateLanceDbSchema(tables: LanceTables): Promise { const authorSchema = await tables.media_authors.schema(); if (!authorSchema.fields.some((field) => field.name === "platform")) { - const arrow = await import("apache-arrow"); - await tables.media_authors.addColumns( - new arrow.Field("platform", new arrow.Utf8(), true), - ); + await tables.media_authors.addColumns([ + { + name: "platform", + valueSql: "CAST(NULL AS STRING)", + }, + ]); } } From cc3ef567a1e0b481f41372ee16eb9b3731dc9f33 Mon Sep 17 00:00:00 2001 From: hmjn Date: Thu, 2 Jul 2026 22:52:55 +0900 Subject: [PATCH 13/19] fix(lancedb): share dump manifest version --- .../src/application/services/backup-service.ts | 3 ++- .../application/services/lancedb-dump-service.ts | 6 +++++- .../src/application/services/maintenance-service.ts | 5 ++++- .../services/maintenance-service.test.ts | 13 +++++++------ .../src/services/lancedb-dump-service.ts | 2 +- 5 files changed, 19 insertions(+), 10 deletions(-) diff --git a/apps/server/src/application/services/backup-service.ts b/apps/server/src/application/services/backup-service.ts index 792db8d2b..ddb8567bf 100644 --- a/apps/server/src/application/services/backup-service.ts +++ b/apps/server/src/application/services/backup-service.ts @@ -10,6 +10,7 @@ import { getErrorMessage } from "@solid-imager/core/utils/get-error-message"; import type { Table } from "drizzle-orm"; import { and, asc, eq, gt, inArray, lt, or, sql } from "drizzle-orm"; import type { PgColumn } from "drizzle-orm/pg-core"; +import { LANCEDB_DUMP_VERSION } from "~/application/services/lancedb-dump-service"; import { db } from "~/infrastructure/db"; import { authorAccounts, @@ -1194,7 +1195,7 @@ export const BackupService = { try { const content = await fs.readFile(manifestPath, "utf-8"); const manifest = JSON.parse(content) as { version?: unknown }; - if (manifest.version !== 3) { + if (manifest.version !== LANCEDB_DUMP_VERSION) { if (!config.lancedb?.autoFullSync) { logger.warn( { mediaSourceId }, diff --git a/apps/server/src/application/services/lancedb-dump-service.ts b/apps/server/src/application/services/lancedb-dump-service.ts index 67e2d0ab4..16d5ebe4e 100644 --- a/apps/server/src/application/services/lancedb-dump-service.ts +++ b/apps/server/src/application/services/lancedb-dump-service.ts @@ -1,6 +1,10 @@ -import { createLanceDbDumpService } from "@solid-imager/application/services/lancedb-dump-service"; +import { + createLanceDbDumpService, + LANCEDB_DUMP_VERSION, +} from "@solid-imager/application/services/lancedb-dump-service"; export type { MediaDumpItemWithImageData } from "@solid-imager/application/ports/lancedb-dump-service"; +export { LANCEDB_DUMP_VERSION }; import { logger } from "~/infrastructure/logger"; diff --git a/apps/server/src/application/services/maintenance-service.ts b/apps/server/src/application/services/maintenance-service.ts index 53f6b43b9..88e0a4f64 100644 --- a/apps/server/src/application/services/maintenance-service.ts +++ b/apps/server/src/application/services/maintenance-service.ts @@ -202,7 +202,10 @@ export class MaintenanceService { try { const content = await fs.readFile(manifestPath, "utf-8"); const manifest = JSON.parse(content) as { version?: unknown }; - return manifest.version === 3; + const { LANCEDB_DUMP_VERSION } = await import( + "~/application/services/lancedb-dump-service" + ); + return manifest.version === LANCEDB_DUMP_VERSION; } catch { return false; } diff --git a/apps/server/src/tests/unit/application/services/maintenance-service.test.ts b/apps/server/src/tests/unit/application/services/maintenance-service.test.ts index c0e02591f..a28781b05 100644 --- a/apps/server/src/tests/unit/application/services/maintenance-service.test.ts +++ b/apps/server/src/tests/unit/application/services/maintenance-service.test.ts @@ -41,6 +41,7 @@ vi.mock("~/application/registry", () => ({ // Mock lancedb-dump-service const mockReadMediaIds = vi.fn(); vi.mock("~/application/services/lancedb-dump-service", () => ({ + LANCEDB_DUMP_VERSION: 4, readMediaIds: mockReadMediaIds, })); @@ -387,9 +388,9 @@ describe("MaintenanceService", () => { ]); mockJobRepo.createIfUnique.mockResolvedValue({ id: "job-new" }); - // Simulate manifest.json exists (version: 3) + // Simulate a current manifest.json. (fs.readFile as unknown as Mock).mockResolvedValue( - JSON.stringify({ version: 3 }), + JSON.stringify({ version: 4 }), ); // Both LanceDB and Postgres have media-1 @@ -417,9 +418,9 @@ describe("MaintenanceService", () => { ]); mockJobRepo.createIfUnique.mockResolvedValue({ id: "job-new" }); - // Simulate manifest.json exists (version: 3) + // Simulate a current manifest.json. (fs.readFile as unknown as Mock).mockResolvedValue( - JSON.stringify({ version: 3 }), + JSON.stringify({ version: 4 }), ); // Postgres: media-1, media-2 @@ -465,9 +466,9 @@ describe("MaintenanceService", () => { ]); mockJobRepo.createIfUnique.mockResolvedValue({ id: "job-new" }); - // Simulate manifest.json exists (version: 3) + // Simulate a current manifest.json. (fs.readFile as unknown as Mock).mockResolvedValue( - JSON.stringify({ version: 3 }), + JSON.stringify({ version: 4 }), ); // readMediaIds throws error diff --git a/packages/application/src/services/lancedb-dump-service.ts b/packages/application/src/services/lancedb-dump-service.ts index c840221f6..16c57cc35 100644 --- a/packages/application/src/services/lancedb-dump-service.ts +++ b/packages/application/src/services/lancedb-dump-service.ts @@ -14,7 +14,7 @@ import type { const ROW_CHUNK_SIZE = 5000; const READ_CHUNK_SIZE = 5000; -const LANCEDB_DUMP_VERSION = 4; +export const LANCEDB_DUMP_VERSION = 4; async function updateLanceDbManifestVersion(lanceDbDir: string): Promise { const manifestPath = path.join(lanceDbDir, "manifest.json"); From b8d4a8c619fea2ffa33607ae392b62f11a7fec1d Mon Sep 17 00:00:00 2001 From: hmjn023 <76728956+hmjn023@users.noreply.github.com> Date: Thu, 2 Jul 2026 22:57:13 +0900 Subject: [PATCH 14/19] Update apps/server/src/tests/unit/application/services/maintenance-service.test.ts Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- .../application/services/maintenance-service.test.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/apps/server/src/tests/unit/application/services/maintenance-service.test.ts b/apps/server/src/tests/unit/application/services/maintenance-service.test.ts index a28781b05..56695aa7f 100644 --- a/apps/server/src/tests/unit/application/services/maintenance-service.test.ts +++ b/apps/server/src/tests/unit/application/services/maintenance-service.test.ts @@ -40,10 +40,13 @@ vi.mock("~/application/registry", () => ({ // Mock lancedb-dump-service const mockReadMediaIds = vi.fn(); -vi.mock("~/application/services/lancedb-dump-service", () => ({ - LANCEDB_DUMP_VERSION: 4, - readMediaIds: mockReadMediaIds, -})); +vi.mock("~/application/services/lancedb-dump-service", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + readMediaIds: mockReadMediaIds, + }; +}); // Mock backup-service const mockQueueSourceLanceDBDelta = vi.fn(); From f32eabf9d5520e142ce2af962cf67d7f5ecd2b11 Mon Sep 17 00:00:00 2001 From: hmjn Date: Thu, 2 Jul 2026 23:10:52 +0900 Subject: [PATCH 15/19] fix(server): suppress expected thumbnail 404 responses --- .../api/sources.$mediaSourceId.thumbnail.$mediaId.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/apps/server/src/routes/api/sources.$mediaSourceId.thumbnail.$mediaId.ts b/apps/server/src/routes/api/sources.$mediaSourceId.thumbnail.$mediaId.ts index 853aa9b2b..394a00b8a 100644 --- a/apps/server/src/routes/api/sources.$mediaSourceId.thumbnail.$mediaId.ts +++ b/apps/server/src/routes/api/sources.$mediaSourceId.thumbnail.$mediaId.ts @@ -14,7 +14,14 @@ export const Route = createFileRoute( const file = Bun.file(thumbnailPath); if (!(await file.exists())) { - return new Response("Thumbnail not found", { status: 404 }); + // A missing thumbnail is an expected transient state while the + // background job is running. The image component treats an empty + // response as a load error and retries, without flooding the + // browser console with expected 404 responses. + return new Response(null, { + status: 204, + headers: { "Cache-Control": "no-store" }, + }); } return new Response(file, { From 94da58d33c718235dfee5a7923964e1372490251 Mon Sep 17 00:00:00 2001 From: hmjn Date: Fri, 3 Jul 2026 00:00:38 +0900 Subject: [PATCH 16/19] fix(review): address valid PR feedback --- .../application/services/backup-service.ts | 108 +++--------------- .../infrastructure/api/routers/ai-router.ts | 43 +++---- .../db/src/repositories/author-repository.ts | 52 ++++++++- packages/ui/src/stores/search-store.test.ts | 79 +++++++++++++ packages/ui/src/stores/search-store.ts | 20 ++++ 5 files changed, 183 insertions(+), 119 deletions(-) create mode 100644 packages/ui/src/stores/search-store.test.ts diff --git a/apps/server/src/application/services/backup-service.ts b/apps/server/src/application/services/backup-service.ts index ddb8567bf..3e031da52 100644 --- a/apps/server/src/application/services/backup-service.ts +++ b/apps/server/src/application/services/backup-service.ts @@ -8,13 +8,12 @@ import { import { localConnectionSchema } from "@solid-imager/core/domain/sources/schemas"; import { getErrorMessage } from "@solid-imager/core/utils/get-error-message"; import type { Table } from "drizzle-orm"; -import { and, asc, eq, gt, inArray, lt, or, sql } from "drizzle-orm"; +import { and, asc, eq, gt, inArray, lt, sql } from "drizzle-orm"; import type { PgColumn } from "drizzle-orm/pg-core"; import { LANCEDB_DUMP_VERSION } from "~/application/services/lancedb-dump-service"; import { db } from "~/infrastructure/db"; import { authorAccounts, - authors, characterIps, characters, ips, @@ -413,92 +412,16 @@ export const BackupService = { _tx, ); const authorInputs = [...authorData.values()]; - const d = _tx ?? db; const authorMap = new Map(); - for (const input of authorInputs) { - let authorId: string | undefined; - if (input.platform && input.accountId) { - const existingAccount = await d - .select({ authorId: authorAccounts.authorId }) - .from(authorAccounts) - .where( - and( - eq(authorAccounts.platform, input.platform), - eq(authorAccounts.accountId, input.accountId), - ), - ) - .limit(1); - authorId = existingAccount[0]?.authorId; - - if (!authorId) { - const legacyCandidates = await d - .select() - .from(authors) - .where( - or( - eq(authors.accountId, input.accountId), - eq(authors.name, input.name), - ), - ); - const candidateIds = legacyCandidates.map((author) => author.id); - const linkedIds = - candidateIds.length > 0 - ? new Set( - ( - await d - .select({ authorId: authorAccounts.authorId }) - .from(authorAccounts) - .where(inArray(authorAccounts.authorId, candidateIds)) - ).map((account) => account.authorId), - ) - : new Set(); - const reusable = legacyCandidates.find( - (author) => - !linkedIds.has(author.id) && - (author.accountId === input.accountId || - (author.name === input.name && !author.accountId)), - ); - if (reusable) { - authorId = reusable.id; - } else { - const inserted = await d - .insert(authors) - .values({ - name: input.name, - accountId: input.accountId, - }) - .returning(); - authorId = inserted[0]?.id; - } - if (authorId) { - await d - .insert(authorAccounts) - .values({ - authorId, - platform: input.platform, - accountId: input.accountId, - }) - .onConflictDoNothing(); - } - } - } else { - const existing = await d - .select({ id: authors.id }) - .from(authors) - .where(eq(authors.name, input.name)) - .limit(1); - authorId = existing[0]?.id; - if (!authorId) { - const inserted = await d - .insert(authors) - .values({ - name: input.name, - accountId: input.accountId ?? null, - }) - .returning(); - authorId = inserted[0]?.id; - } - } + const validAuthorInputs = authorInputs.filter( + (input) => input.name.trim().length > 0, + ); + const { services } = await import("~/application/registry"); + const restoredAuthors = await services + .getAuthorRepository() + .findOrCreateBulk(validAuthorInputs, _tx); + for (const [index, input] of validAuthorInputs.entries()) { + const authorId = restoredAuthors[index]?.id; if (!authorId) continue; authorMap.set(authorRestoreKey(input), authorId); if (!authorMap.has(`name:${input.name}`)) { @@ -1592,6 +1515,12 @@ export const BackupService = { if (cacheValid) { try { const driver = getDriver(mediaSource); + const { readMediaFilePaths } = await import( + "~/application/services/lancedb-dump-service" + ); + const mediaFilePaths = includeImages + ? await readMediaFilePaths(cacheDir) + : []; const passThrough = new PassThrough(); const archive = new archiverMod.TarArchive(); archive.pipe(passThrough); @@ -1601,11 +1530,6 @@ export const BackupService = { archive.directory(cacheDir, false); if (includeImages) { - const { readMediaFilePaths } = await import( - "~/application/services/lancedb-dump-service" - ); - const mediaFilePaths = await readMediaFilePaths(cacheDir); - await Promise.all( mediaFilePaths.map(async (item) => { if (!item.filePath) return; diff --git a/apps/server/src/infrastructure/api/routers/ai-router.ts b/apps/server/src/infrastructure/api/routers/ai-router.ts index bda537112..3682d4413 100644 --- a/apps/server/src/infrastructure/api/routers/ai-router.ts +++ b/apps/server/src/infrastructure/api/routers/ai-router.ts @@ -10,7 +10,6 @@ import { type Media, mediaSchema, } from "@solid-imager/core/domain/media/schemas"; -import type { NapiBBox } from "@solid-imager/core/domain/tagging/schemas"; import { batchCcipExtractionRequestSchema, batchTaggingRequestSchema, @@ -20,8 +19,11 @@ import { ccipExtractionRequestSchema, ccipFeatureRequestSchema, ccipVectorStatusSchema, + detectAndCropResponseSchema, + type NapiBBox, startBatchTaggingResponseSchema, startCcipExtractionResponseSchema, + taggingResponseSchema, tagImageRequestSchema, } from "@solid-imager/core/domain/tagging/schemas"; import { @@ -192,7 +194,6 @@ export const aiRouter = { "file" in input ? { inputType: "file", - fileName: input.file.name, fileSize: input.file.size, } : { @@ -243,12 +244,14 @@ export const aiRouter = { } const fullPath = path.join(connectionInfo.path, media.filePath); const fileBuffer = await readFileBuffer(fullPath); - const result = (await callRemoteTagging( - remoteUrl, - fileBuffer, - path.basename(fullPath), - config.ai.timeoutMs, - )) as import("@solid-imager/core/domain/tagging/schemas").TaggingResponse; + const result = taggingResponseSchema.parse( + await callRemoteTagging( + remoteUrl, + fileBuffer, + path.basename(fullPath), + config.ai.timeoutMs, + ), + ); logger.info( { ...logContext, @@ -689,22 +692,14 @@ export const aiRouter = { const config = services.getConfigService().getConfig(); if (remoteUrl && !isRemoteServerLocal(remoteUrl)) { const fileBuffer = await readFileBuffer(fullPath); - const result = (await callRemoteCrop( - remoteUrl, - fileBuffer, - path.basename(fullPath), - config.ai.timeoutMs, - )) as { - detections: Array<{ - index: number; - bbox: NapiBBox; - label: string; - score: number; - imageBase64: string; - width: number; - height: number; - }>; - }; + const result = detectAndCropResponseSchema.parse( + await callRemoteCrop( + remoteUrl, + fileBuffer, + path.basename(fullPath), + config.ai.timeoutMs, + ), + ); logger.info( { ...logContext, diff --git a/packages/db/src/repositories/author-repository.ts b/packages/db/src/repositories/author-repository.ts index a46d7ebcb..6a285516f 100644 --- a/packages/db/src/repositories/author-repository.ts +++ b/packages/db/src/repositories/author-repository.ts @@ -35,7 +35,7 @@ async function findOrCreateAuthorsBulk( ): Promise { const uniqueInputs = new Map(); for (const input of inputs) { - if (input.name.length > 0) { + if (input.name.trim().length > 0) { uniqueInputs.set(authorInputKey(input), input); } } @@ -116,6 +116,7 @@ async function findOrCreateAuthorsBulk( const accountsToInsert: (typeof authorAccounts.$inferInsert)[] = []; const authorsToInsert: (typeof authors.$inferInsert)[] = []; + const newlyCreatedAuthorByKey = new Map(); for (const [key, input] of unresolved) { const legacyByMatchingAccount = input.accountId ? legacyByAccount.get(input.accountId) @@ -143,6 +144,7 @@ async function findOrCreateAuthorsBulk( name: author.name, accountId: author.accountId, }); + newlyCreatedAuthorByKey.set(key, author.id); } resolved.set(key, author); @@ -159,10 +161,54 @@ async function findOrCreateAuthorsBulk( await client.insert(authors).values(authorsToInsert); } if (accountsToInsert.length > 0) { - await client + const insertedAccounts = await client .insert(authorAccounts) .values(accountsToInsert) - .onConflictDoNothing(); + .onConflictDoNothing() + .returning(); + const insertedKeys = new Set( + insertedAccounts.map( + (account) => `${account.platform}:${account.accountId}`, + ), + ); + const attemptedKeys = new Set( + accountsToInsert.map( + (account) => `${account.platform}:${account.accountId}`, + ), + ); + const conflicted = [...uniqueInputs.entries()].filter( + ([key, input]) => + input.platform && + input.accountId && + attemptedKeys.has(key) && + !insertedKeys.has(key), + ); + if (conflicted.length > 0) { + const conditions = conflicted.flatMap(([, input]) => { + if (!(input.platform && input.accountId)) return []; + const condition = and( + eq(authorAccounts.platform, input.platform), + eq(authorAccounts.accountId, input.accountId), + ); + return condition ? [condition] : []; + }); + const canonicalAccounts = await client + .select({ account: authorAccounts, author: authors }) + .from(authorAccounts) + .innerJoin(authors, eq(authorAccounts.authorId, authors.id)) + .where(or(...conditions)); + for (const row of canonicalAccounts) { + const key = `${row.account.platform}:${row.account.accountId}`; + resolved.set(key, row.author); + } + const orphanIds = conflicted.flatMap(([key]) => { + const id = newlyCreatedAuthorByKey.get(key); + return id ? [id] : []; + }); + if (orphanIds.length > 0) { + await client.delete(authors).where(inArray(authors.id, orphanIds)); + } + } } return [...uniqueInputs.keys()].flatMap((key) => { diff --git a/packages/ui/src/stores/search-store.test.ts b/packages/ui/src/stores/search-store.test.ts new file mode 100644 index 000000000..a490fe3a9 --- /dev/null +++ b/packages/ui/src/stores/search-store.test.ts @@ -0,0 +1,79 @@ +import { beforeEach, describe, expect, it } from "vite-plus/test"; +import { + clearVectorSearchAnchor, + searchState, + setSearchState, +} from "./search-store"; + +class MemoryStorage implements Storage { + private readonly values = new Map(); + + get length(): number { + return this.values.size; + } + + clear(): void { + this.values.clear(); + } + + getItem(key: string): string | null { + return this.values.get(key) ?? null; + } + + key(index: number): string | null { + return [...this.values.keys()][index] ?? null; + } + + removeItem(key: string): void { + this.values.delete(key); + } + + setItem(key: string, value: string): void { + this.values.set(key, value); + } +} + +describe("clearVectorSearchAnchor", () => { + beforeEach(() => { + Object.defineProperty(globalThis, "sessionStorage", { + configurable: true, + value: new MemoryStorage(), + }); + setSearchState({ + mode: "vector", + similarityAnchorMediaId: "media-1", + offset: 10, + scrollY: 100, + }); + }); + + it("clears both the store and persisted vector anchor", () => { + sessionStorage.setItem( + "current-all", + JSON.stringify({ + mode: "vector", + similarityAnchorMediaId: "media-1", + similarityTopK: 50, + }), + ); + + clearVectorSearchAnchor(); + + expect(searchState.similarityAnchorMediaId).toBeNull(); + expect(searchState.offset).toBe(0); + expect(searchState.scrollY).toBe(0); + expect(JSON.parse(sessionStorage.getItem("current-all") ?? "{}")).toEqual({ + mode: "vector", + similarityAnchorMediaId: null, + similarityTopK: 50, + }); + }); + + it("removes malformed persisted state", () => { + sessionStorage.setItem("current-all", "not-json"); + + clearVectorSearchAnchor(); + + expect(sessionStorage.getItem("current-all")).toBeNull(); + }); +}); diff --git a/packages/ui/src/stores/search-store.ts b/packages/ui/src/stores/search-store.ts index 3f8e351ab..c84120f6c 100644 --- a/packages/ui/src/stores/search-store.ts +++ b/packages/ui/src/stores/search-store.ts @@ -67,6 +67,26 @@ export const clearVectorSearchAnchor = () => { offset: 0, scrollY: 0, }); + if (typeof sessionStorage !== "undefined") { + const stored = sessionStorage.getItem("current-all"); + if (!stored) return; + try { + const current: unknown = JSON.parse(stored); + if (typeof current !== "object" || current === null) { + sessionStorage.removeItem("current-all"); + return; + } + sessionStorage.setItem( + "current-all", + JSON.stringify({ + ...current, + similarityAnchorMediaId: null, + }), + ); + } catch { + sessionStorage.removeItem("current-all"); + } + } }; export const getSearchCondition = () => From 1d23e5fbc7e1319a247b1c75d37d636685e701e5 Mon Sep 17 00:00:00 2001 From: hmjn023 Date: Sat, 4 Jul 2026 00:58:17 +0900 Subject: [PATCH 17/19] fix(authors): refresh display names by account identity --- .../repository/author-dedupe.test.ts | 9 ++- apps/xtracter/src/content/twitter.test.ts | 36 ++++++++++ apps/xtracter/src/content/twitter.ts | 54 +++++++++------ .../db/src/repositories/author-repository.ts | 69 +++++++++++++++++-- 4 files changed, 139 insertions(+), 29 deletions(-) create mode 100644 apps/xtracter/src/content/twitter.test.ts diff --git a/apps/server/src/tests/integration/repository/author-dedupe.test.ts b/apps/server/src/tests/integration/repository/author-dedupe.test.ts index dba14ae99..9295e6171 100644 --- a/apps/server/src/tests/integration/repository/author-dedupe.test.ts +++ b/apps/server/src/tests/integration/repository/author-dedupe.test.ts @@ -52,10 +52,10 @@ describe("AuthorRepository Deduplication", () => { expect(count).toHaveLength(1); }); - it("deduplicates an author by platform and account ID", async () => { + it("deduplicates by platform account and refreshes the display name", async () => { const author1 = await AuthorRepository.create({ name: "Original Name", - accountId: "@creator", + accountId: "@Creator", platform: "twitter", }); const author2 = await AuthorRepository.create({ @@ -65,6 +65,7 @@ describe("AuthorRepository Deduplication", () => { }); expect(author2.id).toBe(author1.id); + expect(author2.name).toBe("Updated Display Name"); const accounts = await db .select() .from(authorAccounts) @@ -76,6 +77,10 @@ describe("AuthorRepository Deduplication", () => { ); expect(accounts).toHaveLength(1); expect(accounts[0]?.authorId).toBe(author1.id); + const storedAuthor = await db.query.authors.findFirst({ + where: eq(authors.id, author1.id), + }); + expect(storedAuthor?.name).toBe("Updated Display Name"); }); it("keeps identical account IDs on different platforms separate", async () => { diff --git a/apps/xtracter/src/content/twitter.test.ts b/apps/xtracter/src/content/twitter.test.ts new file mode 100644 index 000000000..61bafcf49 --- /dev/null +++ b/apps/xtracter/src/content/twitter.test.ts @@ -0,0 +1,36 @@ +import { describe, expect, it } from "vite-plus/test"; +import { extractTwitterAuthorIdFromStatusUrl } from "./twitter"; + +describe("extractTwitterAuthorIdFromStatusUrl", () => { + it("extracts the handle from an X status permalink", () => { + expect( + extractTwitterAuthorIdFromStatusUrl( + "https://x.com/Creator_123/status/1234567890", + ), + ).toBe("@Creator_123"); + }); + + it("extracts the handle from a legacy Twitter status permalink", () => { + expect( + extractTwitterAuthorIdFromStatusUrl( + "https://twitter.com/creator/status/1234567890?s=20", + ), + ).toBe("@creator"); + }); + + it("rejects status URLs that do not contain an account handle", () => { + expect( + extractTwitterAuthorIdFromStatusUrl( + "https://x.com/i/web/status/1234567890", + ), + ).toBe(""); + }); + + it("does not interpret display-name mentions as account IDs", () => { + expect( + extractTwitterAuthorIdFromStatusUrl( + "https://x.com/not-a-status/display-name-@C107", + ), + ).toBe(""); + }); +}); diff --git a/apps/xtracter/src/content/twitter.ts b/apps/xtracter/src/content/twitter.ts index c4778afdd..47544172f 100644 --- a/apps/xtracter/src/content/twitter.ts +++ b/apps/xtracter/src/content/twitter.ts @@ -3,7 +3,32 @@ import { querySelectorAllTyped, querySelectorTyped } from "../utils/dom-utils"; const PROCESSED_IMAGE_CLASS = "xtracter-image-processed"; const PROCESSED_VIDEO_CLASS = "xtracter-video-processed"; -const AUTHOR_ID_REGEX = /@\w+/; +const TWITTER_HANDLE_REGEX = /^[A-Za-z0-9_]{1,15}$/; + +export function extractTwitterAuthorIdFromStatusUrl(urlValue: string): string { + try { + const url = new URL(urlValue); + const isTwitterHost = + url.hostname === "x.com" || + url.hostname === "www.x.com" || + url.hostname === "twitter.com" || + url.hostname === "www.twitter.com"; + if (!isTwitterHost) { + return ""; + } + const pathParts = url.pathname.split("/").filter(Boolean); + if ( + pathParts.length < 3 || + pathParts[1] !== "status" || + !TWITTER_HANDLE_REGEX.test(pathParts[0] ?? "") + ) { + return ""; + } + return `@${pathParts[0]}`; + } catch { + return ""; + } +} export function processTwitterMedia( processedMetadata: Map, @@ -125,7 +150,6 @@ function extractMetadataFromUrl(): { authorId: string; tweetUrl: string } { const url = new URL(window.location.href); const pathParts = url.pathname.split("/").filter((p) => p); - let authorId = ""; let tweetUrl = window.location.href; const MIN_PATH_PARTS_FOR_STATUS = 3; @@ -136,11 +160,13 @@ function extractMetadataFromUrl(): { authorId: string; tweetUrl: string } { pathParts.length >= MIN_PATH_PARTS_FOR_STATUS && pathParts[STATUS_PART_INDEX] === "status" ) { - authorId = `@${pathParts[0]}`; tweetUrl = `${url.origin}/${pathParts[0]}/status/${pathParts[TWEET_ID_PART_INDEX]}`; } - return { authorId, tweetUrl }; + return { + authorId: extractTwitterAuthorIdFromStatusUrl(tweetUrl), + tweetUrl, + }; } function extractMetadata( @@ -211,7 +237,7 @@ function determineTargetUrl( const url = new URL(element.src); url.searchParams.set("name", "orig"); return url.toString(); - } catch (_e) { + } catch { if (element instanceof HTMLImageElement) { return element.src; } @@ -238,23 +264,7 @@ function extractFromArticle(article: HTMLElement) { const userNameNode = article.querySelector('div[data-testid="User-Name"]'); const authorName = userNameNode?.querySelector("span")?.innerText || ""; - let authorId = ""; - const userAnchor = userNameNode?.querySelector("a"); - if (userAnchor) { - try { - const url = new URL(userAnchor.href); - const pathParts = url.pathname.split("/").filter((p) => p); - if (pathParts.length > 0) { - authorId = `@${pathParts[0]}`; - } - } catch (_e) { - // Ignored - } - } - - if (!authorId) { - authorId = userNameNode?.textContent?.match(AUTHOR_ID_REGEX)?.[0] || ""; - } + const authorId = extractTwitterAuthorIdFromStatusUrl(tweetUrl); return { tweetText, timestamp, tweetUrl, authorName, authorId }; } diff --git a/packages/db/src/repositories/author-repository.ts b/packages/db/src/repositories/author-repository.ts index a46d7ebcb..b9477a091 100644 --- a/packages/db/src/repositories/author-repository.ts +++ b/packages/db/src/repositories/author-repository.ts @@ -5,7 +5,7 @@ import type { NewAuthor, } from "@solid-imager/core/domain/media/schemas"; import type { IAuthorRepository } from "@solid-imager/core/domain/repositories/author-repository"; -import { and, eq, inArray, or, type SQL } from "drizzle-orm"; +import { and, eq, inArray, or, sql, type SQL } from "drizzle-orm"; import { authorAccounts, authors, mediaAuthors } from "../schema"; import type { DrizzleExecutor } from "../types"; @@ -23,9 +23,37 @@ function mapAuthor(row: typeof authors.$inferSelect): Author { }; } +function normalizeAccountId( + platform: NonNullable, + accountId: string, +): string { + if (platform !== "twitter") return accountId; + return accountId.toLowerCase(); +} + +function normalizeAuthorInput(author: NewAuthor): NewAuthor { + if (!(author.platform && author.accountId)) return author; + return { + ...author, + accountId: normalizeAccountId(author.platform, author.accountId), + }; +} + +function authorIdentityKey( + platform: NonNullable, + accountId: string, +): string { + const normalizedAccountId = normalizeAccountId(platform, accountId); + const identity = + platform === "twitter" + ? normalizedAccountId.replace(/^@/, "") + : normalizedAccountId; + return `${platform}:${identity}`; +} + function authorInputKey(author: NewAuthor): string { return author.platform && author.accountId - ? `${author.platform}:${author.accountId}` + ? authorIdentityKey(author.platform, author.accountId) : `name:${author.name}`; } @@ -34,7 +62,8 @@ async function findOrCreateAuthorsBulk( inputs: NewAuthor[], ): Promise { const uniqueInputs = new Map(); - for (const input of inputs) { + for (const rawInput of inputs) { + const input = normalizeAuthorInput(rawInput); if (input.name.length > 0) { uniqueInputs.set(authorInputKey(input), input); } @@ -46,7 +75,9 @@ async function findOrCreateAuthorsBulk( if (input.platform && input.accountId) { const condition = and( eq(authorAccounts.platform, input.platform), - eq(authorAccounts.accountId, input.accountId), + input.platform === "twitter" + ? sql`lower(regexp_replace(${authorAccounts.accountId}, '^@', '')) = ${input.accountId.replace(/^@/, "")}` + : eq(authorAccounts.accountId, input.accountId), ); if (condition) identityConditions.push(condition); } @@ -61,7 +92,7 @@ async function findOrCreateAuthorsBulk( .where(or(...identityConditions)); for (const row of existingIdentities) { resolved.set( - `${row.account.platform}:${row.account.accountId}`, + authorIdentityKey(row.account.platform, row.account.accountId), row.author, ); } @@ -165,6 +196,34 @@ async function findOrCreateAuthorsBulk( .onConflictDoNothing(); } + const desiredNames = new Map(); + for (const [key, input] of uniqueInputs) { + const author = resolved.get(key); + if ( + author && + input.platform && + input.accountId && + author.name !== input.name + ) { + desiredNames.set(author.id, input.name); + } + } + + const refreshedAuthors = new Map(); + for (const [authorId, name] of desiredNames) { + const [updated] = await client + .update(authors) + .set({ name, updatedAt: new Date() }) + .where(eq(authors.id, authorId)) + .returning(); + if (updated) refreshedAuthors.set(authorId, updated); + } + + for (const [key, author] of resolved) { + const refreshed = refreshedAuthors.get(author.id); + if (refreshed) resolved.set(key, refreshed); + } + return [...uniqueInputs.keys()].flatMap((key) => { const author = resolved.get(key); return author ? [mapAuthor(author)] : []; From 2fb41e05b8294be4d4878dcd4443f9ee5745eb93 Mon Sep 17 00:00:00 2001 From: hmjn Date: Sat, 4 Jul 2026 01:25:54 +0900 Subject: [PATCH 18/19] fix: resolve merge conflict and address PR review comments - Refactor extractMetadataFromUrl to remove duplicate URL parsing (reuse extractTwitterAuthorIdFromStatusUrl) - Batch author display-name updates into single CASE UPDATE instead of per-author loop - Normalize Twitter accountId at save time (strip @ + lowercase) for index-friendly storage - Merge upstream conflict resolution for duplicate author account inserts --- apps/xtracter/src/content/twitter.ts | 20 +---- .../db/src/repositories/author-repository.ts | 79 ++++++++++++++++--- 2 files changed, 71 insertions(+), 28 deletions(-) diff --git a/apps/xtracter/src/content/twitter.ts b/apps/xtracter/src/content/twitter.ts index 47544172f..7106f4154 100644 --- a/apps/xtracter/src/content/twitter.ts +++ b/apps/xtracter/src/content/twitter.ts @@ -147,25 +147,9 @@ function findTweetArticle(element: HTMLElement): HTMLElement | null { } function extractMetadataFromUrl(): { authorId: string; tweetUrl: string } { - const url = new URL(window.location.href); - const pathParts = url.pathname.split("/").filter((p) => p); - - let tweetUrl = window.location.href; - - const MIN_PATH_PARTS_FOR_STATUS = 3; - const STATUS_PART_INDEX = 1; - const TWEET_ID_PART_INDEX = 2; - - if ( - pathParts.length >= MIN_PATH_PARTS_FOR_STATUS && - pathParts[STATUS_PART_INDEX] === "status" - ) { - tweetUrl = `${url.origin}/${pathParts[0]}/status/${pathParts[TWEET_ID_PART_INDEX]}`; - } - return { - authorId: extractTwitterAuthorIdFromStatusUrl(tweetUrl), - tweetUrl, + authorId: extractTwitterAuthorIdFromStatusUrl(window.location.href), + tweetUrl: window.location.href, }; } diff --git a/packages/db/src/repositories/author-repository.ts b/packages/db/src/repositories/author-repository.ts index b9477a091..8cab2ba87 100644 --- a/packages/db/src/repositories/author-repository.ts +++ b/packages/db/src/repositories/author-repository.ts @@ -28,7 +28,7 @@ function normalizeAccountId( accountId: string, ): string { if (platform !== "twitter") return accountId; - return accountId.toLowerCase(); + return accountId.replace(/^@/, "").toLowerCase(); } function normalizeAuthorInput(author: NewAuthor): NewAuthor { @@ -64,7 +64,7 @@ async function findOrCreateAuthorsBulk( const uniqueInputs = new Map(); for (const rawInput of inputs) { const input = normalizeAuthorInput(rawInput); - if (input.name.length > 0) { + if (input.name.trim().length > 0) { uniqueInputs.set(authorInputKey(input), input); } } @@ -147,6 +147,7 @@ async function findOrCreateAuthorsBulk( const accountsToInsert: (typeof authorAccounts.$inferInsert)[] = []; const authorsToInsert: (typeof authors.$inferInsert)[] = []; + const newlyCreatedAuthorByKey = new Map(); for (const [key, input] of unresolved) { const legacyByMatchingAccount = input.accountId ? legacyByAccount.get(input.accountId) @@ -174,6 +175,7 @@ async function findOrCreateAuthorsBulk( name: author.name, accountId: author.accountId, }); + newlyCreatedAuthorByKey.set(key, author.id); } resolved.set(key, author); @@ -190,10 +192,54 @@ async function findOrCreateAuthorsBulk( await client.insert(authors).values(authorsToInsert); } if (accountsToInsert.length > 0) { - await client + const insertedAccounts = await client .insert(authorAccounts) .values(accountsToInsert) - .onConflictDoNothing(); + .onConflictDoNothing() + .returning(); + const insertedKeys = new Set( + insertedAccounts.map( + (account) => authorIdentityKey(account.platform, account.accountId), + ), + ); + const attemptedKeys = new Set( + accountsToInsert.map( + (account) => authorIdentityKey(account.platform, account.accountId), + ), + ); + const conflicted = [...uniqueInputs.entries()].filter( + ([key, input]) => + input.platform && + input.accountId && + attemptedKeys.has(key) && + !insertedKeys.has(key), + ); + if (conflicted.length > 0) { + const conditions = conflicted.flatMap(([, input]) => { + if (!(input.platform && input.accountId)) return []; + const condition = and( + eq(authorAccounts.platform, input.platform), + eq(authorAccounts.accountId, input.accountId), + ); + return condition ? [condition] : []; + }); + const canonicalAccounts = await client + .select({ account: authorAccounts, author: authors }) + .from(authorAccounts) + .innerJoin(authors, eq(authorAccounts.authorId, authors.id)) + .where(or(...conditions)); + for (const row of canonicalAccounts) { + const key = authorIdentityKey(row.account.platform, row.account.accountId); + resolved.set(key, row.author); + } + const orphanIds = conflicted.flatMap(([key]) => { + const id = newlyCreatedAuthorByKey.get(key); + return id ? [id] : []; + }); + if (orphanIds.length > 0) { + await client.delete(authors).where(inArray(authors.id, orphanIds)); + } + } } const desiredNames = new Map(); @@ -210,13 +256,26 @@ async function findOrCreateAuthorsBulk( } const refreshedAuthors = new Map(); - for (const [authorId, name] of desiredNames) { - const [updated] = await client + if (desiredNames.size > 0) { + const ids = [...desiredNames.keys()]; + const cases = ids.map( + (id) => sql`WHEN ${id}::uuid THEN ${desiredNames.get(id)}`, + ); + await client .update(authors) - .set({ name, updatedAt: new Date() }) - .where(eq(authors.id, authorId)) - .returning(); - if (updated) refreshedAuthors.set(authorId, updated); + .set({ + name: sql`CASE ${sql.join(cases, sql` `)} END`, + updatedAt: new Date(), + }) + .where(inArray(authors.id, ids)); + + const updatedRows = await client + .select() + .from(authors) + .where(inArray(authors.id, ids)); + for (const row of updatedRows) { + refreshedAuthors.set(row.id, row); + } } for (const [key, author] of resolved) { From d460c0b3b702ceb3d726cef3f44588c7f34f1261 Mon Sep 17 00:00:00 2001 From: hmjn Date: Sat, 4 Jul 2026 01:35:01 +0900 Subject: [PATCH 19/19] fix: normalize legacy authors.accountId lookup and fix CASE WHEN SQL - Normalize legacy authors.accountId on lookup: build legacyByAccount map with normalized keys (strip @ + lowercase) for Twitter account matching - Search DB for both raw and @-prefixed accountId variants - Fix CASE WHEN SQL to compare authors.id instead of bare UUID cast --- .../db/src/repositories/author-repository.ts | 25 +++++++++++++++---- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/packages/db/src/repositories/author-repository.ts b/packages/db/src/repositories/author-repository.ts index 8cab2ba87..83e7030e0 100644 --- a/packages/db/src/repositories/author-repository.ts +++ b/packages/db/src/repositories/author-repository.ts @@ -51,6 +51,10 @@ function authorIdentityKey( return `${platform}:${identity}`; } +function normalizeLegacyAccountId(accountId: string): string { + return accountId.replace(/^@/, "").toLowerCase(); +} + function authorInputKey(author: NewAuthor): string { return author.platform && author.accountId ? authorIdentityKey(author.platform, author.accountId) @@ -111,7 +115,10 @@ async function findOrCreateAuthorsBulk( const names = [...new Set(unresolved.map(([, input]) => input.name))]; const legacyConditions: SQL[] = []; if (accountIds.length > 0) { - legacyConditions.push(inArray(authors.accountId, accountIds)); + const allVariants = [ + ...new Set(accountIds.flatMap((id) => [id, `@${id}`])), + ]; + legacyConditions.push(inArray(authors.accountId, allVariants)); } if (names.length > 0) { legacyConditions.push(inArray(authors.name, names)); @@ -125,9 +132,17 @@ async function findOrCreateAuthorsBulk( .where(or(...legacyConditions)) : []; const legacyByAccount = new Map( - legacyAuthors.flatMap((author) => - author.accountId ? [[author.accountId, author] as const] : [], - ), + legacyAuthors.flatMap((author) => { + if (!author.accountId) return []; + const normalized = normalizeLegacyAccountId(author.accountId); + const entries: [string, typeof authors.$inferSelect][] = [ + [normalized, author], + ]; + if (normalized !== author.accountId) { + entries.push([author.accountId, author]); + } + return entries; + }), ); const legacyByName = new Map( legacyAuthors.map((author) => [author.name, author] as const), @@ -259,7 +274,7 @@ async function findOrCreateAuthorsBulk( if (desiredNames.size > 0) { const ids = [...desiredNames.keys()]; const cases = ids.map( - (id) => sql`WHEN ${id}::uuid THEN ${desiredNames.get(id)}`, + (id) => sql`WHEN ${authors.id} = ${id}::uuid THEN ${desiredNames.get(id)}`, ); await client .update(authors)