Skip to content

Commit 69b273d

Browse files
etnoyetnoy
committed
batch transactions in sql
1 parent 02c5765 commit 69b273d

File tree

11 files changed

+516
-203
lines changed

11 files changed

+516
-203
lines changed

Diff for: ‎e2e/src/api/specs/library.e2e-spec.ts

+192-3
Original file line numberDiff line numberDiff line change
@@ -421,7 +421,7 @@ describe('/libraries', () => {
421421
const { status } = await request(app)
422422
.post(`/libraries/${library.id}/scan`)
423423
.set('Authorization', `Bearer ${admin.accessToken}`)
424-
.send({ refreshModifiedFiles: true });
424+
.send();
425425
expect(status).toBe(204);
426426

427427
await utils.waitForQueueFinish(admin.accessToken, 'library');
@@ -453,7 +453,7 @@ describe('/libraries', () => {
453453
const { status } = await request(app)
454454
.post(`/libraries/${library.id}/scan`)
455455
.set('Authorization', `Bearer ${admin.accessToken}`)
456-
.send({ refreshModifiedFiles: true });
456+
.send();
457457
expect(status).toBe(204);
458458

459459
await utils.waitForQueueFinish(admin.accessToken, 'library');
@@ -577,7 +577,7 @@ describe('/libraries', () => {
577577
]);
578578
});
579579

580-
it('should not trash an online asset', async () => {
580+
it('should not set an asset offline if its file exists, is in an import path, and not covered by an exclusion pattern', async () => {
581581
const library = await utils.createLibrary(admin.accessToken, {
582582
ownerId: admin.userId,
583583
importPaths: [`${testAssetDirInternal}/temp`],
@@ -601,6 +601,195 @@ describe('/libraries', () => {
601601

602602
expect(assets).toEqual(assetsBefore);
603603
});
604+
605+
it('should set an offline asset to online if its file exists, is in an import path, and not covered by an exclusion pattern', async () => {
606+
utils.createImageFile(`${testAssetDir}/temp/offline/offline.png`);
607+
608+
const library = await utils.createLibrary(admin.accessToken, {
609+
ownerId: admin.userId,
610+
importPaths: [`${testAssetDirInternal}/temp/offline`],
611+
});
612+
613+
await scan(admin.accessToken, library.id);
614+
await utils.waitForQueueFinish(admin.accessToken, 'library');
615+
616+
const { assets } = await utils.searchAssets(admin.accessToken, { libraryId: library.id });
617+
618+
utils.renameImageFile(`${testAssetDir}/temp/offline/offline.png`, `${testAssetDir}/temp/offline.png`);
619+
620+
{
621+
const { status } = await request(app)
622+
.post(`/libraries/${library.id}/scan`)
623+
.set('Authorization', `Bearer ${admin.accessToken}`)
624+
.send();
625+
expect(status).toBe(204);
626+
}
627+
628+
await utils.waitForQueueFinish(admin.accessToken, 'library');
629+
630+
const offlineAsset = await utils.getAssetInfo(admin.accessToken, assets.items[0].id);
631+
expect(offlineAsset.isTrashed).toBe(true);
632+
expect(offlineAsset.originalPath).toBe(`${testAssetDirInternal}/temp/offline/offline.png`);
633+
expect(offlineAsset.isOffline).toBe(true);
634+
635+
{
636+
const { assets } = await utils.searchAssets(admin.accessToken, { libraryId: library.id, withDeleted: true });
637+
expect(assets.count).toBe(1);
638+
}
639+
640+
utils.renameImageFile(`${testAssetDir}/temp/offline.png`, `${testAssetDir}/temp/offline/offline.png`);
641+
642+
{
643+
const { status } = await request(app)
644+
.post(`/libraries/${library.id}/scan`)
645+
.set('Authorization', `Bearer ${admin.accessToken}`)
646+
.send();
647+
expect(status).toBe(204);
648+
}
649+
650+
await utils.waitForQueueFinish(admin.accessToken, 'library');
651+
652+
const backOnlineAsset = await utils.getAssetInfo(admin.accessToken, assets.items[0].id);
653+
654+
expect(backOnlineAsset.isTrashed).toBe(false);
655+
expect(backOnlineAsset.originalPath).toBe(`${testAssetDirInternal}/temp/offline/offline.png`);
656+
expect(backOnlineAsset.isOffline).toBe(false);
657+
658+
{
659+
const { assets } = await utils.searchAssets(admin.accessToken, { libraryId: library.id });
660+
expect(assets.count).toBe(1);
661+
}
662+
});
663+
664+
it('should not set an offline asset to online if its file exists, is not covered by an exclusion pattern, but is outside of all import paths', async () => {
665+
utils.createImageFile(`${testAssetDir}/temp/offline/offline.png`);
666+
667+
const library = await utils.createLibrary(admin.accessToken, {
668+
ownerId: admin.userId,
669+
importPaths: [`${testAssetDirInternal}/temp/offline`],
670+
});
671+
672+
await scan(admin.accessToken, library.id);
673+
await utils.waitForQueueFinish(admin.accessToken, 'library');
674+
675+
const { assets } = await utils.searchAssets(admin.accessToken, { libraryId: library.id });
676+
677+
utils.renameImageFile(`${testAssetDir}/temp/offline/offline.png`, `${testAssetDir}/temp/offline.png`);
678+
679+
{
680+
const { status } = await request(app)
681+
.post(`/libraries/${library.id}/scan`)
682+
.set('Authorization', `Bearer ${admin.accessToken}`)
683+
.send();
684+
expect(status).toBe(204);
685+
}
686+
687+
await utils.waitForQueueFinish(admin.accessToken, 'library');
688+
689+
{
690+
const { assets } = await utils.searchAssets(admin.accessToken, { libraryId: library.id, withDeleted: true });
691+
expect(assets.count).toBe(1);
692+
}
693+
694+
const offlineAsset = await utils.getAssetInfo(admin.accessToken, assets.items[0].id);
695+
696+
expect(offlineAsset.isTrashed).toBe(true);
697+
expect(offlineAsset.originalPath).toBe(`${testAssetDirInternal}/temp/offline/offline.png`);
698+
expect(offlineAsset.isOffline).toBe(true);
699+
700+
utils.renameImageFile(`${testAssetDir}/temp/offline.png`, `${testAssetDir}/temp/offline/offline.png`);
701+
702+
utils.createDirectory(`${testAssetDir}/temp/another-path/`);
703+
704+
await utils.updateLibrary(admin.accessToken, library.id, {
705+
importPaths: [`${testAssetDirInternal}/temp/another-path`],
706+
});
707+
708+
{
709+
const { status } = await request(app)
710+
.post(`/libraries/${library.id}/scan`)
711+
.set('Authorization', `Bearer ${admin.accessToken}`)
712+
.send();
713+
expect(status).toBe(204);
714+
}
715+
716+
await utils.waitForQueueFinish(admin.accessToken, 'library');
717+
718+
const stillOfflineAsset = await utils.getAssetInfo(admin.accessToken, assets.items[0].id);
719+
720+
expect(stillOfflineAsset.isTrashed).toBe(true);
721+
expect(stillOfflineAsset.originalPath).toBe(`${testAssetDirInternal}/temp/offline/offline.png`);
722+
expect(stillOfflineAsset.isOffline).toBe(true);
723+
724+
{
725+
const { assets } = await utils.searchAssets(admin.accessToken, { libraryId: library.id, withDeleted: true });
726+
expect(assets.count).toBe(1);
727+
}
728+
729+
utils.removeDirectory(`${testAssetDir}/temp/another-path/`);
730+
});
731+
732+
it('should not set an offline asset to online if its file exists, is in an import path, but is covered by an exclusion pattern', async () => {
733+
utils.createImageFile(`${testAssetDir}/temp/offline/offline.png`);
734+
735+
const library = await utils.createLibrary(admin.accessToken, {
736+
ownerId: admin.userId,
737+
importPaths: [`${testAssetDirInternal}/temp/offline`],
738+
});
739+
740+
await scan(admin.accessToken, library.id);
741+
await utils.waitForQueueFinish(admin.accessToken, 'library');
742+
743+
const { assets } = await utils.searchAssets(admin.accessToken, { libraryId: library.id });
744+
745+
utils.renameImageFile(`${testAssetDir}/temp/offline/offline.png`, `${testAssetDir}/temp/offline.png`);
746+
747+
{
748+
const { status } = await request(app)
749+
.post(`/libraries/${library.id}/scan`)
750+
.set('Authorization', `Bearer ${admin.accessToken}`)
751+
.send();
752+
expect(status).toBe(204);
753+
}
754+
755+
await utils.waitForQueueFinish(admin.accessToken, 'library');
756+
757+
{
758+
const { assets } = await utils.searchAssets(admin.accessToken, { libraryId: library.id, withDeleted: true });
759+
expect(assets.count).toBe(1);
760+
}
761+
762+
const offlineAsset = await utils.getAssetInfo(admin.accessToken, assets.items[0].id);
763+
764+
expect(offlineAsset.isTrashed).toBe(true);
765+
expect(offlineAsset.originalPath).toBe(`${testAssetDirInternal}/temp/offline/offline.png`);
766+
expect(offlineAsset.isOffline).toBe(true);
767+
768+
utils.renameImageFile(`${testAssetDir}/temp/offline.png`, `${testAssetDir}/temp/offline/offline.png`);
769+
770+
await utils.updateLibrary(admin.accessToken, library.id, { exclusionPatterns: ['**/offline/**'] });
771+
772+
{
773+
const { status } = await request(app)
774+
.post(`/libraries/${library.id}/scan`)
775+
.set('Authorization', `Bearer ${admin.accessToken}`)
776+
.send();
777+
expect(status).toBe(204);
778+
}
779+
780+
await utils.waitForQueueFinish(admin.accessToken, 'library');
781+
782+
const stillOfflineAsset = await utils.getAssetInfo(admin.accessToken, assets.items[0].id);
783+
784+
expect(stillOfflineAsset.isTrashed).toBe(true);
785+
expect(stillOfflineAsset.originalPath).toBe(`${testAssetDirInternal}/temp/offline/offline.png`);
786+
expect(stillOfflineAsset.isOffline).toBe(true);
787+
788+
{
789+
const { assets } = await utils.searchAssets(admin.accessToken, { libraryId: library.id, withDeleted: true });
790+
expect(assets.count).toBe(1);
791+
}
792+
});
604793
});
605794

606795
describe('POST /libraries/:id/validate', () => {

Diff for: ‎e2e/src/utils.ts

+14-1
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import {
1010
Permission,
1111
PersonCreateDto,
1212
SharedLinkCreateDto,
13+
UpdateLibraryDto,
1314
UserAdminCreateDto,
1415
UserPreferencesUpdateDto,
1516
ValidateLibraryDto,
@@ -35,14 +36,15 @@ import {
3536
updateAlbumUser,
3637
updateAssets,
3738
updateConfig,
39+
updateLibrary,
3840
updateMyPreferences,
3941
upsertTags,
4042
validate,
4143
} from '@immich/sdk';
4244
import { BrowserContext } from '@playwright/test';
4345
import { exec, spawn } from 'node:child_process';
4446
import { createHash } from 'node:crypto';
45-
import { existsSync, mkdirSync, rmSync, writeFileSync } from 'node:fs';
47+
import { existsSync, mkdirSync, renameSync, rmSync, writeFileSync } from 'node:fs';
4648
import { tmpdir } from 'node:os';
4749
import path, { dirname } from 'node:path';
4850
import { setTimeout as setAsyncTimeout } from 'node:timers/promises';
@@ -392,6 +394,14 @@ export const utils = {
392394
rmSync(path);
393395
},
394396

397+
renameImageFile: (oldPath: string, newPath: string) => {
398+
if (!existsSync(oldPath)) {
399+
return;
400+
}
401+
402+
renameSync(oldPath, newPath);
403+
},
404+
395405
removeDirectory: (path: string) => {
396406
if (!existsSync(path)) {
397407
return;
@@ -444,6 +454,9 @@ export const utils = {
444454
createLibrary: (accessToken: string, dto: CreateLibraryDto) =>
445455
createLibrary({ createLibraryDto: dto }, { headers: asBearerAuth(accessToken) }),
446456

457+
updateLibrary: (accessToken: string, id: string, dto: UpdateLibraryDto) =>
458+
updateLibrary({ id, updateLibraryDto: dto }, { headers: asBearerAuth(accessToken) }),
459+
447460
validateLibrary: (accessToken: string, id: string, dto: ValidateLibraryDto) =>
448461
validate({ id, validateLibraryDto: dto }, { headers: asBearerAuth(accessToken) }),
449462

Diff for: ‎server/src/interfaces/asset.interface.ts

+2-1
Original file line numberDiff line numberDiff line change
@@ -193,5 +193,6 @@ export interface IAssetRepository {
193193
getChangedDeltaSync(options: AssetDeltaSyncOptions): Promise<AssetEntity[]>;
194194
upsertFile(file: UpsertFileOptions): Promise<void>;
195195
upsertFiles(files: UpsertFileOptions[]): Promise<void>;
196-
updateOffline(pagination: PaginationOptions, library: LibraryEntity): Paginated<AssetEntity>;
196+
updateOffline(library: LibraryEntity): Promise<UpdateResult>;
197+
getNewPaths(libraryId: string, paths: string[]): Promise<string[]>;
197198
}

Diff for: ‎server/src/interfaces/job.interface.ts

+9-3
Original file line numberDiff line numberDiff line change
@@ -143,12 +143,18 @@ export interface IAssetDeleteJob extends IEntityJob {
143143
deleteOnDisk: boolean;
144144
}
145145

146-
export interface ILibraryFileJob extends IEntityJob {
146+
export interface ILibraryFileJob {
147+
libraryId: string;
147148
ownerId: string;
148149
assetPath: string;
149150
}
150151

151-
export interface IBulkEntityJob extends IBaseJob {
152+
export interface ILibraryBulkIdsJob {
153+
libraryId: string;
154+
assetIds: string[];
155+
}
156+
157+
export interface IBulkEntityJob {
152158
ids: string[];
153159
}
154160

@@ -287,7 +293,7 @@ export type JobItem =
287293
| { name: JobName.LIBRARY_SYNC_FILE; data: ILibraryFileJob }
288294
| { name: JobName.LIBRARY_QUEUE_SYNC_FILES; data: IEntityJob }
289295
| { name: JobName.LIBRARY_QUEUE_SYNC_ASSETS; data: IEntityJob }
290-
| { name: JobName.LIBRARY_SYNC_ASSETS; data: IBulkEntityJob }
296+
| { name: JobName.LIBRARY_SYNC_ASSETS; data: ILibraryBulkIdsJob }
291297
| { name: JobName.LIBRARY_DELETE; data: IEntityJob }
292298
| { name: JobName.LIBRARY_QUEUE_SYNC_ALL; data?: IBaseJob }
293299
| { name: JobName.LIBRARY_QUEUE_CLEANUP; data: IBaseJob }

Diff for: ‎server/src/interfaces/library.interface.ts

+7
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,15 @@
1+
import { ADDED_IN_PREFIX } from 'src/constants';
12
import { LibraryStatsResponseDto } from 'src/dtos/library.dto';
23
import { LibraryEntity } from 'src/entities/library.entity';
34

45
export const ILibraryRepository = 'ILibraryRepository';
56

7+
export enum AssetSyncResult {
8+
DO_NOTHING,
9+
UPDATE,
10+
OFFLINE,
11+
}
12+
613
export interface ILibraryRepository {
714
getAll(withDeleted?: boolean): Promise<LibraryEntity[]>;
815
getAllDeleted(): Promise<LibraryEntity[]>;

0 commit comments

Comments
 (0)