feat(server): generate all thumbnails for an asset in one job (#13012)

* wip

cleanup

add success logs, rename method

do thumbhash too

fixes

fix tests

handle `notify`

wip refactor

refactor

* update tests

* update sql

* pr feedback

* remove unused code

* formatting

server/src/config.ts

@@ -20,7 +20,7 @@ import {
   VideoContainer,
 } from 'src/enum';
 import { ConcurrentQueueName, QueueName } from 'src/interfaces/job.interface';
-import { ImageOutputConfig } from 'src/interfaces/media.interface';
+import { ImageOptions } from 'src/interfaces/media.interface';
 
 export interface SystemConfig {
   ffmpeg: {
@@ -110,8 +110,8 @@ export interface SystemConfig {
     template: string;
   };
   image: {
-    thumbnail: ImageOutputConfig;
-    preview: ImageOutputConfig;
+    thumbnail: ImageOptions;
+    preview: ImageOptions;
     colorspace: Colorspace;
     extractEmbedded: boolean;
   };

server/src/dtos/system-config.dto.ts

@@ -492,7 +492,7 @@ class SystemConfigGeneratedImageDto {
   size!: number;
 }
 
-class SystemConfigImageDto {
+export class SystemConfigImageDto {
   @Type(() => SystemConfigGeneratedImageDto)
   @ValidateNested()
   @IsObject()

server/src/interfaces/asset.interface.ts

@@ -141,6 +141,12 @@ export interface AssetUpdateDuplicateOptions {
   duplicateIds: string[];
 }
 
+export interface UpsertFileOptions {
+  assetId: string;
+  type: AssetFileType;
+  path: string;
+}
+
 export type AssetPathEntity = Pick<AssetEntity, 'id' | 'originalPath' | 'isOffline'>;
 
 export const IAssetRepository = 'IAssetRepository';
@@ -194,5 +200,6 @@ export interface IAssetRepository {
   getDuplicates(options: AssetBuilderOptions): Promise<AssetEntity[]>;
   getAllForUserFullSync(options: AssetFullSyncOptions): Promise<AssetEntity[]>;
   getChangedDeltaSync(options: AssetDeltaSyncOptions): Promise<AssetEntity[]>;
-  upsertFile(options: { assetId: string; type: AssetFileType; path: string }): Promise<void>;
+  upsertFile(file: UpsertFileOptions): Promise<void>;
+  upsertFiles(files: UpsertFileOptions[]): Promise<void>;
 }

server/src/interfaces/job.interface.ts

@@ -37,9 +37,7 @@ export enum JobName {
 
   // thumbnails
   QUEUE_GENERATE_THUMBNAILS = 'queue-generate-thumbnails',
-  GENERATE_PREVIEW = 'generate-preview',
-  GENERATE_THUMBNAIL = 'generate-thumbnail',
-  GENERATE_THUMBHASH = 'generate-thumbhash',
+  GENERATE_THUMBNAILS = 'generate-thumbnails',
   GENERATE_PERSON_THUMBNAIL = 'generate-person-thumbnail',
 
   // metadata
@@ -212,9 +210,7 @@ export type JobItem =
 
   // Thumbnails
   | { name: JobName.QUEUE_GENERATE_THUMBNAILS; data: IBaseJob }
-  | { name: JobName.GENERATE_PREVIEW; data: IEntityJob }
-  | { name: JobName.GENERATE_THUMBNAIL; data: IEntityJob }
-  | { name: JobName.GENERATE_THUMBHASH; data: IEntityJob }
+  | { name: JobName.GENERATE_THUMBNAILS; data: IEntityJob }
 
   // User
   | { name: JobName.USER_DELETE_CHECK; data?: IBaseJob }

server/src/interfaces/media.interface.ts

@@ -10,16 +10,44 @@ export interface CropOptions {
   height: number;
 }
 
-export interface ImageOutputConfig {
+export interface ImageOptions {
   format: ImageFormat;
   quality: number;
   size: number;
 }
 
-export interface ThumbnailOptions extends ImageOutputConfig {
+export interface RawImageInfo {
+  width: number;
+  height: number;
+  channels: 1 | 2 | 3 | 4;
+}
+
+interface DecodeImageOptions {
+  colorspace: string;
+  crop?: CropOptions;
+  processInvalidImages: boolean;
+  raw?: RawImageInfo;
+}
+
+export interface DecodeToBufferOptions extends DecodeImageOptions {
+  size: number;
+}
+
+export type GenerateThumbnailOptions = ImageOptions & DecodeImageOptions;
+
+export type GenerateThumbnailFromBufferOptions = GenerateThumbnailOptions & { raw: RawImageInfo };
+
+export type GenerateThumbhashOptions = DecodeImageOptions;
+
+export type GenerateThumbhashFromBufferOptions = GenerateThumbhashOptions & { raw: RawImageInfo };
+
+export interface GenerateThumbnailsOptions {
   colorspace: string;
   crop?: CropOptions;
+  preview?: ImageOptions;
   processInvalidImages: boolean;
+  thumbhash?: boolean;
+  thumbnail?: ImageOptions;
 }
 
 export interface VideoStreamInfo {
@@ -78,6 +106,11 @@ export interface BitrateDistribution {
   unit: string;
 }
 
+export interface ImageBuffer {
+  data: Buffer;
+  info: RawImageInfo;
+}
+
 export interface VideoCodecSWConfig {
   getCommand(target: TranscodeTarget, videoStream: VideoStreamInfo, audioStream: AudioStreamInfo): TranscodeCommand;
 }
@@ -93,8 +126,11 @@ export interface ProbeOptions {
 export interface IMediaRepository {
   // image
   extract(input: string, output: string): Promise<boolean>;
-  generateThumbnail(input: string | Buffer, output: string, options: ThumbnailOptions): Promise<void>;
-  generateThumbhash(imagePath: string): Promise<Buffer>;
+  decodeImage(input: string, options: DecodeToBufferOptions): Promise<ImageBuffer>;
+  generateThumbnail(input: string, options: GenerateThumbnailOptions, outputFile: string): Promise<void>;
+  generateThumbnail(input: Buffer, options: GenerateThumbnailFromBufferOptions, outputFile: string): Promise<void>;
+  generateThumbhash(input: string, options: GenerateThumbhashOptions): Promise<Buffer>;
+  generateThumbhash(input: Buffer, options: GenerateThumbhashFromBufferOptions): Promise<Buffer>;
   getImageDimensions(input: string): Promise<ImageDimensions>;
 
   // video

server/src/queries/asset.repository.sql

@@ -1132,3 +1132,27 @@ RETURNING
   "id",
   "createdAt",
   "updatedAt"
+
+-- AssetRepository.upsertFiles
+INSERT INTO
+  "asset_files" (
+    "id",
+    "assetId",
+    "createdAt",
+    "updatedAt",
+    "type",
+    "path"
+  )
+VALUES
+  (DEFAULT, $1, DEFAULT, DEFAULT, $2, $3)
+ON CONFLICT ("assetId", "type") DO
+UPDATE
+SET
+  "assetId" = EXCLUDED."assetId",
+  "type" = EXCLUDED."type",
+  "path" = EXCLUDED."path",
+  "updatedAt" = DEFAULT
+RETURNING
+  "id",
+  "createdAt",
+  "updatedAt"

server/src/repositories/asset.repository.ts

@@ -801,7 +801,12 @@ export class AssetRepository implements IAssetRepository {
   }
 
   @GenerateSql({ params: [{ assetId: DummyValue.UUID, type: AssetFileType.PREVIEW, path: '/path/to/file' }] })
-  async upsertFile({ assetId, type, path }: { assetId: string; type: AssetFileType; path: string }): Promise<void> {
-    await this.fileRepository.upsert({ assetId, type, path }, { conflictPaths: ['assetId', 'type'] });
+  async upsertFile(file: { assetId: string; type: AssetFileType; path: string }): Promise<void> {
+    await this.fileRepository.upsert(file, { conflictPaths: ['assetId', 'type'] });
+  }
+
+  @GenerateSql({ params: [{ assetId: DummyValue.UUID, type: AssetFileType.PREVIEW, path: '/path/to/file' }] })
+  async upsertFiles(files: { assetId: string; type: AssetFileType; path: string }[]): Promise<void> {
+    await this.fileRepository.upsert(files, { conflictPaths: ['assetId', 'type'] });
   }
 }

server/src/repositories/job.repository.ts

@@ -36,9 +36,7 @@ export const JOBS_TO_QUEUE: Record<JobName, QueueName> = {
 
   // thumbnails
   [JobName.QUEUE_GENERATE_THUMBNAILS]: QueueName.THUMBNAIL_GENERATION,
-  [JobName.GENERATE_PREVIEW]: QueueName.THUMBNAIL_GENERATION,
-  [JobName.GENERATE_THUMBNAIL]: QueueName.THUMBNAIL_GENERATION,
-  [JobName.GENERATE_THUMBHASH]: QueueName.THUMBNAIL_GENERATION,
+  [JobName.GENERATE_THUMBNAILS]: QueueName.THUMBNAIL_GENERATION,
   [JobName.GENERATE_PERSON_THUMBNAIL]: QueueName.THUMBNAIL_GENERATION,
 
   // tags

server/src/repositories/media.repository.ts

@@ -8,10 +8,12 @@ import sharp from 'sharp';
 import { Colorspace, LogLevel } from 'src/enum';
 import { ILoggerRepository } from 'src/interfaces/logger.interface';
 import {
+  DecodeToBufferOptions,
+  GenerateThumbhashOptions,
+  GenerateThumbnailOptions,
   IMediaRepository,
   ImageDimensions,
   ProbeOptions,
-  ThumbnailOptions,
   TranscodeCommand,
   VideoInfo,
 } from 'src/interfaces/media.interface';
@@ -57,19 +59,12 @@ export class MediaRepository implements IMediaRepository {
     return true;
   }
 
-  async generateThumbnail(input: string | Buffer, output: string, options: ThumbnailOptions): Promise<void> {
-    // some invalid images can still be processed by sharp, but we want to fail on them by default to avoid crashes
-    const pipeline = sharp(input, { failOn: options.processInvalidImages ? 'none' : 'error', limitInputPixels: false })
-      .pipelineColorspace(options.colorspace === Colorspace.SRGB ? 'srgb' : 'rgb16')
-      .rotate();
-
-    if (options.crop) {
-      pipeline.extract(options.crop);
-    }
+  decodeImage(input: string, options: DecodeToBufferOptions) {
+    return this.getImageDecodingPipeline(input, options).raw().toBuffer({ resolveWithObject: true });
+  }
 
-    await pipeline
-      .resize(options.size, options.size, { fit: 'outside', withoutEnlargement: true })
-      .withIccProfile(options.colorspace)
+  async generateThumbnail(input: string | Buffer, options: GenerateThumbnailOptions, output: string): Promise<void> {
+    await this.getImageDecodingPipeline(input, options)
       .toFormat(options.format, {
         quality: options.quality,
         // this is default in libvips (except the threshold is 90), but we need to set it manually in sharp
@@ -78,6 +73,40 @@ export class MediaRepository implements IMediaRepository {
       .toFile(output);
   }
 
+  private getImageDecodingPipeline(input: string | Buffer, options: DecodeToBufferOptions) {
+    let pipeline = sharp(input, {
+      // some invalid images can still be processed by sharp, but we want to fail on them by default to avoid crashes
+      failOn: options.processInvalidImages ? 'none' : 'error',
+      limitInputPixels: false,
+      raw: options.raw,
+    });
+
+    if (!options.raw) {
+      pipeline = pipeline
+        .pipelineColorspace(options.colorspace === Colorspace.SRGB ? 'srgb' : 'rgb16')
+        .withIccProfile(options.colorspace)
+        .rotate();
+    }
+
+    if (options.crop) {
+      pipeline = pipeline.extract(options.crop);
+    }
+
+    return pipeline.resize(options.size, options.size, { fit: 'outside', withoutEnlargement: true });
+  }
+
+  async generateThumbhash(input: string | Buffer, options: GenerateThumbhashOptions): Promise<Buffer> {
+    const [{ rgbaToThumbHash }, { data, info }] = await Promise.all([
+      import('thumbhash'),
+      sharp(input, options)
+        .resize(100, 100, { fit: 'inside', withoutEnlargement: true })
+        .raw()
+        .ensureAlpha()
+        .toBuffer({ resolveWithObject: true }),
+    ]);
+    return Buffer.from(rgbaToThumbHash(info.width, info.height, data));
+  }
+
   async probe(input: string, options?: ProbeOptions): Promise<VideoInfo> {
     const results = await probe(input, options?.countFrames ? ['-count_packets'] : []); // gets frame count quickly: https://stackoverflow.com/a/28376817
     return {
@@ -150,19 +179,6 @@ export class MediaRepository implements IMediaRepository {
     });
   }
 
-  async generateThumbhash(imagePath: string): Promise<Buffer> {
-    const maxSize = 100;
-
-    const { data, info } = await sharp(imagePath)
-      .resize(maxSize, maxSize, { fit: 'inside', withoutEnlargement: true })
-      .raw()
-      .ensureAlpha()
-      .toBuffer({ resolveWithObject: true });
-
-    const thumbhash = await import('thumbhash');
-    return Buffer.from(thumbhash.rgbaToThumbHash(info.width, info.height, data));
-  }
-
   async getImageDimensions(input: string): Promise<ImageDimensions> {
     const { width = 0, height = 0 } = await sharp(input).metadata();
     return { width, height };

server/src/services/asset.service.spec.ts

@@ -395,7 +395,7 @@ describe(AssetService.name, () => {
     it('should run the refresh thumbnails job', async () => {
       accessMock.asset.checkOwnerAccess.mockResolvedValue(new Set(['asset-1']));
       await sut.run(authStub.admin, { assetIds: ['asset-1'], name: AssetJobName.REGENERATE_THUMBNAIL });
-      expect(jobMock.queueAll).toHaveBeenCalledWith([{ name: JobName.GENERATE_PREVIEW, data: { id: 'asset-1' } }]);
+      expect(jobMock.queueAll).toHaveBeenCalledWith([{ name: JobName.GENERATE_THUMBNAILS, data: { id: 'asset-1' } }]);
     });
 
     it('should run the transcode video', async () => {

server/src/services/asset.service.ts

@@ -322,7 +322,7 @@ export class AssetService {
         }
 
         case AssetJobName.REGENERATE_THUMBNAIL: {
-          jobs.push({ name: JobName.GENERATE_PREVIEW, data: { id } });
+          jobs.push({ name: JobName.GENERATE_THUMBNAILS, data: { id } });
           break;
         }
 

server/src/services/job.service.spec.ts

@@ -288,7 +288,7 @@ describe(JobService.name, () => {
       },
       {
         item: { name: JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE, data: { id: 'asset-1', source: 'upload' } },
-        jobs: [JobName.GENERATE_PREVIEW],
+        jobs: [JobName.GENERATE_THUMBNAILS],
       },
       {
         item: { name: JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE, data: { id: 'asset-1' } },
@@ -299,28 +299,16 @@ describe(JobService.name, () => {
         jobs: [],
       },
       {
-        item: { name: JobName.GENERATE_PREVIEW, data: { id: 'asset-1' } },
-        jobs: [JobName.GENERATE_THUMBNAIL, JobName.GENERATE_THUMBHASH],
+        item: { name: JobName.GENERATE_THUMBNAILS, data: { id: 'asset-1' } },
+        jobs: [],
       },
       {
-        item: { name: JobName.GENERATE_PREVIEW, data: { id: 'asset-1', source: 'upload' } },
-        jobs: [
-          JobName.GENERATE_THUMBNAIL,
-          JobName.GENERATE_THUMBHASH,
-          JobName.SMART_SEARCH,
-          JobName.FACE_DETECTION,
-          JobName.VIDEO_CONVERSION,
-        ],
+        item: { name: JobName.GENERATE_THUMBNAILS, data: { id: 'asset-1', source: 'upload' } },
+        jobs: [JobName.SMART_SEARCH, JobName.FACE_DETECTION, JobName.VIDEO_CONVERSION],
       },
       {
-        item: { name: JobName.GENERATE_PREVIEW, data: { id: 'asset-live-image', source: 'upload' } },
-        jobs: [
-          JobName.GENERATE_THUMBNAIL,
-          JobName.GENERATE_THUMBHASH,
-          JobName.SMART_SEARCH,
-          JobName.FACE_DETECTION,
-          JobName.VIDEO_CONVERSION,
-        ],
+        item: { name: JobName.GENERATE_THUMBNAILS, data: { id: 'asset-live-image', source: 'upload' } },
+        jobs: [JobName.SMART_SEARCH, JobName.FACE_DETECTION, JobName.VIDEO_CONVERSION],
       },
       {
         item: { name: JobName.SMART_SEARCH, data: { id: 'asset-1' } },
@@ -338,11 +326,11 @@ describe(JobService.name, () => {
 
     for (const { item, jobs } of tests) {
       it(`should queue ${jobs.length} jobs when a ${item.name} job finishes successfully`, async () => {
-        if (item.name === JobName.GENERATE_PREVIEW && item.data.source === 'upload') {
+        if (item.name === JobName.GENERATE_THUMBNAILS && item.data.source === 'upload') {
           if (item.data.id === 'asset-live-image') {
-            assetMock.getByIds.mockResolvedValue([assetStub.livePhotoStillAsset]);
+            assetMock.getByIdsWithAllRelations.mockResolvedValue([assetStub.livePhotoStillAsset]);
           } else {
-            assetMock.getByIds.mockResolvedValue([assetStub.livePhotoMotionAsset]);
+            assetMock.getByIdsWithAllRelations.mockResolvedValue([assetStub.livePhotoMotionAsset]);
           }
         }
 
@@ -361,7 +349,7 @@ describe(JobService.name, () => {
         }
       });
 
-      it(`should not queue any jobs when ${item.name} finishes with 'false'`, async () => {
+      it(`should not queue any jobs when ${item.name} fails`, async () => {
         await sut.init(makeMockHandlers(JobStatus.FAILED));
         await jobMock.addHandler.mock.calls[0][2](item);